diff options
2022 files changed, 27977 insertions, 15863 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 863ce8dd9de..12b5a615beb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,10 +149,7 @@ add_subdirectory(vespaclient-java) add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) -if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin" AND - NOT DEFINED VESPA_USE_SANITIZER) - add_subdirectory(vespamalloc) -endif() +add_subdirectory(vespamalloc) add_subdirectory(vsm) add_subdirectory(zkfacade) add_subdirectory(zookeeper-command-line-client) diff --git a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt index cee859c070c..8b0e5a3882e 100644 --- a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt +++ b/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt @@ -1,3 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_test(NAME ann_benchmark_test NO_VALGRIND COMMAND ${PYTHON_EXECUTABLE} -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS vespa_ann_benchmark) +if(NOT DEFINED VESPA_USE_SANITIZER) + vespa_add_test(NAME ann_benchmark_test NO_VALGRIND COMMAND ${PYTHON_EXECUTABLE} -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS vespa_ann_benchmark) +endif() diff --git a/application-model/pom.xml b/application-model/pom.xml index 19823a12e67..3abf9851d5c 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -31,6 +31,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> </dependencies> <build> diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java index 704217843fb..ad73905c395 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -20,17 +19,14 @@ public enum InfrastructureApplication { CONFIG_SERVER("zone-config-servers", NodeType.config), PROXY_HOST("proxy-host", NodeType.proxyhost), PROXY("routing", NodeType.proxy), - TENANT_HOST("tenant-host", NodeType.host), - DEV_HOST("dev-host", NodeType.devhost); + TENANT_HOST("tenant-host", NodeType.host); private final ApplicationId id; private final NodeType nodeType; /** Returns all applications that MAY be encountered in hosted Vespa, e.g. not DEV_HOST. */ - public static List<InfrastructureApplication> inHosted() { - return Stream.of(values()) - .filter(application -> application != DEV_HOST) - .collect(Collectors.toList()); + public static List<InfrastructureApplication> toList() { + return List.of(values()); } public static InfrastructureApplication withNodeType(NodeType nodeType) { diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index a021577f838..45989770da9 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -75,17 +75,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/application/src/test/java/com/yahoo/application/TestDocProc.java b/application/src/test/java/com/yahoo/application/TestDocProc.java index 37d428c773e..c2d24d38115 100644 --- a/application/src/test/java/com/yahoo/application/TestDocProc.java +++ b/application/src/test/java/com/yahoo/application/TestDocProc.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application; -import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.Processing; import com.yahoo.document.config.DocumentmanagerConfig; @@ -12,7 +11,6 @@ import com.yahoo.document.config.DocumentmanagerConfig; public class TestDocProc extends DocumentProcessor { public TestDocProc(DocumentmanagerConfig config) { - new ConfigSubscriber().subscribe(DocumentmanagerConfig.class, "client"); } @Override diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java index 193fec3f7bb..f76008b4e02 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application.container; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import com.yahoo.application.Application; import com.yahoo.application.Networking; import com.yahoo.application.container.handler.Request; @@ -17,6 +18,7 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * Verify that we can create a JDisc (and hence Application) instance capable of doing model evaluation @@ -37,6 +39,7 @@ public class ContainerModelEvaluationTest { @Test public void testCreateApplicationInstanceWithModelEvaluation() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); try (Application application = Application.fromApplicationPackage(new File("src/test/app-packages/model-evaluation"), Networking.disable)) { diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index a44f1845d4e..08521216736 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -93,6 +93,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>orchestrator</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java index 4d37d263fab..941d8d46dcd 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.inject.Inject; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.RestApiRequestHandler; @@ -27,7 +27,7 @@ public class IdentityProviderRequestHandler extends RestApiRequestHandler<Identi private final InstanceValidator instanceValidator; @Inject - public IdentityProviderRequestHandler(LoggingRequestHandler.Context context, + public IdentityProviderRequestHandler(ThreadedHttpRequestHandler.Context context, IdentityDocumentGenerator documentGenerator, InstanceValidator instanceValidator) { super(context, IdentityProviderRequestHandler::createRestApi); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index fdb598639a7..85fdd6e6d43 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.ca.restapi; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.server.jetty.RequestUtils; @@ -50,7 +50,7 @@ import java.util.stream.Stream; * * @author mpolden */ -public class CertificateAuthorityApiHandler extends LoggingRequestHandler { +public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { private final SecretStore secretStore; private final Certificates certificates; diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 9ed4c47c023..b1b03b60ce6 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -26,7 +26,7 @@ <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-archiver</artifactId> - <version>3.1.1</version> + <version>3.5.0</version> </dependency> <dependency> <groupId>org.apache.maven.plugin-tools</groupId> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java index 054e820b1a4..7c34539921b 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Optional; - /** * Main entry point for class analysis * @@ -21,6 +20,7 @@ import java.util.Optional; * @author ollivir */ public class Analyze { + public static ClassFileMetaData analyzeClass(File classFile) { return analyzeClass(classFile, null); } @@ -44,7 +44,7 @@ public class Analyze { } static Optional<String> internalNameToClassName(String internalClassName) { - if(internalClassName == null) { + if (internalClassName == null) { return Optional.empty(); } else { return getClassName(Type.getObjectType(internalClassName)); @@ -53,12 +53,14 @@ public class Analyze { static Optional<String> getClassName(Type aType) { switch (aType.getSort()) { - case Type.ARRAY: - return getClassName(aType.getElementType()); - case Type.OBJECT: - return Optional.of(aType.getClassName()); - default: - return Optional.empty(); + case Type.ARRAY: + return getClassName(aType.getElementType()); + case Type.OBJECT: + return Optional.of(aType.getClassName()); + case Type.METHOD: + return getClassName(aType.getReturnType()); + default: + return Optional.empty(); } } @@ -89,4 +91,5 @@ public class Analyze { } }; } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index 88b752ae945..307509f0452 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -24,8 +24,9 @@ import java.util.Set; * @author ollivir */ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { + private String name = null; - private Set<String> imports = new HashSet<>(); + private final Set<String> imports = new HashSet<>(); private Optional<ExportPackageAnnotation> exportPackageAnnotation = Optional.empty(); private final Optional<ArtifactVersion> defaultExportPackageVersion; @@ -168,4 +169,5 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { assert (!imports.contains("int")); return new ClassFileMetaData(name, imports, exportPackageAnnotation); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java index b4c44c9ed40..051df41d62b 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java @@ -14,6 +14,7 @@ import java.util.Set; * @author ollivir */ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; private final Set<String> imports = new HashSet<>(); @@ -46,4 +47,5 @@ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector public void visitEnd() { analyzeClassVisitor.addImports(imports); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java index b7d1291d54a..7913f315cdd 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -20,7 +20,9 @@ import java.util.Set; * @author ollivir */ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { + private final Set<String> imports = new HashSet<>(); + private final AnalyzeClassVisitor analyzeClassVisitor; AnalyzeMethodVisitor(AnalyzeClassVisitor analyzeClassVisitor) { @@ -104,13 +106,14 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bootstrapMethod, Object... bootstrapMethodArgs) { + addImportWithTypeDesc(desc); for (Object arg : bootstrapMethodArgs) { if (arg instanceof Type) { addImport((Type) arg); } else if (arg instanceof Handle) { addImportWithInternalName(((Handle) arg).getOwner()); Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); - } else if ((arg instanceof Number) == false && (arg instanceof String) == false) { + } else if ( ! (arg instanceof Number) && ! (arg instanceof String)) { throw new AssertionError("Unexpected type " + arg.getClass() + " with value '" + arg + "'"); } } @@ -165,4 +168,5 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { @Override public void visitCode() { } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java index d3c32e11201..61c37e99edf 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java @@ -12,10 +12,10 @@ import java.util.Set; * @author Tony Vaagenes * @author ollivir */ - class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; - private Set<String> imports = new HashSet<>(); + private final Set<String> imports = new HashSet<>(); AnalyzeSignatureVisitor(AnalyzeClassVisitor analyzeClassVisitor) { super(Opcodes.ASM7); @@ -116,4 +116,5 @@ class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollecto if (signature != null) new SignatureReader(signature).acceptType(new AnalyzeSignatureVisitor(analyzeClassVisitor)); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java index 4195f342e92..5601430a27f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public class ClassFileMetaData { + private final String name; private final Set<String> referencedClasses; private final Optional<ExportPackageAnnotation> exportPackage; @@ -32,4 +33,5 @@ public class ClassFileMetaData { public Optional<ExportPackageAnnotation> getExportPackage() { return exportPackage; } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java index 955056b2306..7f3fb9522f7 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java @@ -9,6 +9,7 @@ import java.util.regex.Pattern; * @author ollivir */ public class ExportPackageAnnotation { + private final int major; private final int minor; private final int micro; @@ -59,4 +60,5 @@ public class ExportPackageAnnotation { public int hashCode() { return Objects.hash(major, minor, micro, qualifier); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java index 1dc1e49897d..e341ef1a80f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public interface ImportCollector { + Set<String> imports(); default void addImportWithTypeDesc(String typeDescriptor) { @@ -32,4 +33,5 @@ public interface ImportCollector { default void addImport(Optional<String> anImport) { anImport.ifPresent(pkg -> imports().add(pkg)); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java index 3d20bfd54df..9eef8a55c01 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public class Packages { + public static class PackageMetaData { public final Set<String> definedPackages; public final Set<String> referencedExternalPackages; @@ -40,4 +41,5 @@ public class Packages { referencedPackages.removeAll(definedPackages); return new PackageMetaData(definedPackages, referencedPackages); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java index f5978148d1a..5b4c9590935 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java @@ -48,6 +48,7 @@ abstract class AbstractAssembleBundleMojo extends AbstractMojo { MavenArchiver mavenArchiver = new MavenArchiver(); mavenArchiver.setArchiver(jarArchiver); mavenArchiver.setOutputFile(jarFile.toFile()); + mavenArchiver.configureReproducible("1"); try { mavenArchiver.createArchive(session, project, archiveConfiguration); } catch (Exception e) { diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java index 4502f88d158..1bb6cb8976e 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -7,6 +7,7 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.Derived; import com.yahoo.container.plugin.classanalysis.sampleclasses.Dummy; import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface3; import com.yahoo.container.plugin.classanalysis.sampleclasses.Methods; import org.junit.Test; @@ -24,6 +25,7 @@ import static org.junit.Assert.assertTrue; * @author Tony Vaagenes */ public class AnalyzeMethodBodyTest { + @Test public void require_that_class_of_locals_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Base.class))); @@ -65,6 +67,11 @@ public class AnalyzeMethodBodyTest { } @Test + public void require_that_functional_interface_usage_is_included() { + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface3.class))); + } + + @Test public void require_that_class_owning_method_handler_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(ClassWithMethod.class))); } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java index 9eacc0625b5..c2567e96ed9 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java @@ -8,6 +8,7 @@ import java.io.File; * @author ollivir */ public class TestUtilities { + public static ClassFileMetaData analyzeClass(Class<?> clazz) { return Analyze.analyzeClass(classFile(name(clazz))); } @@ -19,4 +20,5 @@ public class TestUtilities { public static String name(Class<?> clazz) { return clazz.getName(); } + } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java new file mode 100644 index 00000000000..d492cd6ed50 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +/** + * Input for class analysis tests. + * + * @author bratseth + */ +public interface Interface3 extends java.util.function.Supplier<String> { + +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java index fcc7057dac8..3fba4b3381b 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java @@ -3,6 +3,7 @@ package com.yahoo.container.plugin.classanalysis.sampleclasses; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Input for class analysis tests. @@ -10,6 +11,7 @@ import java.util.Map; */ @SuppressWarnings("unused") public class Methods { + public void method1() { Base b = new Base(); System.out.println(Fields.field2.size()); @@ -29,5 +31,15 @@ public class Methods { Derived d = new Derived(); } + public void method3() { + var result = methodTakingFunctionalArgument((Interface3)() -> "hello"); + System.out.println(result); + } + + public String methodTakingFunctionalArgument(Supplier<String> function) { + return function.get(); + } + public void methodTakingGenericArgument(Map<String, List<Dummy>> map) {} + } diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index 032d98c96fe..faab67781d1 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -15,13 +15,31 @@ import ( var overwriteKey bool +const apiKeyLongDoc = `Create a new user API key for authentication with Vespa Cloud. + +The API key will be stored in the Vespa CLI home directory +(see 'vespa help config'). Other commands will then automatically load the API +key as necessary. + +It's possible to override the API key used through environment variables. This +can be useful in continuous integration systems. + +* VESPA_CLI_API_KEY containing the key directly: + + export VESPA_CLI_API_KEY="my api key" + +* VESPA_CLI_API_KEY_FILE containing path to the key: + + export VESPA_CLI_API_KEY_FILE=/path/to/api-key + +Note that when overriding API key through environment variables, that key will +always be used. It's not possible to specify a tenant-specific key.` + func init() { apiKeyCmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") apiKeyCmd.MarkPersistentFlagRequired(applicationFlag) } -var example string - func apiKeyExample() string { if vespa.Auth0AccessTokenEnabled() { return "$ vespa auth api-key -a my-tenant.my-app.my-instance" @@ -33,40 +51,46 @@ func apiKeyExample() string { var apiKeyCmd = &cobra.Command{ Use: "api-key", Short: "Create a new user API key for authentication with Vespa Cloud", + Long: apiKeyLongDoc, Example: apiKeyExample(), DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), - Run: doApiKey, + RunE: doApiKey, } var deprecatedApiKeyCmd = &cobra.Command{ Use: "api-key", Short: "Create a new user API key for authentication with Vespa Cloud", + Long: apiKeyLongDoc, Example: apiKeyExample(), DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), Hidden: true, Deprecated: "use 'vespa auth api-key' instead", - Run: doApiKey, + RunE: doApiKey, } -func doApiKey(_ *cobra.Command, _ []string) { +func doApiKey(_ *cobra.Command, _ []string) error { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return fmt.Errorf("could not load config: %w", err) + } + app, err := getApplication() + if err != nil { + return err } - app := getApplication() apiKeyFile := cfg.APIKeyPath(app.Tenant) if util.PathExists(apiKeyFile) && !overwriteKey { - printErrHint(fmt.Errorf("File %s already exists", apiKeyFile), "Use -f to overwrite it") + err := fmt.Errorf("refusing to overwrite %s", apiKeyFile) + printErrHint(err, "Use -f to overwrite it") printPublicKey(apiKeyFile, app.Tenant) - return + return ErrCLI{error: err, quiet: true} } apiKey, err := vespa.CreateAPIKey() if err != nil { - fatalErr(err, "Could not create API key") - return + return fmt.Errorf("could not create api key: %w", err) } if err := ioutil.WriteFile(apiKeyFile, apiKey, 0600); err == nil { printSuccess("API private key written to ", apiKeyFile) @@ -74,41 +98,40 @@ func doApiKey(_ *cobra.Command, _ []string) { if vespa.Auth0AccessTokenEnabled() { if err == nil { if err := cfg.Set(cloudAuthFlag, "api-key"); err != nil { - fatalErr(err, "Could not write config") + return fmt.Errorf("could not write config: %w", err) } if err := cfg.Write(); err != nil { - fatalErr(err) + return err } } } + return nil } else { - fatalErr(err, "Failed to write ", apiKeyFile) + return fmt.Errorf("failed to write: %s: %w", apiKeyFile, err) } } -func printPublicKey(apiKeyFile, tenant string) { +func printPublicKey(apiKeyFile, tenant string) error { pemKeyData, err := ioutil.ReadFile(apiKeyFile) if err != nil { - fatalErr(err, "Failed to read ", apiKeyFile) - return + return fmt.Errorf("failed to read: %s: %w", apiKeyFile, err) } key, err := vespa.ECPrivateKeyFrom(pemKeyData) if err != nil { - fatalErr(err, "Failed to load key") - return + return fmt.Errorf("failed to load key: %w", err) } pemPublicKey, err := vespa.PEMPublicKeyFrom(key) if err != nil { - fatalErr(err, "Failed to extract public key") - return + return fmt.Errorf("failed to extract public key: %w", err) } fingerprint, err := vespa.FingerprintMD5(pemPublicKey) if err != nil { - fatalErr(err, "Failed to extract fingerprint") + return fmt.Errorf("failed to extract fingerprint: %w", err) } log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") log.Printf(color.Cyan("%s/tenant/%s/keys").String(), getConsoleURL(), tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") + return nil } diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index b08758ae21d..df3cf144180 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -18,6 +18,6 @@ func TestAPIKey(t *testing.T) { assert.Contains(t, out, "Success: API private key written to "+keyFile+"\n") out, outErr := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n") + assert.Contains(t, outErr, "Error: refusing to overwrite "+keyFile+"\nHint: Use -f to overwrite it\n") assert.Contains(t, out, "This is your public key") } diff --git a/client/go/cmd/auth.go b/client/go/cmd/auth.go index 9322f8d0808..68d7fa00fdf 100644 --- a/client/go/cmd/auth.go +++ b/client/go/cmd/auth.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -21,14 +23,13 @@ func init() { } var authCmd = &cobra.Command{ - Use: "auth", - Short: "Manage Vespa Cloud credentials", - Long: `Manage Vespa Cloud credentials.`, - + Use: "auth", + Short: "Manage Vespa Cloud credentials", + Long: `Manage Vespa Cloud credentials.`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - // Root command does nothing - cmd.Help() - exitFunc(1) + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) }, } diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index e79a45d3af8..f338a1ba81a 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -5,15 +5,41 @@ package cmd import ( "fmt" + "os" + "path/filepath" + "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" - "os" - "path/filepath" ) var overwriteCertificate bool +const longDoc = `Create a new private key and self-signed certificate for Vespa Cloud deployment. + +The private key and certificate will be stored in the Vespa CLI home directory +(see 'vespa help config'). Other commands will then automatically load the +certificate as necessary. + +It's possible to override the private key and certificate used through +environment variables. This can be useful in continuous integration systems. + +* VESPA_CLI_DATA_PLANE_CERT and VESPA_CLI_DATA_PLANE_KEY containing the + certificate and private key directly: + + export VESPA_CLI_DATA_PLANE_CERT="my cert" + export VESPA_CLI_DATA_PLANE_KEY="my private key" + +* VESPA_CLI_DATA_PLANE_CERT_FILE and VESPA_CLI_DATA_PLANE_KEY_FILE containing + paths to the certificate and private key: + + export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert + export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key + +Note that when overriding key pair through environment variables, that key pair +will always be used for all applications. It's not possible to specify an +application-specific key.` + func init() { certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") certCmd.MarkPersistentFlagRequired(applicationFlag) @@ -30,96 +56,90 @@ func certExample() string { var certCmd = &cobra.Command{ Use: "cert", Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", + Long: longDoc, Example: certExample(), DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MaximumNArgs(1), - Run: doCert, + RunE: doCert, } var deprecatedCertCmd = &cobra.Command{ Use: "cert", Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", + Long: longDoc, Example: "$ vespa cert -a my-tenant.my-app.my-instance", DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MaximumNArgs(1), Deprecated: "use 'vespa auth cert' instead", Hidden: true, - Run: doCert, + RunE: doCert, } -func doCert(_ *cobra.Command, args []string) { - app := getApplication() +func doCert(_ *cobra.Command, args []string) error { + app, err := getApplication() + if err != nil { + return err + } pkg, err := vespa.FindApplicationPackage(applicationSource(args), false) if err != nil { - fatalErr(err) - return + return err } cfg, err := LoadConfig() if err != nil { - fatalErr(err) - return + return err } privateKeyFile, err := cfg.PrivateKeyPath(app) if err != nil { - fatalErr(err) - return + return err } certificateFile, err := cfg.CertificatePath(app) if err != nil { - fatalErr(err) - return + return err } if !overwriteCertificate { hint := "Use -f flag to force overwriting" if pkg.HasCertificate() { - fatalErrHint(fmt.Errorf("Application package %s already contains a certificate", pkg.Path), hint) - return + return errHint(fmt.Errorf("application package %s already contains a certificate", pkg.Path), hint) } if util.PathExists(privateKeyFile) { - fatalErrHint(fmt.Errorf("Private key %s already exists", color.Cyan(privateKeyFile)), hint) - return + return errHint(fmt.Errorf("private key %s already exists", color.Cyan(privateKeyFile)), hint) } if util.PathExists(certificateFile) { - fatalErrHint(fmt.Errorf("Certificate %s already exists", color.Cyan(certificateFile)), hint) - return + return errHint(fmt.Errorf("certificate %s already exists", color.Cyan(certificateFile)), hint) } } if pkg.IsZip() { - var msg string + var hint string if vespa.Auth0AccessTokenEnabled() { - msg = "Try running 'mvn clean' before 'vespa auth cert', and then 'mvn package'" + hint = "Try running 'mvn clean' before 'vespa auth cert', and then 'mvn package'" } else { - msg = "Try running 'mvn clean' before 'vespa cert', and then 'mvn package'" + hint = "Try running 'mvn clean' before 'vespa cert', and then 'mvn package'" } - fatalErrHint(fmt.Errorf("Cannot add certificate to compressed application package %s", pkg.Path), - msg) - return + return errHint(fmt.Errorf("cannot add certificate to compressed application package %s", pkg.Path), hint) } keyPair, err := vespa.CreateKeyPair() if err != nil { - fatalErr(err, "Could not create key pair") - return + return err } pkgCertificateFile := filepath.Join(pkg.Path, "security", "clients.pem") if err := os.MkdirAll(filepath.Dir(pkgCertificateFile), 0755); err != nil { - fatalErr(err, "Could not create security directory") - return + return fmt.Errorf("could not create security directory: %w", err) } if err := keyPair.WriteCertificateFile(pkgCertificateFile, overwriteCertificate); err != nil { - fatalErr(err, "Could not write certificate") - return + return fmt.Errorf("could not write certificate to application package: %w", err) } if err := keyPair.WriteCertificateFile(certificateFile, overwriteCertificate); err != nil { - fatalErr(err, "Could not write certificate") - return + return fmt.Errorf("could not write certificate: %w", err) } if err := keyPair.WritePrivateKeyFile(privateKeyFile, overwriteCertificate); err != nil { - fatalErr(err, "Could not write private key") - return + return fmt.Errorf("could not write private key: %w", err) } printSuccess("Certificate written to ", color.Cyan(pkgCertificateFile)) printSuccess("Certificate written to ", color.Cyan(certificateFile)) printSuccess("Private key written to ", color.Cyan(privateKeyFile)) + return nil } diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index 96b626b5c98..aae40db43a6 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -29,7 +29,7 @@ func TestCert(t *testing.T) { assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), out) _, outErr := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, fmt.Sprintf("Error: Application package %s already contains a certificate", appDir)) + assert.Contains(t, outErr, fmt.Sprintf("Error: application package %s already contains a certificate", appDir)) } func TestCertCompressedPackage(t *testing.T) { @@ -42,7 +42,7 @@ func TestCertCompressedPackage(t *testing.T) { assert.Nil(t, err) _, outErr := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Contains(t, outErr, "Error: Cannot add certificate to compressed application package") + assert.Contains(t, outErr, "Error: cannot add certificate to compressed application package") err = os.Remove(zipFile) assert.Nil(t, err) diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 6fe3c0d5a29..7a25613947a 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -45,62 +45,61 @@ directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment variable.`, Example: "$ vespa clone vespa-cloud/album-recommendation my-app", DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { if listApps { apps, err := listSampleApps() if err != nil { - fatalErr(err, "Could not list sample applications") - return + return fmt.Errorf("could not list sample applications: %w", err) } for _, app := range apps { log.Print(app) } - } else { - if len(args) != 2 { - fatalErr(nil, "Expected exactly 2 arguments") - return - } - cloneApplication(args[0], args[1]) + return nil + } + if len(args) != 2 { + return fmt.Errorf("expected exactly 2 arguments, got %d", len(args)) } + return cloneApplication(args[0], args[1]) }, } -func cloneApplication(source string, name string) { - zipFile := getSampleAppsZip() +func cloneApplication(applicationName string, applicationDir string) error { + zipFile, err := getSampleAppsZip() + if err != nil { + return err + } defer zipFile.Close() - zipReader, zipOpenError := zip.OpenReader(zipFile.Name()) - if zipOpenError != nil { - fatalErr(zipOpenError, "Could not open sample apps zip '", color.Cyan(zipFile.Name()), "'") - return + r, err := zip.OpenReader(zipFile.Name()) + if err != nil { + return fmt.Errorf("could not open sample apps zip '%s': %w", color.Cyan(zipFile.Name()), err) } - defer zipReader.Close() + defer r.Close() found := false - for _, f := range zipReader.File { - zipEntryPrefix := "sample-apps-master/" + source + "/" - if strings.HasPrefix(f.Name, zipEntryPrefix) { + for _, f := range r.File { + dirPrefix := "sample-apps-master/" + applicationName + "/" + if strings.HasPrefix(f.Name, dirPrefix) { if !found { // Create destination directory lazily when source is found - createErr := os.Mkdir(name, 0755) + createErr := os.Mkdir(applicationDir, 0755) if createErr != nil { - fatalErr(createErr, "Could not create directory '", color.Cyan(name), "'") - return + return fmt.Errorf("could not create directory '%s': %w", color.Cyan(applicationDir), createErr) } } found = true - copyError := copy(f, name, zipEntryPrefix) - if copyError != nil { - fatalErr(copyError, "Could not copy zip entry '", color.Cyan(f.Name), "' to ", color.Cyan(name)) - return + if err := copy(f, applicationDir, dirPrefix); err != nil { + return fmt.Errorf("could not copy zip entry '%s': %w", color.Cyan(f.Name), err) } } } if !found { - fatalErrHint(fmt.Errorf("Could not find source application '%s'", color.Cyan(source)), "Use -f to ignore the cache") + return errHint(fmt.Errorf("could not find source application '%s'", color.Cyan(applicationName)), "Use -f to ignore the cache") } else { - log.Print("Created ", color.Cyan(name)) + log.Print("Created ", color.Cyan(applicationDir)) } + return nil } func openOutputFile() (*os.File, error) { @@ -126,72 +125,66 @@ func useCache(cacheFile *os.File) (bool, error) { return stat.Size() > 0 && time.Now().Before(expiry), nil } -func getSampleAppsZip() *os.File { +func getSampleAppsZip() (*os.File, error) { f, err := openOutputFile() if err != nil { - fatalErr(err, "Could not determine location of cache file") - return nil + return nil, fmt.Errorf("could not determine location of cache file: %w", err) } useCache, err := useCache(f) if err != nil { - fatalErr(err, "Could not determine cache status", "Try ignoring the cache with the -f flag") - return nil + return nil, errHint(fmt.Errorf("could not determine cache status: %w", err), "Try ignoring the cache with the -f flag") } if useCache { log.Print(color.Yellow("Using cached sample apps ...")) - return f + return f, nil } - err = util.Spinner(color.Yellow("Downloading sample apps ...").String(), func() error { request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) if err != nil { - fatalErr(err, "Invalid URL") - return nil + return fmt.Errorf("invalid url: %w", err) } response, err := util.HttpDo(request, time.Minute*60, "GitHub") if err != nil { - fatalErr(err, "Could not download sample apps from GitHub") - return nil + return fmt.Errorf("could not download sample apps: %w", err) } defer response.Body.Close() if response.StatusCode != 200 { - fatalErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode) - return nil + return fmt.Errorf("could not download sample apps: github returned status %d", response.StatusCode) + } + if err := f.Truncate(0); err != nil { + return fmt.Errorf("could not truncate sample apps file: %s: %w", f.Name(), err) } if _, err := io.Copy(f, response.Body); err != nil { - fatalErr(err, "Could not write sample apps to file: ", f.Name()) - return nil + return fmt.Errorf("could not write sample apps to file: %s: %w", f.Name(), err) } - return err + return nil }) - - return f + return f, err } func copy(f *zip.File, destinationDir string, zipEntryPrefix string) error { destinationPath := filepath.Join(destinationDir, filepath.FromSlash(strings.TrimPrefix(f.Name, zipEntryPrefix))) if strings.HasSuffix(f.Name, "/") { if f.Name != zipEntryPrefix { // root is already created - createError := os.Mkdir(destinationPath, 0755) - if createError != nil { - return createError + if err := os.Mkdir(destinationPath, 0755); err != nil { + return err } } } else { - zipEntry, zipEntryOpenError := f.Open() - if zipEntryOpenError != nil { - return zipEntryOpenError + r, err := f.Open() + if err != nil { + return err } - defer zipEntry.Close() - - destination, createError := os.Create(destinationPath) - if createError != nil { - return createError + defer r.Close() + destination, err := os.Create(destinationPath) + if err != nil { + return err } - - _, copyError := io.Copy(destination, zipEntry) - if copyError != nil { - return copyError + if _, err := io.Copy(destination, r); err != nil { + return err + } + if err := os.Chmod(destinationPath, f.Mode()); err != nil { + return err } } return nil diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 054dc7b21fb..af8b686b111 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -15,7 +15,7 @@ import ( ) func TestClone(t *testing.T) { - assertCreated("album-recommendation-selfhosted", "mytestapp", t) + assertCreated("text-search", "mytestapp", t) } func assertCreated(sampleAppName string, app string, t *testing.T) { @@ -32,6 +32,9 @@ func assertCreated(sampleAppName string, app string, t *testing.T) { assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application"))) servicesStat, _ := os.Stat(filepath.Join(app, "src", "main", "application", "services.xml")) - servicesSize := int64(2474) + servicesSize := int64(1772) assert.Equal(t, servicesSize, servicesStat.Size()) + + scriptStat, _ := os.Stat(filepath.Join(app, "bin", "convert-msmarco.sh")) + assert.Equal(t, os.FileMode(0755), scriptStat.Mode()) } diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index eb55021b536..135dda895d4 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -63,9 +63,6 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) rootCmd.Flags().VisitAll(resetFlag) documentCmd.Flags().VisitAll(resetFlag) - // Do not exit in tests - exitFunc = func(code int) {} - // Capture stdout and execute command var capturedOut bytes.Buffer var capturedErr bytes.Buffer @@ -79,7 +76,7 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) // Execute command and return output rootCmd.SetArgs(append(cmd.args, cmd.moreArgs...)) - rootCmd.Execute() + Execute() return capturedOut.String(), capturedErr.String() } diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 3a6e43e7ffe..a195a3e1975 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -5,6 +5,7 @@ package cmd import ( + "crypto/tls" "fmt" "io/ioutil" "log" @@ -45,10 +46,10 @@ instead. Configuration is written to $HOME/.vespa by default. This path can be overridden by setting the VESPA_CLI_HOME environment variable.`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - // Root command does nothing - cmd.Help() - exitFunc(1) + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) }, } @@ -57,36 +58,33 @@ var setConfigCmd = &cobra.Command{ Short: "Set a configuration option.", Example: "$ vespa config set target cloud", DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(2), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err } if err := cfg.Set(args[0], args[1]); err != nil { - fatalErr(err) - } else { - if err := cfg.Write(); err != nil { - fatalErr(err) - } + return err } + return cfg.Write() }, } var getConfigCmd = &cobra.Command{ - Use: "get option-name", - Short: "Get a configuration option", - Example: "$ vespa config get target", + Use: "get [option-name]", + Short: "Show given configuration option, or all configuration options", + Example: `$ vespa config get +$ vespa config get target`, Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err } - if len(args) == 0 { // Print all values var flags []string for flag := range flagToConfigBindings { @@ -99,6 +97,7 @@ var getConfigCmd = &cobra.Command{ } else { printOption(cfg, args[0]) } + return nil }, } @@ -107,14 +106,20 @@ type Config struct { createDirs bool } +type KeyPair struct { + KeyPair tls.Certificate + CertificateFile string + PrivateKeyFile string +} + func LoadConfig() (*Config, error) { home, err := vespaCliHome() if err != nil { - return nil, err + return nil, fmt.Errorf("could not detect config directory: %w", err) } c := &Config{Home: home, createDirs: true} if err := c.load(); err != nil { - return nil, err + return nil, fmt.Errorf("could not load config: %w", err) } return c, nil } @@ -146,11 +151,44 @@ func (c *Config) PrivateKeyPath(app vespa.ApplicationID) (string, error) { return c.applicationFilePath(app, "data-plane-private-key.pem") } +func (c *Config) X509KeyPair(app vespa.ApplicationID) (KeyPair, error) { + cert, certOk := os.LookupEnv("VESPA_CLI_DATA_PLANE_CERT") + key, keyOk := os.LookupEnv("VESPA_CLI_DATA_PLANE_KEY") + if certOk && keyOk { + // Use key pair from environment + kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) + return KeyPair{KeyPair: kp}, err + } + privateKeyFile, err := c.PrivateKeyPath(app) + if err != nil { + return KeyPair{}, err + } + certificateFile, err := c.CertificatePath(app) + if err != nil { + return KeyPair{}, err + } + kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile) + if err != nil { + return KeyPair{}, err + } + return KeyPair{ + KeyPair: kp, + CertificateFile: certificateFile, + PrivateKeyFile: privateKeyFile, + }, nil +} + func (c *Config) APIKeyPath(tenantName string) string { + if override, ok := os.LookupEnv("VESPA_CLI_API_KEY_FILE"); ok { + return override + } return filepath.Join(c.Home, tenantName+".api-key.pem") } func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { + if override, ok := os.LookupEnv("VESPA_CLI_API_KEY"); ok { + return []byte(override), nil + } return ioutil.ReadFile(c.APIKeyPath(tenantName)) } diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 0e74e53c5e5..ba0df781715 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -10,7 +10,7 @@ import ( func TestConfig(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") - assertConfigCommandErr(t, "invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") + assertConfigCommandErr(t, "Error: invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") assertConfigCommand(t, "", homeDir, "config", "set", "target", "cloud") @@ -19,7 +19,7 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, "", homeDir, "config", "set", "target", "https://127.0.0.1") assertConfigCommand(t, "target = https://127.0.0.1\n", homeDir, "config", "get", "target") - assertConfigCommandErr(t, "invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") + assertConfigCommandErr(t, "Error: invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") assertConfigCommand(t, "application = <unset>\n", homeDir, "config", "get", "application") assertConfigCommand(t, "", homeDir, "config", "set", "application", "t1.a1.i1") assertConfigCommand(t, "application = t1.a1.i1\n", homeDir, "config", "get", "application") @@ -27,7 +27,7 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, "application = t1.a1.i1\ncolor = auto\nquiet = false\ntarget = https://127.0.0.1\nwait = 0\n", homeDir, "config", "get") assertConfigCommand(t, "", homeDir, "config", "set", "wait", "60") - assertConfigCommandErr(t, "wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") + assertConfigCommandErr(t, "Error: wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") assertConfigCommand(t, "wait = 60\n", homeDir, "config", "get", "wait") } diff --git a/client/go/cmd/curl.go b/client/go/cmd/curl.go index 2496ddc3abc..47d0dcde95b 100644 --- a/client/go/cmd/curl.go +++ b/client/go/cmd/curl.go @@ -2,72 +2,115 @@ package cmd import ( + "errors" + "fmt" "log" "os" "strings" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/auth0" "github.com/vespa-engine/vespa/client/go/curl" + "github.com/vespa-engine/vespa/client/go/vespa" ) var curlDryRun bool +var curlService string func init() { rootCmd.AddCommand(curlCmd) curlCmd.Flags().BoolVarP(&curlDryRun, "dry-run", "n", false, "Print the curl command that would be executed") + curlCmd.Flags().StringVarP(&curlService, "service", "s", "query", "Which service to query. Must be \"deploy\", \"document\" or \"query\"") } var curlCmd = &cobra.Command{ Use: "curl [curl-options] path", - Short: "Query Vespa using curl", - Long: `Query Vespa using curl. + Short: "Access Vespa directly using curl", + Long: `Access Vespa directly using curl. -Execute curl with the appropriate URL, certificate and private key for your application.`, - Example: `$ vespa curl /search/?yql=query -$ vespa curl -- -v --data-urlencode "yql=select * from sources * where title contains 'foo';" /search/ -$ vespa curl -t local -- -v /search/?yql=query +Execute curl with the appropriate URL, certificate and private key for your application. + +For a more high-level interface to query and feeding, see the 'query' and 'document' commands. `, + Example: `$ vespa curl /ApplicationStatus +$ vespa curl -- -X POST -H "Content-Type:application/json" --data-binary @src/test/resources/A-Head-Full-of-Dreams.json /document/v1/namespace/music/docid/1 +$ vespa curl -- -v --data-urlencode "yql=select * from music where album contains 'head';" /search/\?hits=5`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err } - app := getApplication() - privateKeyFile, err := cfg.PrivateKeyPath(app) + app, err := getApplication() if err != nil { - fatalErr(err) - return + return err } - certificateFile, err := cfg.CertificatePath(app) + service, err := getService(curlService, 0, "") if err != nil { - fatalErr(err) - return + return err } - service := getService("query", 0, "") url := joinURL(service.BaseURL, args[len(args)-1]) rawArgs := args[:len(args)-1] c, err := curl.RawArgs(url, rawArgs...) if err != nil { - fatalErr(err) - return + return err + } + switch curlService { + case "deploy": + t, err := getTarget() + if err != nil { + return err + } + if t.Type() == "cloud" { + if !vespa.Auth0AccessTokenEnabled() { + return errors.New("accessing control plane using curl subcommand is only supported for Auth0 device flow") + } + if err := addCloudAuth0Authentication(cfg, c); err != nil { + return err + } + } + case "document", "query": + privateKeyFile, err := cfg.PrivateKeyPath(app) + if err != nil { + return err + } + certificateFile, err := cfg.CertificatePath(app) + if err != nil { + return err + } + c.PrivateKey = privateKeyFile + c.Certificate = certificateFile + default: + return fmt.Errorf("service not found: %s", curlService) } - c.PrivateKey = privateKeyFile - c.Certificate = certificateFile if curlDryRun { log.Print(c.String()) } else { if err := c.Run(os.Stdout, os.Stderr); err != nil { - fatalErr(err, "Failed to run curl") - return + return fmt.Errorf("failed to execute curl: %w", err) } } + return nil }, } +func addCloudAuth0Authentication(cfg *Config, c *curl.Command) error { + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) + if err != nil { + return err + } + + system, err := a.PrepareSystem(auth0.ContextWithCancel()) + if err != nil { + return err + } + c.Header("Authorization", "Bearer "+system.AccessToken) + return nil +} + func joinURL(baseURL, path string) string { baseURL = strings.TrimSuffix(baseURL, "/") path = strings.TrimPrefix(path, "/") diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index d5021e19cf2..e593f48a390 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -18,4 +18,8 @@ func TestCurl(t *testing.T) { filepath.Join(homeDir, "t1.a1.i1", "data-plane-private-key.pem"), filepath.Join(homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) assert.Equal(t, expected, out) + + out, _ = execute(command{homeDir: homeDir, args: []string{"curl", "-s", "deploy", "-n", "/application/v4/tenant/foo"}}, t, httpClient) + expected = "curl https://127.0.0.1:19071/application/v4/tenant/foo\n" + assert.Equal(t, expected, out) } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index ae39afc3773..e35188933e1 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -18,9 +18,8 @@ const ( ) var ( - zoneArg string - logLevelArg string - sessionOrRunID int64 + zoneArg string + logLevelArg string ) func init() { @@ -51,39 +50,45 @@ $ vespa deploy -t cloud -z dev.aws-us-east-1c # -z can be omitted here as this $ vespa deploy -t cloud -z perf.aws-us-east-1c`, Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) if err != nil { - fatalErr(nil, err.Error()) - return + return err } cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err + } + target, err := getTarget() + if err != nil { + return err + } + opts, err := getDeploymentOpts(cfg, pkg, target) + if err != nil { + return err + } + sessionOrRunID, err := vespa.Deploy(opts) + if err != nil { + return err } - target := getTarget() - opts := getDeploymentOpts(cfg, pkg, target) - if sessionOrRunID, err := vespa.Deploy(opts); err == nil { - fmt.Print("\n") - if opts.IsCloud() { - printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) - } else { - printSuccess("Deployed ", color.Cyan(pkg.Path)) - } - if opts.IsCloud() { - log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) - log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", - getConsoleURL(), - opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, - opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, - sessionOrRunID))) - } - waitForQueryService(sessionOrRunID) + fmt.Print("\n") + if opts.IsCloud() { + printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) } else { - fatalErr(nil, err.Error()) + printSuccess("Deployed ", color.Cyan(pkg.Path)) } + if opts.IsCloud() { + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) + log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", + getConsoleURL(), + opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, + opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, + sessionOrRunID))) + } + waitForQueryService(sessionOrRunID) + return nil }, } @@ -92,31 +97,32 @@ var prepareCmd = &cobra.Command{ Short: "Prepare an application package for activation", Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) if err != nil { - fatalErr(err, "Could not find application package") - return + return fmt.Errorf("could not find application package: %w", err) } cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err + } + target, err := getTarget() + if err != nil { + return err } - target := getTarget() sessionID, err := vespa.Prepare(vespa.DeploymentOpts{ ApplicationPackage: pkg, Target: target, }) - if err == nil { - if err := cfg.WriteSessionID(vespa.DefaultApplication, sessionID); err != nil { - fatalErr(err, "Could not write session ID") - return - } - printSuccess("Prepared ", color.Cyan(pkg.Path), " with session ", sessionID) - } else { - fatalErr(nil, err.Error()) + if err != nil { + return err + } + if err := cfg.WriteSessionID(vespa.DefaultApplication, sessionID); err != nil { + return fmt.Errorf("could not write session id: %w", err) } + printSuccess("Prepared ", color.Cyan(pkg.Path), " with session ", sessionID) + return nil }, } @@ -125,33 +131,34 @@ var activateCmd = &cobra.Command{ Short: "Activate (deploy) a previously prepared application package", Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { pkg, err := vespa.FindApplicationPackage(applicationSource(args), true) if err != nil { - fatalErr(err, "Could not find application package") - return + return fmt.Errorf("could not find application package: %w", err) } cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err } sessionID, err := cfg.ReadSessionID(vespa.DefaultApplication) if err != nil { - fatalErr(err, "Could not read session ID") - return + return fmt.Errorf("could not read session id: %w", err) + } + target, err := getTarget() + if err != nil { + return err } - target := getTarget() err = vespa.Activate(sessionID, vespa.DeploymentOpts{ ApplicationPackage: pkg, Target: target, }) - if err == nil { - printSuccess("Activated ", color.Cyan(pkg.Path), " with session ", sessionID) - waitForQueryService(sessionID) - } else { - fatalErr(nil, err.Error()) + if err != nil { + return err } + printSuccess("Activated ", color.Cyan(pkg.Path), " with session ", sessionID) + waitForQueryService(sessionID) + return nil }, } diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index 5bb45e70fad..a37a433397f 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -164,7 +164,7 @@ func assertApplicationPackageError(t *testing.T, cmd string, status int, expecte client.NextResponse(status, returnBody) _, outErr := execute(command{args: []string{cmd, "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, - "Error: Invalid application package (Status "+strconv.Itoa(status)+")\n\n"+expectedMessage+"\n", + "Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", outErr) } @@ -173,6 +173,6 @@ func assertDeployServerError(t *testing.T, status int, errorMessage string) { client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, - "Error: Error from deploy service at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", + "Error: error from deploy service at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", outErr) } diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 84c384e701e..8dab813ec68 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -46,9 +46,14 @@ To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html should be used instead of this.`, Example: `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - printResult(vespa.Send(args[0], documentService(), operationOptions()), false) + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService() + if err != nil { + return err + } + return printResult(vespa.Send(args[0], service, operationOptions()), false) }, } @@ -62,11 +67,16 @@ If the document id is specified both as an argument and in the file the argument Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService() + if err != nil { + return err + } if len(args) == 1 { - printResult(vespa.Put("", args[0], documentService(), operationOptions()), false) + return printResult(vespa.Put("", args[0], service, operationOptions()), false) } else { - printResult(vespa.Put(args[0], args[1], documentService(), operationOptions()), false) + return printResult(vespa.Put(args[0], args[1], service, operationOptions()), false) } }, } @@ -80,11 +90,16 @@ If the document id is specified both as an argument and in the file the argument Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService() + if err != nil { + return err + } if len(args) == 1 { - printResult(vespa.Update("", args[0], documentService(), operationOptions()), false) + return printResult(vespa.Update("", args[0], service, operationOptions()), false) } else { - printResult(vespa.Update(args[0], args[1], documentService(), operationOptions()), false) + return printResult(vespa.Update(args[0], args[1], service, operationOptions()), false) } }, } @@ -98,11 +113,16 @@ If the document id is specified both as an argument and in the file the argument Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService() + if err != nil { + return err + } if strings.HasPrefix(args[0], "id:") { - printResult(vespa.RemoveId(args[0], documentService(), operationOptions()), false) + return printResult(vespa.RemoveId(args[0], service, operationOptions()), false) } else { - printResult(vespa.RemoveOperation(args[0], documentService(), operationOptions()), false) + return printResult(vespa.RemoveOperation(args[0], service, operationOptions()), false) } }, } @@ -112,13 +132,18 @@ var documentGetCmd = &cobra.Command{ Short: "Gets a document", Args: cobra.ExactArgs(1), DisableAutoGenTag: true, + SilenceUsage: true, Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, - Run: func(cmd *cobra.Command, args []string) { - printResult(vespa.Get(args[0], documentService(), operationOptions()), true) + RunE: func(cmd *cobra.Command, args []string) error { + service, err := documentService() + if err != nil { + return err + } + return printResult(vespa.Get(args[0], service, operationOptions()), true) }, } -func documentService() *vespa.Service { return getService("document", 0, "") } +func documentService() (*vespa.Service, error) { return getService("document", 0, "") } func operationOptions() vespa.OperationOptions { return vespa.OperationOptions{ @@ -134,7 +159,7 @@ func curlOutput() io.Writer { return ioutil.Discard } -func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { +func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) error { out := stdout if !result.Success { out = stderr @@ -158,6 +183,9 @@ func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { } if !result.Success { - exitFunc(1) + err := errHint(fmt.Errorf("document operation failed")) + err.quiet = true + return err } + return nil } diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index f3a5fbe9543..2b596e9893b 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -97,7 +97,10 @@ func TestDocumentGet(t *testing.T) { func assertDocumentSend(arguments []string, expectedOperation string, expectedMethod string, expectedDocumentId string, expectedPayloadFile string, t *testing.T) { client := &mockHttpClient{} - documentURL := documentServiceURL(client) + documentURL, err := documentServiceURL(client) + if err != nil { + t.Fatal(err) + } expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) expectedURL := documentURL + "/document/v1/" + expectedPath out, errOut := execute(command{args: arguments}, t, client) @@ -123,7 +126,10 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe func assertDocumentGet(arguments []string, documentId string, t *testing.T) { client := &mockHttpClient{} - documentURL := documentServiceURL(client) + documentURL, err := documentServiceURL(client) + if err != nil { + t.Fatal(err) + } client.NextResponse(200, "{\"fields\":{\"foo\":\"bar\"}}") assert.Equal(t, `{ @@ -160,6 +166,10 @@ func assertDocumentServerError(t *testing.T, status int, errorMessage string) { outErr) } -func documentServiceURL(client *mockHttpClient) string { - return getService("document", 0, "").BaseURL +func documentServiceURL(client *mockHttpClient) (string, error) { + service, err := getService("document", 0, "") + if err != nil { + return "", err + } + return service.BaseURL, nil } diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index f065ae0c680..03cdaecbfce 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -5,10 +5,8 @@ package cmd import ( - "crypto/tls" "encoding/json" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -18,34 +16,15 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -var exitFunc = os.Exit // To allow overriding Exit in tests - -func fatalErrHint(err error, hints ...string) { - printErrHint(err, hints...) - exitFunc(1) -} - -func fatalErr(err error, msg ...interface{}) { - printErr(err, msg...) - exitFunc(1) -} - func printErrHint(err error, hints ...string) { - if err != nil { - printErr(nil, err.Error()) - } + printErr(err) for _, hint := range hints { fmt.Fprintln(stderr, color.Cyan("Hint:"), hint) } } -func printErr(err error, msg ...interface{}) { - if len(msg) > 0 { - fmt.Fprintln(stderr, color.Red("Error:"), fmt.Sprint(msg...)) - } - if err != nil { - fmt.Fprintln(stderr, color.Yellow(err)) - } +func printErr(err error) { + fmt.Fprintln(stderr, color.Red("Error:"), err) } func printSuccess(msg ...interface{}) { @@ -82,13 +61,16 @@ func vespaCliCacheDir() (string, error) { return cacheDir, nil } -func deploymentFromArgs() vespa.Deployment { +func deploymentFromArgs() (vespa.Deployment, error) { zone, err := vespa.ZoneFromString(zoneArg) if err != nil { - fatalErrHint(err, "Zone format is <env>.<region>") + return vespa.Deployment{}, err + } + app, err := getApplication() + if err != nil { + return vespa.Deployment{}, err } - app := getApplication() - return vespa.Deployment{Application: app, Zone: zone} + return vespa.Deployment{Application: app, Zone: zone}, nil } func applicationSource(args []string) string { @@ -98,49 +80,48 @@ func applicationSource(args []string) string { return "." } -func getApplication() vespa.ApplicationID { +func getApplication() (vespa.ApplicationID, error) { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return vespa.ApplicationID{} + return vespa.ApplicationID{}, err } app, err := cfg.Get(applicationFlag) if err != nil { - fatalErrHint(err, "No application specified. Try the --"+applicationFlag+" flag") - return vespa.ApplicationID{} + return vespa.ApplicationID{}, errHint(fmt.Errorf("no application specified: %w", err), "Try the --"+applicationFlag+" flag") } application, err := vespa.ApplicationFromString(app) if err != nil { - fatalErrHint(err, "Application format is <tenant>.<app>.<instance>") - return vespa.ApplicationID{} + return vespa.ApplicationID{}, errHint(err, "application format is <tenant>.<app>.<instance>") } - return application + return application, nil } -func getTargetType() string { +func getTargetType() (string, error) { cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return "" + return "", err } target, err := cfg.Get(targetFlag) if err != nil { - fatalErr(err, "A valid target must be specified") + return "", fmt.Errorf("invalid target: %w", err) } - return target + return target, nil } -func getService(service string, sessionOrRunID int64, cluster string) *vespa.Service { - t := getTarget() +func getService(service string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { + t, err := getTarget() + if err != nil { + return nil, err + } timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { - log.Printf("Waiting up to %d %s for service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) + log.Printf("Waiting up to %d %s for %s service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds"), color.Cyan(service)) } s, err := t.Service(service, timeout, sessionOrRunID, cluster) if err != nil { - fatalErr(err, "Invalid service: ", service) + return nil, fmt.Errorf("service %s not found: %w", service, err) } - return s + return s, nil } func getEndpointsOverride() string { return os.Getenv("VESPA_CLI_ENDPOINTS") } @@ -169,49 +150,47 @@ func getApiURL() string { return "https://api.vespa-external.aws.oath.cloud:4443" } -func getTarget() vespa.Target { - targetType := getTargetType() +func getTarget() (vespa.Target, error) { + targetType, err := getTargetType() + if err != nil { + return nil, err + } if strings.HasPrefix(targetType, "http") { - return vespa.CustomTarget(targetType) + return vespa.CustomTarget(targetType), nil } switch targetType { case "local": - return vespa.LocalTarget() + return vespa.LocalTarget(), nil case "cloud": cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return nil + return nil, err + } + deployment, err := deploymentFromArgs() + if err != nil { + return nil, err + } + endpoints, err := getEndpointsFromEnv() + if err != nil { + return nil, err } - deployment := deploymentFromArgs() - endpoints := getEndpointsFromEnv() var apiKey []byte = nil - apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant)) + apiKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) if !vespa.Auth0AccessTokenEnabled() && endpoints == nil { if err != nil { - fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") + return nil, errHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") } } - privateKeyFile, err := cfg.PrivateKeyPath(deployment.Application) - if err != nil { - fatalErr(err) - return nil - } - certificateFile, err := cfg.CertificatePath(deployment.Application) + kp, err := cfg.X509KeyPair(deployment.Application) if err != nil { - fatalErr(err) - return nil - } - kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile) - if err != nil { - var msg string + var hint string if vespa.Auth0AccessTokenEnabled() { - msg = "Deployment to cloud requires a certificate. Try 'vespa auth cert'" + hint = "Deployment to cloud requires a certificate. Try 'vespa auth cert'" } else { - msg = "Deployment to cloud requires a certificate. Try 'vespa cert'" + hint = "Deployment to cloud requires a certificate. Try 'vespa cert'" } - fatalErrHint(err, msg) + return nil, errHint(err, hint) } var cloudAuth string if vespa.Auth0AccessTokenEnabled() { @@ -227,11 +206,14 @@ func getTarget() vespa.Target { cloudAuth = "" } - return vespa.CloudTarget(getApiURL(), deployment, apiKey, + return vespa.CloudTarget( + getApiURL(), + deployment, + apiKey, vespa.TLSOptions{ - KeyPair: kp, - CertificateFile: certificateFile, - PrivateKeyFile: privateKeyFile, + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, }, vespa.LogOptions{ Writer: stdout, @@ -240,14 +222,17 @@ func getTarget() vespa.Target { cfg.AuthConfigPath(), getSystemName(), cloudAuth, - endpoints) + endpoints, + ), nil } - fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL") - return nil + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL") } -func waitForService(service string, sessionOrRunID int64) { - s := getService(service, sessionOrRunID, "") +func waitForService(service string, sessionOrRunID int64) error { + s, err := getService(service, sessionOrRunID, "") + if err != nil { + return err + } timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) @@ -257,57 +242,58 @@ func waitForService(service string, sessionOrRunID int64) { log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Green("ready")) } else { if err == nil { - err = fmt.Errorf("Status %d", status) + err = fmt.Errorf("status %d", status) } - fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready")) + return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.Cyan(s.BaseURL), color.Red("not ready"), err) } + return nil } -func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) vespa.DeploymentOpts { +func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) (vespa.DeploymentOpts, error) { opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} if opts.IsCloud() { - deployment := deploymentFromArgs() + deployment, err := deploymentFromArgs() + if err != nil { + return vespa.DeploymentOpts{}, err + } if !opts.ApplicationPackage.HasCertificate() { - var msg string + var hint string if vespa.Auth0AccessTokenEnabled() { - msg = "Try 'vespa auth cert'" + hint = "Try 'vespa auth cert'" } else { - msg = "Try 'vespa cert'" + hint = "Try 'vespa cert'" } - fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", msg) - return opts + return vespa.DeploymentOpts{}, errHint(fmt.Errorf("missing certificate in application package"), "Applications in Vespa Cloud require a certificate", hint) } - var err error opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) if !vespa.Auth0AccessTokenEnabled() { if err != nil { - fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") - return opts + return vespa.DeploymentOpts{}, errHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") } } opts.Deployment = deployment } - return opts + return opts, nil } -func getEndpointsFromEnv() map[string]string { +func getEndpointsFromEnv() (map[string]string, error) { endpointsString := getEndpointsOverride() if endpointsString == "" { - return nil + return nil, nil } var endpoints endpoints urlsByCluster := make(map[string]string) if err := json.Unmarshal([]byte(endpointsString), &endpoints); err != nil { - fatalErrHint(err, "Endpoints must be valid JSON") + return nil, fmt.Errorf("endpoints must be valid json: %w", err) } if len(endpoints.Endpoints) == 0 { - fatalErr(fmt.Errorf("endpoints must be non-empty")) + return nil, fmt.Errorf("endpoints must be non-empty") } for _, endpoint := range endpoints.Endpoints { urlsByCluster[endpoint.Cluster] = endpoint.URL } - return urlsByCluster + return urlsByCluster, nil } type endpoints struct { diff --git a/client/go/cmd/log.go b/client/go/cmd/log.go index 4577e890959..d61eaecf35b 100644 --- a/client/go/cmd/log.go +++ b/client/go/cmd/log.go @@ -32,15 +32,21 @@ var logCmd = &cobra.Command{ Long: `Show the Vespa log. The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. + +Logs for the past hour are shown if no arguments are given. `, Example: `$ vespa log 1h $ vespa log --nldequote=false 10m $ vespa log --from 2021-08-25T15:00:00Z --to 2021-08-26T02:00:00Z $ vespa log --follow`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - target := getTarget() + RunE: func(cmd *cobra.Command, args []string) error { + target, err := getTarget() + if err != nil { + return err + } options := vespa.LogOptions{ Level: vespa.LogLevel(levelArg), Follow: followArg, @@ -49,30 +55,32 @@ $ vespa log --follow`, } if options.Follow { if fromArg != "" || toArg != "" || len(args) > 0 { - fatalErr(fmt.Errorf("cannot combine --from/--to or relative time with --follow"), "Could not follow logs") + return fmt.Errorf("cannot combine --from/--to or relative time with --follow") } options.From = time.Now().Add(-5 * time.Minute) } else { from, to, err := parsePeriod(args) if err != nil { - fatalErr(err, "Invalid period") - return + return fmt.Errorf("invalid period: %w", err) } options.From = from options.To = to } if err := target.PrintLog(options); err != nil { - fatalErr(err, "Could not retrieve logs") + return fmt.Errorf("could not retrieve logs: %w", err) } + return nil }, } func parsePeriod(args []string) (time.Time, time.Time, error) { - if len(args) == 1 { - if fromArg != "" || toArg != "" { - return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) + relativePeriod := fromArg == "" || toArg == "" + if relativePeriod { + period := "1h" + if len(args) > 0 { + period = args[0] } - d, err := time.ParseDuration(args[0]) + d, err := time.ParseDuration(period) if err != nil { return time.Time{}, time.Time{}, err } @@ -82,6 +90,8 @@ func parsePeriod(args []string) (time.Time, time.Time, error) { to := time.Now() from := to.Add(d) return from, to, nil + } else if len(args) > 0 { + return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) } from, err := time.Parse(time.RFC3339, fromArg) if err != nil { diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go index f239bebc488..4b32927ca17 100644 --- a/client/go/cmd/log_test.go +++ b/client/go/cmd/log_test.go @@ -24,5 +24,5 @@ func TestLog(t *testing.T) { assert.Equal(t, expected, out) _, errOut := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h"}}, t, httpClient) - assert.Equal(t, "Error: Invalid period\ncannot combine --from/--to with relative value: 1h\n", errOut) + assert.Equal(t, "Error: invalid period: cannot combine --from/--to with relative value: 1h\n", errOut) } diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index 5011b290b9f..f4438cdbb24 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -12,6 +12,7 @@ var loginCmd = &cobra.Command{ Short: "Authenticate the Vespa CLI", Example: "$ vespa auth login", DisableAutoGenTag: true, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() cfg, err := LoadConfig() @@ -29,7 +30,7 @@ var loginCmd = &cobra.Command{ return err } if err := cfg.Write(); err != nil { - fatalErr(err) + return err } } } diff --git a/client/go/cmd/man.go b/client/go/cmd/man.go index d90898117de..01fffd38a32 100644 --- a/client/go/cmd/man.go +++ b/client/go/cmd/man.go @@ -2,6 +2,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "github.com/spf13/cobra/doc" ) @@ -16,13 +18,14 @@ var manCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Hidden: true, // Not intended to be called by users DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { dir := args[0] err := doc.GenManTree(rootCmd, nil, dir) if err != nil { - fatalErr(err, "Failed to write man pages") - return + return fmt.Errorf("failed to write man pages: %w", err) } printSuccess("Man pages written to ", dir) + return nil }, } diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index c686f1d29ad..89f401a356e 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -33,10 +33,10 @@ Configure and deploy your application package to production in Vespa Cloud.`, Example: `$ vespa prod init $ vespa prod submit`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - // Root command does nothing - cmd.Help() - exitFunc(1) + SilenceUsage: false, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) }, } @@ -53,47 +53,50 @@ Reference: https://cloud.vespa.ai/en/reference/services https://cloud.vespa.ai/en/reference/deployment`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { appSource := applicationSource(args) pkg, err := vespa.FindApplicationPackage(appSource, false) if err != nil { - fatalErr(err) - return + return err } if pkg.IsZip() { - fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path), + return errHint(fmt.Errorf("cannot modify compressed application package %s", pkg.Path), "Try running 'mvn clean' and run this command again") - return } deploymentXML, err := readDeploymentXML(pkg) if err != nil { - fatalErr(err, "Could not read deployment.xml") - return + return fmt.Errorf("could not read deployment.xml: %w", err) } servicesXML, err := readServicesXML(pkg) if err != nil { - fatalErr(err, "A services.xml declaring your cluster(s) must exist") - return + return fmt.Errorf("a services.xml declaring your cluster(s) must exist: %w", err) } fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"), "!\nBefore modification a backup of the original file will be created.\n\n") fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") + fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.Green("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) r := bufio.NewReader(stdin) - deploymentXML = updateRegions(r, deploymentXML) - servicesXML = updateNodes(r, servicesXML) + deploymentXML, err = updateRegions(r, deploymentXML) + if err != nil { + return err + } + servicesXML, err = updateNodes(r, servicesXML) + if err != nil { + return err + } fmt.Fprintln(stdout) if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil { - fatalErr(err) - return + return err } if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil { - fatalErr(err) - return + return err } + return nil }, } @@ -116,49 +119,55 @@ For more information about production deployments in Vespa Cloud see: https://cloud.vespa.ai/en/getting-to-production https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, + SilenceUsage: true, Example: `$ mvn package # when adding custom Java components $ vespa prod submit`, - Run: func(cmd *cobra.Command, args []string) { - target := getTarget() + RunE: func(cmd *cobra.Command, args []string) error { + target, err := getTarget() + if err != nil { + return err + } if target.Type() != "cloud" { - fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type())) - return + return fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()) } appSource := applicationSource(args) pkg, err := vespa.FindApplicationPackage(appSource, true) if err != nil { - fatalErr(err) - return + return err } cfg, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") - return + return err } if !pkg.HasDeployment() { - fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init") - return + return errHint(fmt.Errorf("no deployment.xml found"), "Try creating one with vespa prod init") } if pkg.TestPath == "" { - fatalErrHint(fmt.Errorf("No tests found"), + return errHint(fmt.Errorf("no tests found"), "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", "See https://cloud.vespa.ai/en/getting-to-production") - return } - verifyTests(pkg.TestPath, target) + // TODO: Always verify tests. Do it before packaging, when running Maven from this CLI. + if !pkg.IsZip() { + verifyTests(pkg.TestPath, target) + } isCI := os.Getenv("CI") != "" if !isCI { fmt.Fprintln(stderr, color.Yellow("Warning:"), "We recommend doing this only from a CD job") printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production") } - opts := getDeploymentOpts(cfg, pkg, target) + opts, err := getDeploymentOpts(cfg, pkg, target) + if err != nil { + return err + } if err := vespa.Submit(opts); err != nil { - fatalErr(err, "Could not submit application for deployment") + return fmt.Errorf("could not submit application for deployment: %w", err) } else { printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment") log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application))) } + return nil }, } @@ -193,24 +202,27 @@ func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) er return ioutil.WriteFile(dst, []byte(contents), 0644) } -func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment { - regions := promptRegions(r, deploymentXML) +func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) (xml.Deployment, error) { + regions, err := promptRegions(r, deploymentXML) + if err != nil { + return xml.Deployment{}, err + } parts := strings.Split(regions, ",") regionElements := xml.Regions(parts...) if err := deploymentXML.Replace("prod", "region", regionElements); err != nil { - fatalErr(err, "Could not update region elements in deployment.xml") + return xml.Deployment{}, fmt.Errorf("could not update region elements in deployment.xml: %w", err) } // TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we // therefore remove <test> elements here. // This can be improved by supporting <test> elements in xml package and allow specifying testing as part of // region prompt, e.g. region1;test,region2 if err := deploymentXML.Replace("prod", "test", nil); err != nil { - fatalErr(err, "Could not remove test elements in deployment.xml") + return xml.Deployment{}, fmt.Errorf("could not remove test elements in deployment.xml: %w", err) } - return deploymentXML + return deploymentXML, nil } -func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string { +func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) (string, error) { fmt.Fprintln(stdout, color.Cyan("> Deployment regions")) fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones")) fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a")) @@ -235,47 +247,56 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string { return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator) } -func updateNodes(r *bufio.Reader, servicesXML xml.Services) xml.Services { +func updateNodes(r *bufio.Reader, servicesXML xml.Services) (xml.Services, error) { for _, c := range servicesXML.Container { - nodes := promptNodes(r, c.ID, c.Nodes) + nodes, err := promptNodes(r, c.ID, c.Nodes) + if err != nil { + return xml.Services{}, err + } if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil { - fatalErr(err) - return xml.Services{} + return xml.Services{}, err } } for _, c := range servicesXML.Content { - nodes := promptNodes(r, c.ID, c.Nodes) + nodes, err := promptNodes(r, c.ID, c.Nodes) + if err != nil { + return xml.Services{}, err + } if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil { - fatalErr(err) - return xml.Services{} + return xml.Services{}, err } } - return servicesXML + return servicesXML, nil } -func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) xml.Nodes { - count := promptNodeCount(r, clusterID, defaultValue.Count) +func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml.Nodes, error) { + count, err := promptNodeCount(r, clusterID, defaultValue.Count) + if err != nil { + return xml.Nodes{}, err + } const autoSpec = "auto" defaultSpec := autoSpec resources := defaultValue.Resources if resources != nil { defaultSpec = defaultValue.Resources.String() } - spec := promptResources(r, clusterID, defaultSpec) + spec, err := promptResources(r, clusterID, defaultSpec) + if err != nil { + return xml.Nodes{}, err + } if spec == autoSpec { resources = nil } else { r, err := xml.ParseResources(spec) if err != nil { - fatalErr(err) // Should not happen as resources have already been validated - return xml.Nodes{} + return xml.Nodes{}, err // Should not happen as resources have already been validated } resources = &r } - return xml.Nodes{Count: count, Resources: resources} + return xml.Nodes{Count: count, Resources: resources}, nil } -func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string { +func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) (string, error) { fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster")) fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services")) fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]")) @@ -286,7 +307,7 @@ func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator) } -func promptResources(r *bufio.Reader, clusterID string, resources string) string { +func promptResources(r *bufio.Reader, clusterID string, resources string) (string, error) { fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster")) fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services")) fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb")) @@ -321,7 +342,7 @@ func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) { return xml.ReadServices(f) } -func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) string { +func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) (string, error) { var input string for input == "" { fmt.Fprint(stdout, question) @@ -333,8 +354,7 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu var err error input, err = r.ReadString('\n') if err != nil { - fatalErr(err) - return "" + return "", err } input = strings.TrimSpace(input) if input == "" { @@ -347,7 +367,7 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu input = "" } } - return input + return input, nil } func verifyTests(testsParent string, target vespa.Target) { @@ -357,20 +377,20 @@ func verifyTests(testsParent string, target vespa.Target) { verifyTest(testsParent, "production-test", target, false) } -func verifyTest(testsParent string, suite string, target vespa.Target, required bool) { +func verifyTest(testsParent string, suite string, target vespa.Target, required bool) error { testDirectory := filepath.Join(testsParent, "tests", suite) _, err := os.Stat(testDirectory) if err != nil { if required { if errors.Is(err, os.ErrNotExist) { - fatalErrHint(fmt.Errorf("No %s tests found", suite), + return errHint(fmt.Errorf("no %s tests found: %w", suite, err), fmt.Sprintf("No such directory: %s", testDirectory), "See https://cloud.vespa.ai/en/reference/testing") } - fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") + return errHint(err, "See https://cloud.vespa.ai/en/reference/testing") } - return + return nil } - - runTests(testDirectory, true) + _, _, err = runTests(testDirectory, true) + return err } diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index a4f3ebd6b56..4e635f87a75 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -46,8 +46,8 @@ func TestProdInit(t *testing.T) { // Verify contents deploymentPath := filepath.Join(pkgDir, "src", "main", "application", "deployment.xml") deploymentXML := readFileString(t, deploymentPath) - assert.Contains(t, deploymentXML, `<region active="true">aws-us-west-2a</region>`) - assert.Contains(t, deploymentXML, `<region active="true">aws-eu-west-1a</region>`) + assert.Contains(t, deploymentXML, `<region>aws-us-west-2a</region>`) + assert.Contains(t, deploymentXML, `<region>aws-eu-west-1a</region>`) servicesPath := filepath.Join(pkgDir, "src", "main", "application", "services.xml") servicesXML := readFileString(t, servicesPath) @@ -90,7 +90,7 @@ func createApplication(t *testing.T, pkgDir string, java bool) { deploymentXML := `<deployment version="1.0"> <prod> - <region active="true">aws-us-east-1c</region> + <region>aws-us-east-1c</region> </prod> </deployment>` if err := ioutil.WriteFile(filepath.Join(appDir, "deployment.xml"), []byte(deploymentXML), 0644); err != nil { diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index 6638c275330..8ee022d7061 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -5,6 +5,7 @@ package cmd import ( + "fmt" "log" "net/http" "net/url" @@ -19,49 +20,62 @@ var queryTimeoutSecs int func init() { rootCmd.AddCommand(queryCmd) - queryCmd.Flags().IntVarP(&queryTimeoutSecs, "timeout", "T", 10, "Timeout for the query request in seconds") + queryCmd.Flags().IntVarP(&queryTimeoutSecs, "timeout", "T", 10, "Timeout for the query in seconds") } var queryCmd = &cobra.Command{ Use: "query query-parameters", Short: "Issue a query to Vespa", - Example: `$ vespa query "yql=select * from sources * where title contains 'foo';" hits=5`, + Example: `$ vespa query "yql=select * from music where album contains 'head';" hits=5`, Long: `Issue a query to Vespa. Any parameter from https://docs.vespa.ai/en/reference/query-api-reference.html can be set by the syntax [parameter-name]=[value].`, // TODO: Support referencing a query json file DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - query(args) + RunE: func(cmd *cobra.Command, args []string) error { + return query(args) }, } -func query(arguments []string) { - service := getService("query", 0, "") +func query(arguments []string) error { + service, err := getService("query", 0, "") + if err != nil { + return err + } url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() for i := 0; i < len(arguments); i++ { key, value := splitArg(arguments[i]) urlQuery.Set(key, value) } + queryTimeout := urlQuery.Get("timeout") + if queryTimeout == "" { + // No timeout set by user, use the timeout option + queryTimeout = fmt.Sprintf("%ds", queryTimeoutSecs) + urlQuery.Set("timeout", queryTimeout) + } url.RawQuery = urlQuery.Encode() - - response, err := service.Do(&http.Request{URL: url}, time.Second*time.Duration(queryTimeoutSecs)) + deadline, err := time.ParseDuration(queryTimeout) + if err != nil { + return fmt.Errorf("invalid query timeout: %w", err) + } + response, err := service.Do(&http.Request{URL: url}, deadline+time.Second) // Slightly longer than query timeout if err != nil { - fatalErr(nil, "Request failed: ", err) - return + return fmt.Errorf("request failed: %w", err) } defer response.Body.Close() if response.StatusCode == 200 { log.Print(util.ReaderToJSON(response.Body)) } else if response.StatusCode/100 == 4 { - fatalErr(nil, "Invalid query: ", response.Status, "\n", util.ReaderToJSON(response.Body)) + return fmt.Errorf("invalid query: %s\n%s", response.Status, util.ReaderToJSON(response.Body)) } else { - fatalErr(nil, response.Status, " from container at ", color.Cyan(url.Host), "\n", util.ReaderToJSON(response.Body)) + return fmt.Errorf("%s from container at %s\n%s", response.Status, color.Cyan(url.Host), util.ReaderToJSON(response.Body)) } + return nil } func splitArg(argument string) (string, string) { diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 55046ae49ba..09b1afb2b84 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -13,25 +13,25 @@ import ( func TestQuery(t *testing.T) { assertQuery(t, - "?yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", "select from sources * where title contains 'foo'") } func TestQueryNonJsonResult(t *testing.T) { assertQuery(t, - "?yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", "select from sources * where title contains 'foo'") } func TestQueryWithMultipleParameters(t *testing.T) { assertQuery(t, - "?hits=5&yql=select+from+sources+%2A+where+title+contains+%27foo%27", - "select from sources * where title contains 'foo'", "hits=5") + "?hits=5&timeout=20s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "select from sources * where title contains 'foo'", "hits=5", "timeout=20s") } func TestQueryWithExplicitYqlParameter(t *testing.T) { assertQuery(t, - "?yql=select+from+sources+%2A+where+title+contains+%27foo%27", + "?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", "yql=select from sources * where title contains 'foo'") } @@ -50,7 +50,10 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { "{\n \"query\": \"result\"\n}\n", executeCommand(t, client, []string{"query"}, query), "query output") - queryURL := queryServiceURL(client) + queryURL, err := queryServiceURL(client) + if err != nil { + t.Fatal(err) + } assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } @@ -59,7 +62,7 @@ func assertQueryError(t *testing.T, status int, errorMessage string) { client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) assert.Equal(t, - "Error: Invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", + "Error: invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", outErr, "error output") } @@ -74,6 +77,10 @@ func assertQueryServiceError(t *testing.T, status int, errorMessage string) { "error output") } -func queryServiceURL(client *mockHttpClient) string { - return getService("query", 0, "").BaseURL +func queryServiceURL(client *mockHttpClient) (string, error) { + service, err := getService("query", 0, "") + if err != nil { + return "", err + } + return service.BaseURL, nil } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 087be7c352d..e25c5cb2d0d 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -17,6 +17,15 @@ import ( "github.com/spf13/cobra" ) +// ErrCLI is an error returned to the user. It wraps an exit status, a regular error and optional hints for resolving +// the error. +type ErrCLI struct { + Status int + quiet bool + hints []string + error +} + var ( rootCmd = &cobra.Command{ Use: "vespa command-name", @@ -28,8 +37,14 @@ Prefer web service API's to this in production. Vespa documentation: https://docs.vespa.ai`, DisableAutoGenTag: true, - PersistentPreRun: func(cmd *cobra.Command, args []string) { - configureOutput() + SilenceErrors: true, // We have our own error printing + SilenceUsage: false, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return configureOutput() + }, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("invalid command: %s", args[0]) }, } @@ -64,7 +79,7 @@ func isTerminal() bool { return false } -func configureOutput() { +func configureOutput() error { if quietArg { stdout = ioutil.Discard } @@ -73,11 +88,11 @@ func configureOutput() { config, err := LoadConfig() if err != nil { - fatalErr(err, "Could not load config") + return err } colorValue, err := config.Get(colorFlag) if err != nil { - fatalErr(err) + return err } colorize := false @@ -88,9 +103,10 @@ func configureOutput() { colorize = true case "never": default: - fatalErrHint(fmt.Errorf("Invalid value for %s option", colorFlag), "Must be \"auto\", \"never\" or \"always\"") + return errHint(fmt.Errorf("invalid value for %s option", colorFlag), "Must be \"auto\", \"never\" or \"always\"") } color = aurora.NewAurora(colorize) + return nil } func init() { @@ -106,5 +122,20 @@ func init() { bindFlagToConfig(quietFlag, rootCmd) } -// Execute executes the root command. -func Execute() error { return rootCmd.Execute() } +// errHint creates a new CLI error, with optional hints that will be printed after the error +func errHint(err error, hints ...string) ErrCLI { return ErrCLI{Status: 1, hints: hints, error: err} } + +// Execute executes command and prints any errors. +func Execute() error { + err := rootCmd.Execute() + if err != nil { + if cliErr, ok := err.(ErrCLI); ok { + if !cliErr.quiet { + printErrHint(cliErr, cliErr.hints...) + } + } else { + printErr(err) + } + } + return err +} diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index c72df481547..711dba4aa9d 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -20,9 +20,10 @@ var statusCmd = &cobra.Command{ Short: "Verify that a service is ready to use (query by default)", Example: `$ vespa status query`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - waitForService("query", 0) + RunE: func(cmd *cobra.Command, args []string) error { + return waitForService("query", 0) }, } @@ -31,9 +32,10 @@ var statusQueryCmd = &cobra.Command{ Short: "Verify that the query service is ready to use (default)", Example: `$ vespa status query`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), - Run: func(cmd *cobra.Command, args []string) { - waitForService("query", 0) + RunE: func(cmd *cobra.Command, args []string) error { + return waitForService("query", 0) }, } @@ -42,9 +44,10 @@ var statusDocumentCmd = &cobra.Command{ Short: "Verify that the document service is ready to use", Example: `$ vespa status document`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), - Run: func(cmd *cobra.Command, args []string) { - waitForService("document", 0) + RunE: func(cmd *cobra.Command, args []string) error { + return waitForService("document", 0) }, } @@ -53,8 +56,9 @@ var statusDeployCmd = &cobra.Command{ Short: "Verify that the deploy service is ready to use", Example: `$ vespa status deploy`, DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), - Run: func(cmd *cobra.Command, args []string) { - waitForService("deploy", 0) + RunE: func(cmd *cobra.Command, args []string) error { + return waitForService("deploy", 0) }, } diff --git a/client/go/cmd/status_test.go b/client/go/cmd/status_test.go index 757ef5f3b06..631aa511459 100644 --- a/client/go/cmd/status_test.go +++ b/client/go/cmd/status_test.go @@ -82,7 +82,7 @@ func assertQueryStatusError(target string, args []string, t *testing.T) { cmd = append(cmd, args...) _, outErr := execute(command{args: cmd}, t, client) assert.Equal(t, - "Error: Container (query API) at "+target+" is not ready\nStatus 500\n", + "Error: Container (query API) at "+target+" is not ready: status 500\n", outErr, "vespa status container") } diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index 262b57eff33..179a8043a2a 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -9,9 +9,6 @@ import ( "crypto/tls" "encoding/json" "fmt" - "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/util" - "github.com/vespa-engine/vespa/client/go/vespa" "io/ioutil" "math" "net/http" @@ -20,6 +17,10 @@ import ( "path/filepath" "strings" "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/vespa" ) func init() { @@ -39,8 +40,13 @@ See https://cloud.vespa.ai/en/reference/testing.html for details.`, $ vespa test src/test/application/tests/system-test/feed-and-query.json`, Args: cobra.ExactArgs(1), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - if count, failed := runTests(args[0], false); len(failed) != 0 { + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + count, failed, err := runTests(args[0], false) + if err != nil { + return err + } + if len(failed) != 0 { plural := "s" if count == 1 { plural = "" @@ -49,26 +55,27 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, for _, test := range failed { fmt.Fprintln(stdout, test) } - exitFunc(3) + return ErrCLI{Status: 3, error: fmt.Errorf("tests failed"), quiet: true} } else { plural := "s" if count == 1 { plural = "" } fmt.Fprintf(stdout, "\n%s %d test%s OK\n", color.Green("Success:"), count, plural) + return nil } }, } -func runTests(rootPath string, dryRun bool) (int, []string) { +func runTests(rootPath string, dryRun bool) (int, []string, error) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { - fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") + return 0, nil, errHint(err, "See https://cloud.vespa.ai/en/reference/testing") } else if stat.IsDir() { tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required. if err != nil { - fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") + return 0, nil, errHint(err, "See https://cloud.vespa.ai/en/reference/testing") } context := testContext{testsPath: rootPath, dryRun: dryRun} previousFailed := false @@ -79,7 +86,10 @@ func runTests(rootPath string, dryRun bool) (int, []string) { fmt.Fprintln(stdout, "") previousFailed = false } - failure := runTest(testPath, context) + failure, err := runTest(testPath, context) + if err != nil { + return 0, nil, err + } if failure != "" { failed = append(failed, failure) previousFailed = true @@ -88,27 +98,30 @@ func runTests(rootPath string, dryRun bool) (int, []string) { } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun}) + failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun}) + if err != nil { + return 0, nil, err + } if failure != "" { failed = append(failed, failure) } count++ } if count == 0 { - fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing") + return 0, nil, errHint(fmt.Errorf("failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing") } - return count, failed + return count, failed, nil } // Runs the test at the given path, and returns the specified test name if the test fails -func runTest(testPath string, context testContext) string { +func runTest(testPath string, context testContext) (string, error) { var test test testBytes, err := ioutil.ReadFile(testPath) if err != nil { - fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") + return "", errHint(err, "See https://cloud.vespa.ai/en/reference/testing") } if err = json.Unmarshal(testBytes, &test); err != nil { - fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") + return "", errHint(fmt.Errorf("failed parsing test at %s: %w", testPath, err), "See https://cloud.vespa.ai/en/reference/testing") } testName := test.Name @@ -122,12 +135,12 @@ func runTest(testPath string, context testContext) string { defaultParameters, err := getParameters(test.Defaults.ParametersRaw, filepath.Dir(testPath)) if err != nil { fmt.Fprintln(stderr) - fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing") + return "", errHint(fmt.Errorf("invalid default parameters for %s: %w", testName, err), "See https://cloud.vespa.ai/en/reference/testing") } if len(test.Steps) == 0 { fmt.Fprintln(stderr) - fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") + return "", errHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") } for i, step := range test.Steps { stepName := fmt.Sprintf("Step %d", i+1) @@ -137,12 +150,12 @@ func runTest(testPath string, context testContext) string { failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context) if err != nil { fmt.Fprintln(stderr) - fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing") + return "", errHint(fmt.Errorf("error in %s: %w", stepName, err), "See https://cloud.vespa.ai/en/reference/testing") } if !context.dryRun { if failure != "" { fmt.Fprintf(stdout, " %s\n%s:\n%s\n", color.Red("failed"), stepName, longFailure) - return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure), nil } if i == 0 { fmt.Fprintf(stdout, " ") @@ -153,7 +166,7 @@ func runTest(testPath string, context testContext) string { if !context.dryRun { fmt.Fprintln(stdout, color.Green(" OK")) } - return "" + return "", nil } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails @@ -194,7 +207,11 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin } externalEndpoint := requestUrl.IsAbs() if !externalEndpoint && !context.dryRun { - service, err = context.target().Service("query", 0, 0, cluster) + target, err := context.target() + if err != nil { + return "", "", err + } + service, err = target.Service("query", 0, 0, cluster) if err != nil { return "", "", err } @@ -453,9 +470,13 @@ type testContext struct { dryRun bool } -func (t *testContext) target() vespa.Target { +func (t *testContext) target() (vespa.Target, error) { if t.lazyTarget == nil { - t.lazyTarget = getTarget() + target, err := getTarget() + if err != nil { + return nil, err + } + t.lazyTarget = target } - return t.lazyTarget + return t.lazyTarget, nil } diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index 6649353df77..2c59bbbc030 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -5,9 +5,6 @@ package cmd import ( - "fmt" - "github.com/vespa-engine/vespa/client/go/util" - "github.com/vespa-engine/vespa/client/go/vespa" "io/ioutil" "net/http" "net/url" @@ -16,6 +13,9 @@ import ( "strings" "testing" + "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/vespa" + "github.com/stretchr/testify/assert" ) @@ -40,7 +40,6 @@ func TestSuite(t *testing.T) { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) - fmt.Println(outBytes) assert.Equal(t, string(expectedBytes), outBytes) assert.Equal(t, "", errBytes) } @@ -51,7 +50,7 @@ func TestIllegalFileReference(t *testing.T) { client.NextStatus(200) _, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/illegal-reference.json"}}, t, client) assertRequests([]*http.Request{createRequest("GET", "http://127.0.0.1:8080/search/", "{}")}, client, t) - assert.Equal(t, "\nError: path may not point outside src/test/application, but 'foo/../../../../this-is-not-ok.json' does\nHint: Error in Step 2\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) + assert.Equal(t, "\nError: error in Step 2: path may not point outside src/test/application, but 'foo/../../../../this-is-not-ok.json' does\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestProductionTest(t *testing.T) { @@ -72,7 +71,7 @@ func TestTestWithoutAssertions(t *testing.T) { func TestSuiteWithoutTests(t *testing.T) { client := &mockHttpClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) - assert.Equal(t, "Error: Failed to find any tests at testdata/tests/staging-test\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) + assert.Equal(t, "Error: failed to find any tests at testdata/tests/staging-test\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestSingleTest(t *testing.T) { diff --git a/client/go/cmd/testdata/sample-apps-master.zip b/client/go/cmd/testdata/sample-apps-master.zip Binary files differindex 6ad49361072..c8fb40af713 100644 --- a/client/go/cmd/testdata/sample-apps-master.zip +++ b/client/go/cmd/testdata/sample-apps-master.zip diff --git a/client/go/cmd/version.go b/client/go/cmd/version.go index d2760402851..2660dfe7d61 100644 --- a/client/go/cmd/version.go +++ b/client/go/cmd/version.go @@ -46,14 +46,14 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Show current version and check for updates", DisableAutoGenTag: true, + SilenceUsage: true, Args: cobra.ExactArgs(0), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { log.Printf("vespa version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) if !skipVersionCheck && sp.isTerminal() { - if err := checkVersion(); err != nil { - fatalErr(err) - } + return checkVersion() } + return nil }, } diff --git a/client/go/cmd/vespa/main.go b/client/go/cmd/vespa/main.go index 32828b15aa4..f7ce064f3a5 100644 --- a/client/go/cmd/vespa/main.go +++ b/client/go/cmd/vespa/main.go @@ -5,12 +5,17 @@ package main import ( - "github.com/vespa-engine/vespa/client/go/cmd" "os" + + "github.com/vespa-engine/vespa/client/go/cmd" ) func main() { if err := cmd.Execute(); err != nil { - os.Exit(1) + if cliErr, ok := err.(cmd.ErrCLI); ok { + os.Exit(cliErr.Status) + } else { + os.Exit(1) + } } } diff --git a/client/go/go.mod b/client/go/go.mod index 70eea958933..08851a01f28 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -5,7 +5,6 @@ go 1.15 require ( github.com/briandowns/spinner v1.16.0 github.com/fatih/color v1.10.0 // indirect - github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/lestrrat-go/jwx v1.2.9 github.com/logrusorgru/aurora/v3 v3.0.0 diff --git a/client/go/go.sum b/client/go/go.sum index 59656af2b35..d27d422e5cc 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -178,8 +178,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd h1:nIzoSW6OhhppWLm4yqBwZsKJlAayUu5FGozhrF3ETSM= -github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd/go.mod h1:MEQrHur0g8VplbLOv5vXmDzacSaH9Z7XhcgsSh1xciU= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= diff --git a/client/go/util/http.go b/client/go/util/http.go index d5b8e3128ff..f95d52dabad 100644 --- a/client/go/util/http.go +++ b/client/go/util/http.go @@ -49,7 +49,7 @@ func CreateClient(timeout time.Duration) HttpClient { func HttpGet(host string, path string, description string) (*http.Response, error) { url, err := url.Parse(host + path) if err != nil { - return nil, fmt.Errorf("Invalid target URL: %s: %w", host+path, err) + return nil, fmt.Errorf("invalid target url: %s: %w", host+path, err) } return HttpDo(&http.Request{URL: url}, time.Second*10, description) } diff --git a/client/go/util/io.go b/client/go/util/io.go index 23bfec84879..68b50733006 100644 --- a/client/go/util/io.go +++ b/client/go/util/io.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "strings" ) @@ -53,7 +54,8 @@ func ReaderToJSON(reader io.Reader) string { // AtomicWriteFile atomically writes data to filename. func AtomicWriteFile(filename string, data []byte) error { - tmpFile, err := ioutil.TempFile("", "vespa") + dir := filepath.Dir(filename) + tmpFile, err := ioutil.TempFile(dir, "vespa") if err != nil { return err } diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index d52fc969c37..6ef4995e181 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -133,7 +133,7 @@ func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { if !ap.IsZip() { tempZip, err := ioutil.TempFile("", "vespa") if err != nil { - return nil, fmt.Errorf("Could not create a temporary zip file for the application package: %w", err) + return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) } defer func() { tempZip.Close() @@ -146,7 +146,7 @@ func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { } f, err := os.Open(zipFile) if err != nil { - return nil, fmt.Errorf("Could not open application package at %s: %w", ap.Path, err) + return nil, fmt.Errorf("could not open application package at %s: %w", ap.Path, err) } return f, nil } @@ -403,9 +403,9 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) func checkResponse(req *http.Request, response *http.Response, serviceDescription string) error { if response.StatusCode/100 == 4 { - return fmt.Errorf("Invalid application package (%s)\n\n%s", response.Status, extractError(response.Body)) + return fmt.Errorf("invalid application package (%s)\n%s", response.Status, extractError(response.Body)) } else if response.StatusCode != 200 { - return fmt.Errorf("Error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), req.URL.Host, response.Status, util.ReaderToJSON(response.Body)) + return fmt.Errorf("error from %s at %s (%s):\n%s", strings.ToLower(serviceDescription), req.URL.Host, response.Status, util.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 204dda6538f..f50716a5a3a 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -31,7 +31,7 @@ const ( queryService = "query" documentService = "document" - waitRetryInterval = 2 * time.Second + retryInterval = 2 * time.Second ) // Service represents a Vespa service. @@ -291,6 +291,9 @@ func (t *cloudTarget) PrepareApiRequest(req *http.Request, sigKeyId string) erro func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { a, err := auth0.GetAuth0(t.authConfigPath, t.systemName, t.apiURL) + if err != nil { + return err + } system, err := a.PrepareSystem(auth0.ContextWithCancel()) if err != nil { return err @@ -412,7 +415,7 @@ func (t *cloudTarget) printLog(response jobResponse, last int64) int64 { var msgs []logMessage for step, stepMsgs := range response.Log { for _, msg := range stepMsgs { - if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level { + if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level || LogLevel(msg.Type) == 3 { continue } msgs = append(msgs, msg) @@ -562,11 +565,11 @@ func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, time return statusCode, nil } } - timeLeft := deadline.Sub(time.Now()) - if loopOnce || timeLeft < waitRetryInterval { + timeLeft := time.Until(deadline) + if loopOnce || timeLeft < retryInterval { break } - time.Sleep(waitRetryInterval) + time.Sleep(retryInterval) } return statusCode, httpErr } diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go index e900b50cbb0..f1c598bec05 100644 --- a/client/go/vespa/xml/config.go +++ b/client/go/vespa/xml/config.go @@ -16,7 +16,7 @@ var DefaultDeployment Deployment func init() { defaultDeploymentRaw := `<deployment version="1.0"> <prod> - <region active="true">aws-us-east-1c</region> + <region>aws-us-east-1c</region> </prod> </deployment>` d, err := ReadDeployment(strings.NewReader(defaultDeploymentRaw)) @@ -44,8 +44,7 @@ type Prod struct { } type Region struct { - Name string `xml:",chardata"` - Active bool `xml:"active,attr"` + Name string `xml:",chardata"` } func (d Deployment) String() string { return d.rawXML.String() } @@ -142,14 +141,33 @@ func ReadServices(r io.Reader) (Services, error) { func Regions(names ...string) []Region { var regions []Region for _, z := range names { - regions = append(regions, Region{Name: z, Active: true}) + regions = append(regions, Region{Name: z}) } return regions } // ParseResources parses nodes resources from string s. func ParseResources(s string) (Resources, error) { - parts := strings.Split(s, ",") + var parts []string + inRange := false + var sb strings.Builder + for _, c := range s { + if inRange { + if c == ']' { + inRange = false + } + } else { + if c == '[' { + inRange = true + } else if c == ',' { + parts = append(parts, sb.String()) + sb.Reset() + continue + } + } + sb.WriteRune(c) + } + parts = append(parts, sb.String()) if len(parts) != 3 { return Resources{}, fmt.Errorf("invalid resources: %q", s) } @@ -180,11 +198,11 @@ func ParseNodeCount(s string) (int, int, error) { if len(parts) != 2 { return 0, 0, parseErr } - min, err := strconv.Atoi(parts[0]) + min, err := strconv.Atoi(strings.TrimSpace(parts[0])) if err != nil { return 0, 0, parseErr } - max, err := strconv.Atoi(parts[1]) + max, err := strconv.Atoi(strings.TrimSpace(parts[1])) if err != nil { return 0, 0, parseErr } @@ -208,10 +226,10 @@ func IsProdRegion(s string, system string) bool { func parseResource(field, s string) (string, error) { parts := strings.SplitN(s, "=", 2) - if len(parts) != 2 || parts[0] != field { + if len(parts) != 2 || strings.TrimSpace(parts[0]) != field { return "", fmt.Errorf("invalid value for %s field: %q", field, s) } - return parts[1], nil + return strings.TrimSpace(parts[1]), nil } // ReplaceRaw finds all elements of name in rawXML and replaces their contents with value. diff --git a/client/go/vespa/xml/config_test.go b/client/go/vespa/xml/config_test.go index 9d18636473b..0c94285a6e1 100644 --- a/client/go/vespa/xml/config_test.go +++ b/client/go/vespa/xml/config_test.go @@ -10,15 +10,15 @@ func TestReplaceDeployment(t *testing.T) { in := ` <deployment version="1.0"> <prod> - <region active="true">us-north-1</region> - <region active="false">eu-north-2</region> + <region>us-north-1</region> + <region>eu-north-2</region> </prod> </deployment>` out := `<deployment version="1.0"> <prod> - <region active="true">eu-south-1</region> - <region active="true">us-central-1</region> + <region>eu-south-1</region> + <region>us-central-1</region> </prod> </deployment> ` @@ -31,12 +31,12 @@ func TestReplaceDeploymentWithInstance(t *testing.T) { <deployment version="1.0"> <instance id="default"> <prod> - <region active="true">us-north-1</region> + <region>us-north-1</region> </prod> </instance> <instance id="beta"> <prod> - <region active="true">eu-south-1</region> + <region>eu-south-1</region> </prod> </instance> </deployment>` @@ -44,14 +44,14 @@ func TestReplaceDeploymentWithInstance(t *testing.T) { out := `<deployment version="1.0"> <instance id="default"> <prod> - <region active="true">us-central-1</region> - <region active="true">eu-west-1</region> + <region>us-central-1</region> + <region>eu-west-1</region> </prod> </instance> <instance id="beta"> <prod> - <region active="true">us-central-1</region> - <region active="true">eu-west-1</region> + <region>us-central-1</region> + <region>eu-west-1</region> </prod> </instance> </deployment> @@ -132,16 +132,16 @@ func TestReplaceRemovesElement(t *testing.T) { in := ` <deployment version="1.0"> <prod> - <region active="true">eu-south-1</region> - <region active="true">us-central-1</region> + <region>eu-south-1</region> + <region>us-central-1</region> <test>us-central-1</test> </prod> </deployment>` out := `<deployment version="1.0"> <prod> - <region active="true">eu-south-1</region> - <region active="true">us-central-1</region> + <region>eu-south-1</region> + <region>us-central-1</region> </prod> </deployment> ` @@ -247,11 +247,14 @@ func TestParseResources(t *testing.T) { assertResources(t, "vcpu=2,memory=4Gb", Resources{}, true) assertResources(t, "memory=4Gb,vcpu=2,disk=100Gb", Resources{}, true) assertResources(t, "vcpu=2,memory=4Gb,disk=100Gb", Resources{Vcpu: "2", Memory: "4Gb", Disk: "100Gb"}, false) + assertResources(t, " vcpu = 4, memory =8Gb, disk=500Gb ", Resources{Vcpu: "4", Memory: "8Gb", Disk: "500Gb"}, false) + assertResources(t, "vcpu=[2.5, 8],memory=[32Gb,150Gb],disk=[100Gb, 1Tb]", Resources{Vcpu: "[2.5, 8]", Memory: "[32Gb,150Gb]", Disk: "[100Gb, 1Tb]"}, false) } func TestParseNodeCount(t *testing.T) { assertNodeCount(t, "2", 2, 2, false) assertNodeCount(t, "[4,8]", 4, 8, false) + assertNodeCount(t, "[ 4, 8 ]", 4, 8, false) assertNodeCount(t, "foo", 0, 0, true) assertNodeCount(t, "[foo,bar]", 0, 0, true) diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 99aa3f9e1b2..bb54509852a 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -30,7 +30,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.44.v20210927</jetty.version> + <jetty.version>9.4.45.v20220203</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <junit5.version>5.8.1</junit5.version> <junit5.platform.version>1.8.1</junit5.platform.version> diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java index b0642e11fdf..939f09c9e33 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apputil.communication.http; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.CompletionHandler; @@ -22,12 +22,12 @@ import java.util.logging.Logger; * @author Harald Musum * @author Vegard Sjonfjell */ -public class JDiscHttpRequestHandler extends LoggingRequestHandler { +public class JDiscHttpRequestHandler extends ThreadedHttpRequestHandler { private static final Logger log = Logger.getLogger(JDiscHttpRequestHandler.class.getName()); private final HttpRequestHandler requestHandler; - public JDiscHttpRequestHandler(HttpRequestHandler handler, LoggingRequestHandler.Context parentCtx) { + public JDiscHttpRequestHandler(HttpRequestHandler handler, ThreadedHttpRequestHandler.Context parentCtx) { super(parentCtx); this.requestHandler = handler; } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java index ab05819b625..bf759e3dc76 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java @@ -18,7 +18,7 @@ public class StateRestApiV2HandlerTest { ClusterInfoConfig config = new ClusterInfoConfig( new ClusterInfoConfig.Builder().clusterId("cluster-id").nodeCount(1)); ClusterInfoConfig.Builder clusterConfig = new ClusterInfoConfig.Builder(); - new StateRestApiV2Handler(controller, config, StateRestApiV2Handler.testOnlyContext()); + new StateRestApiV2Handler(controller, config, StateRestApiV2Handler.testContext()); } @Test diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java index 765ce109d93..7a204aa791d 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java @@ -8,7 +8,7 @@ public class StatusHandlerTest { @Test public void testSimple() { ClusterController controller = new ClusterController(); - StatusHandler handler = new StatusHandler(controller, StatusHandler.testOnlyContext()); + StatusHandler handler = new StatusHandler(controller, StatusHandler.testContext()); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index ab638a1da7d..db86df88fc5 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -283,7 +283,7 @@ public class MasterElectionTest extends FleetControllerTest { waitForCompleteCycles(); timer.advanceTime(options.zooKeeperSessionTimeout); waitForZookeeperDisconnected(); - // Noone can be master if server is unavailable + // No one can be master if server is unavailable log.log(Level.INFO, "Checking master status"); for (int i=0; i<fleetControllers.size(); ++i) { assertFalse("Index " + i, fleetControllers.get(i).isMaster()); diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java index 4fe7ee74d6d..246092034de 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java @@ -46,6 +46,7 @@ public class Reindexer { private static final Logger log = Logger.getLogger(Reindexer.class.getName()); static final Duration failureGrace = Duration.ofMinutes(10); + static final Duration PROGRESS_TOKEN_STORE_INTERVAL = Duration.ofSeconds(60); private final Cluster cluster; private final List<Trigger> ready; @@ -160,7 +161,7 @@ public class Reindexer { public void onProgress(ProgressToken token) { super.onProgress(token); status.updateAndGet(value -> value.progressed(token)); - if (progressLastStored.get().isBefore(clock.instant().minusSeconds(10))) { + if (progressLastStored.get().isBefore(clock.instant().minus(PROGRESS_TOKEN_STORE_INTERVAL))) { progressLastStored.set(clock.instant()); database.writeReindexing(reindexing.updateAndGet(value -> value.with(type, status.get())), cluster.name()); metrics.dump(reindexing.get()); @@ -208,6 +209,7 @@ public class Reindexer { parameters.setThrottlePolicy(new DynamicThrottlePolicy().setWindowSizeIncrement(speed) .setWindowSizeDecrementFactor(3) .setResizeRate(5) + .setMaxWindowSize(128) .setMinWindowSize(3 + (int) (5 * speed))); parameters.setRemoteDataHandler(cluster.name()); parameters.setMaxPending(8); diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 0d77792de6d..44d729c802e 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -3,8 +3,7 @@ package ai.vespa.reindexing; import ai.vespa.reindexing.Reindexing.Status; import ai.vespa.reindexing.Reindexing.Trigger; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.document.DocumentType; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.document.DocumentTypeManager; import com.yahoo.path.Path; import com.yahoo.slime.Cursor; @@ -18,7 +17,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -87,7 +85,7 @@ public class ReindexingCurator { try { return curator.lock(lockPath(cluster), lockTimeout); } - catch (UncheckedTimeoutException e) { // TODO jonmv: Avoid use of guava classes. + catch (UncheckedTimeoutException e) { throw new ReindexingLockException(e); } } diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java index e784f070188..a9642c591ea 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java @@ -100,17 +100,23 @@ public class ReindexingMaintainer extends AbstractComponent { reindexer.shutdown(); executor.shutdown(); - if ( ! executor.awaitTermination(45, TimeUnit.SECONDS)) + + executor.awaitTermination(5, TimeUnit.SECONDS); // Give it 5s to complete gracefully. + + curator.close(); // Close the underlying curator independently to force shutdown + + if ( !executor.isShutdown() && ! executor.awaitTermination(5, TimeUnit.SECONDS)) log.log(WARNING, "Failed to shut down reindexing within timeout"); } catch (InterruptedException e) { log.log(WARNING, "Interrupted while waiting for reindexing to shut down"); Thread.currentThread().interrupt(); } - if ( ! executor.isShutdown()) - executor.shutdownNow(); + if ( ! executor.isShutdown()) { + List<Runnable> remaining = executor.shutdownNow(); + log.log(WARNING, "Number of tasks remaining at hard shutdown: " + remaining.size()); + } - curator.close(); } static List<Trigger> parseReady(ReindexingConfig.Clusters cluster, DocumentTypeManager manager) { diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index 25aaa0bce13..654481aee33 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.utils.staterestapi.server; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import java.util.logging.Level; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index cac9d21ee1f..ae9e7a22129 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -194,9 +194,10 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, java.util.List, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", + "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, java.util.List, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", "public com.yahoo.config.provision.InstanceName name()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision upgradeRevision()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout upgradeRollout()", "public java.util.List changeBlocker()", "public java.util.Optional globalServiceId()", @@ -365,6 +366,23 @@ "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy conservative" ] }, + "com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision[] values()", + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision separate", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRevision latest" + ] + }, "com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout": { "superClass": "java.lang.Enum", "interfaces": [], @@ -379,7 +397,8 @@ ], "fields": [ "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading" + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" ] }, "com.yahoo.config.application.api.DeploymentSpec": { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index d07df82fda1..c40ce1ebeb1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -25,9 +25,6 @@ import java.util.jar.JarEntry; * Represents an application package, that is, used as input when creating a VespaModel and as * a general reference to all contents in an application. * - * The class hides detail as to whether the source is local files or ZooKeeper - * data in config server. - * * @author Vegard Havdal */ public interface ApplicationPackage { @@ -129,7 +126,6 @@ public interface ApplicationPackage { * Returns the files in a directory as readers. The readers <b>must</b> * be closed by the caller. * - * * @param pathFromRoot the relative path string from the root of the application package * @param suffix the suffix of files to return, or null to return all * @param recurse return files in all subdirectories (recursively) as well @@ -141,7 +137,7 @@ public interface ApplicationPackage { /** Same as getFiles(pathFromRoot, suffix, false) */ default List<NamedReader> getFiles(Path pathFromRoot, String suffix) { - return getFiles(pathFromRoot,suffix,false); + return getFiles(pathFromRoot, suffix, false); } /** Returns the major version this application is valid for, or empty if it is valid for all versions */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index 67ddb9ef83c..ea38860c29b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -31,6 +31,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { private final InstanceName name; private final DeploymentSpec.UpgradePolicy upgradePolicy; + private final DeploymentSpec.UpgradeRevision upgradeRevision; private final DeploymentSpec.UpgradeRollout upgradeRollout; private final List<DeploymentSpec.ChangeBlocker> changeBlockers; private final Optional<String> globalServiceId; @@ -41,6 +42,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { public DeploymentInstanceSpec(InstanceName name, List<DeploymentSpec.Step> steps, DeploymentSpec.UpgradePolicy upgradePolicy, + DeploymentSpec.UpgradeRevision upgradeRevision, DeploymentSpec.UpgradeRollout upgradeRollout, List<DeploymentSpec.ChangeBlocker> changeBlockers, Optional<String> globalServiceId, @@ -51,6 +53,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { super(steps); this.name = name; this.upgradePolicy = upgradePolicy; + this.upgradeRevision = upgradeRevision; this.upgradeRollout = upgradeRollout; this.changeBlockers = changeBlockers; this.globalServiceId = globalServiceId; @@ -150,6 +153,9 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ public DeploymentSpec.UpgradePolicy upgradePolicy() { return upgradePolicy; } + /** Returns the upgrade revision strategy of this, which is separate if none is specified */ + public DeploymentSpec.UpgradeRevision upgradeRevision() { return upgradeRevision; } + /** Returns the upgrade rollout strategy of this, which is separate if none is specified */ public DeploymentSpec.UpgradeRollout upgradeRollout() { return upgradeRollout; } @@ -198,6 +204,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { DeploymentInstanceSpec other = (DeploymentInstanceSpec) o; return globalServiceId.equals(other.globalServiceId) && upgradePolicy == other.upgradePolicy && + upgradeRevision == other.upgradeRevision && upgradeRollout == other.upgradeRollout && changeBlockers.equals(other.changeBlockers) && steps().equals(other.steps()) && @@ -208,7 +215,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { @Override public int hashCode() { - return Objects.hash(globalServiceId, upgradePolicy, upgradeRollout, changeBlockers, steps(), athenzService, notifications, endpoints); + return Objects.hash(globalServiceId, upgradePolicy, upgradeRevision, upgradeRollout, changeBlockers, steps(), athenzService, notifications, endpoints); } @Override diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 88363db6e49..8ad42b1d4a8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -550,14 +550,23 @@ public class DeploymentSpec { } + /** Determines when application changes deploy, when an older revision is already rolling out. */ + public enum UpgradeRevision { + /** Separate: Application changes wait for previous application changes to complete, unless they fail. */ + separate, + /** Latest: Application changes immediately supersede previous application changes, unless currently blocked. */ + latest + } + + /** Determines when application changes deploy, when there is already an ongoing platform upgrade. */ public enum UpgradeRollout { /** Separate: Application changes wait for upgrade to complete, unless upgrade fails. */ separate, /** Leading: Application changes are allowed to start and catch up to the platform upgrade. */ - leading + leading, // /** Simultaneous: Application changes deploy independently of platform upgrades. */ - // simultaneous + simultaneous } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 8f866654d56..b12d4024591 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -165,6 +165,7 @@ public class DeploymentSpecXmlReader { // Values where the parent may provide a default DeploymentSpec.UpgradePolicy upgradePolicy = readUpgradePolicy(instanceTag, parentTag); + DeploymentSpec.UpgradeRevision upgradeRevision = readUpgradeRevision(instanceTag, parentTag); DeploymentSpec.UpgradeRollout upgradeRollout = readUpgradeRollout(instanceTag, parentTag); List<DeploymentSpec.ChangeBlocker> changeBlockers = readChangeBlockers(instanceTag, parentTag); Optional<AthenzService> athenzService = mostSpecificAttribute(instanceTag, athenzServiceAttribute).map(AthenzService::from); @@ -183,6 +184,7 @@ public class DeploymentSpecXmlReader { .map(name -> new DeploymentInstanceSpec(InstanceName.from(name), steps, upgradePolicy, + upgradeRevision, upgradeRollout, changeBlockers, globalServiceId.asOptional(), @@ -472,6 +474,25 @@ public class DeploymentSpecXmlReader { } } + private DeploymentSpec.UpgradeRevision readUpgradeRevision(Element parent, Element fallbackParent) { + Element upgradeElement = XML.getChild(parent, upgradeTag); + if (upgradeElement == null) + upgradeElement = XML.getChild(fallbackParent, upgradeTag); + if (upgradeElement == null) + return DeploymentSpec.UpgradeRevision.separate; + + String revision = upgradeElement.getAttribute("revision"); + if (revision.isEmpty()) + return DeploymentSpec.UpgradeRevision.separate; + + switch (revision) { + case "separate": return DeploymentSpec.UpgradeRevision.separate; + case "latest": return DeploymentSpec.UpgradeRevision.latest; + default: throw new IllegalArgumentException("Illegal upgrade revision '" + revision + "': " + + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRevision.values())); + } + } + private DeploymentSpec.UpgradeRollout readUpgradeRollout(Element parent, Element fallbackParent) { Element upgradeElement = XML.getChild(parent, upgradeTag); if (upgradeElement == null) @@ -486,9 +507,9 @@ public class DeploymentSpecXmlReader { switch (rollout) { case "separate": return DeploymentSpec.UpgradeRollout.separate; case "leading": return DeploymentSpec.UpgradeRollout.leading; - // case "simultaneous": return DeploymentSpec.UpgradePolicy.conservative; - default: throw new IllegalArgumentException("Illegal upgrade policy '" + rollout + "': " + - "Must be one of " + Arrays.toString(DeploymentSpec.UpgradePolicy.values())); + case "simultaneous": return DeploymentSpec.UpgradeRollout.simultaneous; + default: throw new IllegalArgumentException("Illegal upgrade rollout '" + rollout + "': " + + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRollout.values())); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 08ec615e4c0..a879b335a34 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -138,6 +138,7 @@ public class ApplicationClusterEndpoint { } public static class DnsName { + private static final int MAX_LABEL_LENGTH = 63; private final String name; @@ -150,11 +151,6 @@ public class ApplicationClusterEndpoint { return name; } - // TODO: remove when 7.508 is latest version - public static DnsName sharedNameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - return sharedNameFrom(SystemName.main, cluster, applicationId, suffix); - } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" @@ -162,11 +158,6 @@ public class ApplicationClusterEndpoint { return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label } - // TODO remove this method when 7.508 is latest version - public static DnsName sharedL4NameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - return sharedL4NameFrom(SystemName.main, cluster, applicationId, suffix); - } - public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java index 2cd2e980761..20fcb6b599b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java @@ -5,5 +5,11 @@ package com.yahoo.config.model.api; import java.util.List; public interface ApplicationClusterInfo { + List<ApplicationClusterEndpoint> endpoints(); + + boolean getDeferChangesUntilRestart(); + + String name(); + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e1707a56825..b4ece2dc28b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,37 +82,36 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int availableProcessors() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } - @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } + @ModelFeatureFlag(owners = {"geirst"}, removeAfter = "7.541") default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.75; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 1; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double tlsSizeFraction() { return 0.02; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } - @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean inhibitDefaultMergesWhenGlobalMergesPending() { return false; } + @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } + @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } } @@ -159,6 +158,7 @@ public interface ModelContext { default List<String> tlsCiphersOverride() { return List.of(); } default List<String> zoneDnsSuffixes() { return List.of(); } + List<String> environmentVariables(); } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 2fa2ba83291..f6af155ffc2 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -121,6 +121,7 @@ public class DeploymentSpecTest { assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); + assertEquals(DeploymentSpec.UpgradeRevision.separate, spec.requireInstance("default").upgradeRevision()); assertEquals(DeploymentSpec.UpgradeRollout.separate, spec.requireInstance("default").upgradeRollout()); } @@ -364,18 +365,37 @@ public class DeploymentSpecTest { } @Test + public void productionSpecWithUpgradeRevision() { + StringReader r = new StringReader( + "<deployment>" + + " <instance id='default'>" + + " <upgrade revision='latest' />" + + " </instance>" + + " <instance id='custom'/>" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("latest", spec.requireInstance("default").upgradeRevision().toString()); + assertEquals("separate", spec.requireInstance("custom").upgradeRevision().toString()); + } + + @Test public void productionSpecWithUpgradeRollout() { StringReader r = new StringReader( "<deployment>" + " <instance id='default'>" + " <upgrade rollout='leading' />" + " </instance>" + + " <instance id='aggressive'>" + + " <upgrade rollout='simultaneous' />" + + " </instance>" + " <instance id='custom'/>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("leading", spec.requireInstance("default").upgradeRollout().toString()); assertEquals("separate", spec.requireInstance("custom").upgradeRollout().toString()); + assertEquals("simultaneous", spec.requireInstance("aggressive").upgradeRollout().toString()); } @Test @@ -397,10 +417,10 @@ public class DeploymentSpecTest { public void upgradePolicyDefault() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <upgrade policy='canary' rollout='leading'/>" + + " <upgrade policy='canary' rollout='leading' revision='latest' />" + " <instance id='instance1'/>" + " <instance id='instance2'>" + - " <upgrade policy='conservative' rollout='separate'/>" + + " <upgrade policy='conservative' rollout='separate' revision='separate'/>" + " </instance>" + "</deployment>" ); @@ -408,6 +428,8 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("canary", spec.requireInstance("instance1").upgradePolicy().toString()); assertEquals("conservative", spec.requireInstance("instance2").upgradePolicy().toString()); + assertEquals("latest", spec.requireInstance("instance1").upgradeRevision().toString()); + assertEquals("separate", spec.requireInstance("instance2").upgradeRevision().toString()); assertEquals("leading", spec.requireInstance("instance1").upgradeRollout().toString()); assertEquals("separate", spec.requireInstance("instance2").upgradeRollout().toString()); } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 430e471afd0..5f36a26ccef 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -69,6 +69,7 @@ <extensions>true</extensions> <configuration> <instructions> + <_fixupmessages>"Classes found in the wrong directory"</_fixupmessages> <!-- Hide warnings for multi-release jars --> <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName> <Bundle-Version>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}</Bundle-Version> <Embed-Dependency>*;scope=compile|runtime</Embed-Dependency> diff --git a/config-model/src/main/Makefile b/config-model/src/main/Makefile index 85f0c888446..7b16bedb4c7 100644 --- a/config-model/src/main/Makefile +++ b/config-model/src/main/Makefile @@ -5,12 +5,12 @@ outputdir=../../target/generated-sources/trang/resources/schema trangjar=../../target/trang.jar -all: ${outputdir} ${outputdir}/services.rng ${outputdir}/hosts.rng ${outputdir}/container-include.rng ${outputdir}/services.xsd ${outputdir}/hosts.xsd ${outputdir}/container-include.xsd ${outputdir}/deployment.xsd +all: ${outputdir} ${outputdir}/services.rng ${outputdir}/hosts.rng ${outputdir}/container-include.rng ${outputdir}/services.xsd ${outputdir}/hosts.xsd ${outputdir}/container-include.xsd ${outputdir}/deployment.xsd ${outputdir}/validation-overrides.xsd ${outputdir}: mkdir -p ${outputdir} -${outputdir}/services.rng: ${srcdir}/services.rnc ${srcdir}/common.rnc ${srcdir}/admin.rnc ${srcdir}/clients.rnc ${srcdir}/docproc.rnc ${srcdir}/routing.rnc ${srcdir}/clients-v2.rnc ${srcdir}/content.rnc ${srcdir}/genericmodule.rnc ${srcdir}/legacygenericcluster.rnc ${srcdir}/genericcluster.rnc ${srcdir}/legacygenericmodule.rnc ${srcdir}/containercluster.rnc +${outputdir}/services.rng: ${srcdir}/services.rnc ${srcdir}/common.rnc ${srcdir}/admin.rnc ${srcdir}/clients.rnc ${srcdir}/docproc.rnc ${srcdir}/routing.rnc ${srcdir}/clients-v2.rnc ${srcdir}/content.rnc ${srcdir}/genericmodule.rnc ${srcdir}/genericcluster.rnc ${srcdir}/legacygenericmodule.rnc ${srcdir}/containercluster.rnc java -jar $(trangjar) -I rnc -O rng ${srcdir}/services.rnc ${outputdir}/services.rng ${outputdir}/services.xsd: ${outputdir}/services.rng @@ -39,7 +39,7 @@ ${outputdir}/deployment.xsd: ${outputdir}/deployment.rng ${outputdir}/validation-overrides.rng: ${srcdir}/validation-overrides.rnc java -jar $(trangjar) -I rnc -O rng ${srcdir}/validation-overrides.rnc ${outputdir}/validation-overrides.rng -${outputdir}/deployment.xsd: ${outputdir}/validation-overrides.rng +${outputdir}/validation-overrides.xsd: ${outputdir}/validation-overrides.rng java -jar $(trangjar) -I rng -O xsd ${outputdir}/validation-overrides.rng ${outputdir}/validation-overrides.xsd clean: diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 8d192414871..0e0f992952a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -138,12 +138,9 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter continue; } if (tagName.equals("config")) { - // TODO: disallow on Vespa 8 + // Top level config, mainly to be used by the Vespa team. continue; } - if (tagName.equals("cluster")) { - throw new IllegalArgumentException("<" + tagName + "> on top-level is not allowed anymore"); - } if ((tagName.equals("clients")) && deployState.isHosted()) throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); @@ -153,7 +150,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId); if (builders.isEmpty()) - throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); + throw new IllegalArgumentException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); for (ConfigModelBuilder builder : builders) { if ( ! model2Element.containsKey(builder)) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index dac62ce7e1b..300a77d4a6b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -28,10 +28,10 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; +import com.yahoo.searchdefinition.Application; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -40,7 +40,6 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; import com.yahoo.vespa.model.container.search.SemanticRules; -import com.yahoo.vespa.model.search.NamedSchema; import java.io.File; import java.io.FileNotFoundException; @@ -69,7 +68,7 @@ public class DeployState implements ConfigDefinitionStore { private final DeployLogger logger; private final FileRegistry fileRegistry; private final DocumentModel documentModel; - private final List<NamedSchema> schemas; + private final List<Schema> schemas; private final ApplicationPackage applicationPackage; private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; @@ -103,8 +102,7 @@ public class DeployState implements ConfigDefinitionStore { return new Builder().applicationPackage(applicationPackage).build(); } - private DeployState(ApplicationPackage applicationPackage, - SearchDocumentModel searchDocumentModel, + private DeployState(Application application, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, ExecutorService executor, @@ -130,15 +128,15 @@ public class DeployState implements ConfigDefinitionStore { this.fileRegistry = fileRegistry; this.executor = executor; this.rankProfileRegistry = rankProfileRegistry; - this.applicationPackage = applicationPackage; + this.applicationPackage = application.applicationPackage(); this.properties = properties; this.vespaVersion = vespaVersion; this.previousModel = previousModel; this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); this.provisioned = provisioned; - this.schemas = searchDocumentModel.getSchemas(); - this.documentModel = searchDocumentModel.getDocumentModel(); + this.schemas = List.copyOf(application.schemas().values()); + this.documentModel = application.documentModel(); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; this.endpoints = Set.copyOf(endpoints); @@ -161,7 +159,7 @@ public class DeployState implements ConfigDefinitionStore { return hostsReader == null ? new SingleNodeProvisioner() : new HostsXmlProvisioner(hostsReader); } catch (IOException e) { - throw new IllegalStateException("Could not read hosts.xml", e); + throw new RuntimeException("Could not read hosts.xml", e); } } @@ -217,7 +215,7 @@ public class DeployState implements ConfigDefinitionStore { ImportedMlModels importedModels = new ImportedMlModels(importFrom, executor, modelImporters); for (var entry : importedModels.getSkippedModels().entrySet()) { deployLogger.logApplicationPackage(Level.WARNING, "Skipping import of model " + entry.getKey() + " as an exception " + - "occurred during import. Error: " + entry.getValue()); + "occurred during import. Error: " + entry.getValue()); } return importedModels; } @@ -236,9 +234,7 @@ public class DeployState implements ConfigDefinitionStore { return applicationPackage; } - public List<NamedSchema> getSchemas() { - return schemas; - } + public List<Schema> getSchemas() { return schemas; } public DocumentModel getDocumentModel() { return documentModel; @@ -444,9 +440,10 @@ public class DeployState implements ConfigDefinitionStore { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); - SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, queryProfiles, validationParameters); - return new DeployState(applicationPackage, - searchDocumentModel, + Application application = new ApplicationBuilder(applicationPackage, fileRegistry, logger, properties, + rankProfileRegistry, queryProfiles.getRegistry()) + .build(! validationParameters.ignoreValidationErrors()); + return new DeployState(application, rankProfileRegistry, fileRegistry, executor, @@ -470,46 +467,6 @@ public class DeployState implements ConfigDefinitionStore { reindexing); } - private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, - ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSchemas(); - Map<String, String> names = new LinkedHashMap<>(); - SchemaBuilder builder = new SchemaBuilder(applicationPackage, fileRegistry, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); - for (NamedReader reader : readers) { - try { - String readerName = reader.getName(); - String topLevelName = builder.importReader(reader, readerName); - String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); - names.put(topLevelName, sdName); - if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + - "top level element ('" + topLevelName + - "') are not equal for file '" + readerName + "'"); - } - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); - } catch (IOException e) { - throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); - } finally { - closeIgnoreException(reader.getReader()); - } - } - builder.build(! validationParameters.ignoreValidationErrors()); - return SearchDocumentModel.fromBuilderAndNames(builder, names); - } - - private static String stripSuffix(String nodeName, String postfix) { - assert (nodeName.endsWith(postfix)); - return nodeName.substring(0, nodeName.length() - postfix.length()); - } - - @SuppressWarnings("EmptyCatchBlock") - private static void closeIgnoreException(Reader reader) { - try { - reader.close(); - } catch(Exception e) {} - } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java deleted file mode 100644 index aaba9a2b98c..00000000000 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.deploy; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.search.NamedSchema; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Internal helper class to retrieve document model and search definitions. - * - * @author Ulf Lilleengen - */ -public class SearchDocumentModel { - - private final DocumentModel documentModel; - private final List<NamedSchema> schemas; - - public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) { - this.documentModel = documentModel; - this.schemas = schemas; - - } - - public DocumentModel getDocumentModel() { - return documentModel; - } - - public List<NamedSchema> getSchemas() { - return schemas; - } - - public static SearchDocumentModel fromBuilderAndNames(SchemaBuilder builder, Map<String, String> names) { - List<NamedSchema> ret = new ArrayList<>(); - for (Schema schema : builder.getSchemaList()) { - ret.add(new NamedSchema(names.get(schema.getName()), schema)); - } - return new SearchDocumentModel(builder.getModel(), ret); - } - - public static SearchDocumentModel fromBuilder(SchemaBuilder builder) { - List<NamedSchema> ret = new ArrayList<>(); - for (Schema schema : builder.getSchemaList()) { - ret.add(new NamedSchema(schema.getName(), schema)); - } - return new SearchDocumentModel(builder.getModel(), ret); - } - -} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 28933a37ffa..9ae1a75b858 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.deploy; -import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -52,29 +51,30 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean enableFeedBlockInDistributor = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 100; - private boolean ignoreMergeQueueLimit = true; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); - private double resourceLimitDisk = 0.8; + private double resourceLimitDisk = 0.75; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; - private int distributorMergeBusyWait = 1; private int maxUnCommittedMemory = 123456; - private boolean distributorEnhancedMaintenanceScheduling = true; - private boolean asyncApplyBucketDiff = true; private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; + private double persistenceThrottlingWsDecrementFactor = 1.2; + private double persistenceThrottlingWsBackoff = 0.95; + private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; + private boolean useV8GeoPositions = false; + private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @@ -107,7 +107,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -115,22 +114,23 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } - @Override public int metricsproxyNumThreads() { return 1; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } - @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } - @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } + @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } + @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } + @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } + @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { @@ -199,11 +199,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) { - this.ignoreMergeQueueLimit = ignoreMergeQueueLimit; - return this; - } - public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -230,7 +225,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea } public TestProperties setConfigServerSpecs(List<Spec> configServerSpecs) { - this.configServerSpecs = ImmutableList.copyOf(configServerSpecs); + this.configServerSpecs = List.copyOf(configServerSpecs); return this; } @@ -259,11 +254,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { - enableFeedBlockInDistributor = enabled; - return this; - } - public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) { maxActivationInhibitedOutOfSyncGroups = nGroups; return this; @@ -304,21 +294,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setDistributorMergeBusyWait(int value) { - distributorMergeBusyWait = value; - return this; - } - - public TestProperties distributorEnhancedMaintenanceScheduling(boolean enhancedScheduling) { - distributorEnhancedMaintenanceScheduling = enhancedScheduling; - return this; - } - - public TestProperties setAsyncApplyBucketDiff(boolean value) { - asyncApplyBucketDiff = value; - return this; - } - public TestProperties setUnorderedMergeChaining(boolean unordered) { unorderedMergeChaining = unordered; return this; @@ -344,6 +319,36 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + + public TestProperties setPersistenceThrottlingWsDecrementFactor(double factor) { + this.persistenceThrottlingWsDecrementFactor = factor; + return this; + } + + public TestProperties setPersistenceThrottlingWsBackoff(double backoff) { + this.persistenceThrottlingWsBackoff = backoff; + return this; + } + + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { + this.inhibitDefaultMergesWhenGlobalMergesPending = value; + return this; + } + + public TestProperties setUseV8GeoPositions(boolean value) { + this.useV8GeoPositions = value; + return this; + } + + public TestProperties setEnvironmentVariables(List<String> value) { + this.environmentVariables = value; + return this; + } + public TestProperties setAvoidRenamingSummaryFeatures(boolean value) { this.avoidRenamingSummaryFeatures = value; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 47a5fb24a43..f64d71ee364 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -160,7 +160,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce * because config IDs must be registered through setConfigId(). */ public final String getConfigId() { - if (configId == null) throw new RuntimeException("The system topology must be frozen first."); + if (configId == null) throw new IllegalStateException("The system topology must be frozen first."); return configId; } @@ -234,6 +234,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce return didApply; } + @SuppressWarnings("removal") // TODO Vespa 8: remove private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) { ConfigInstance.Builder override; if (builder instanceof GenericConfig.GenericConfigBuilder) { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index c678618311e..c9a03dad65e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -72,7 +72,7 @@ public class Hosts { for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { String name = hostE.getAttribute("name"); if (name.equals("")) { - throw new RuntimeException("Missing 'name' attribute for host."); + throw new IllegalArgumentException("Missing 'name' attribute for host."); } if ("localhost".equals(name)) { name = HostName.getLocalhost(); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 5822b1bca05..824bf248b5c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -19,7 +19,7 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -95,8 +96,8 @@ public class MockApplicationPackage implements ApplicationPackage { /** Returns the root of this application package relative to the current dir */ protected File root() { return root; } + @SuppressWarnings("deprecation") // not redundant @Override - @SuppressWarnings("deprecation") // NOT redundant public String getApplicationName() { return "mock application"; } @@ -118,23 +119,26 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); - SchemaBuilder schemaBuilder = new SchemaBuilder(this, - new MockFileRegistry(), - new BaseDeployLogger(), - new TestProperties(), - new RankProfileRegistry(), - queryProfileRegistry); - for (String sd : schemas) { - try { - String name = schemaBuilder.importString(sd); - readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } + for (String sd : schemas) + readers.add(new NamedReader(extractSdName(sd) + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); return readers; } + /** To avoid either double parsing or supplying a name explicitly */ + private String extractSdName(String sd) { + String s = sd.split("\n")[0]; + if (s.startsWith("schema")) + s = s.substring("schema".length()).trim(); + else if (s.startsWith("search")) + s = s.substring("search".length()).trim(); + else + throw new IllegalArgumentException("Expected the first line of a schema but got '" + sd + "'"); + int end = s.indexOf(' '); + if (end < 0) + end = s.indexOf('}'); + return s.substring(0, end).trim(); + } + @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); @@ -142,7 +146,21 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getFiles(Path dir, String fileSuffix, boolean recurse) { - return new ArrayList<>(); + try { + if (dir.elements().contains(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getName())) return List.of(); // No legacy paths + File dirFile = new File(root, dir.getName()); + if ( ! dirFile.exists()) return List.of(); + if (recurse) throw new RuntimeException("Recurse not implemented"); + List<NamedReader> readers = new ArrayList<>(); + for (var i = Files.list(dirFile.toPath()).filter(p -> p.getFileName().toString().endsWith(fileSuffix)).iterator(); i.hasNext(); ) { + var file = i.next(); + readers.add(new NamedReader(file.toString(), IOUtils.createReader(file.toString()))); + } + return readers; + } + catch (IOException e) { + throw new RuntimeException(e); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java index 8848759b415..9f4eeeb44c9 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java @@ -27,8 +27,8 @@ public class DataTypeRepo implements DataTypeCollection { public DataTypeRepo add(DataType type) { if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) { - throw new IllegalStateException("Data type '" + type.getName() + "', id '" + - type.getId() + "' is already registered."); + throw new IllegalArgumentException("Data type '" + type.getName() + "', id '" + + type.getId() + "' is already registered."); } typeByName.put(type.getName(), type); typeById.put(type.getId(), type); @@ -43,9 +43,7 @@ public class DataTypeRepo implements DataTypeCollection { } public DataTypeRepo replace(DataType type) { - if (!typeByName.containsKey(type.getName()) || - !typeById.containsKey(type.getId())) - { + if (!typeByName.containsKey(type.getName()) || !typeById.containsKey(type.getId())) { throw new IllegalStateException("Data type '" + type.getName() + "' is not registered."); } var oldByName = typeByName.remove(type.getName()); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java index 42572ae763a..885db34510b 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java @@ -28,10 +28,10 @@ public class DocumentTypeRepo implements DocumentTypeCollection { public DocumentTypeRepo add(NewDocumentType type) { if (typeByName.containsKey(type.getFullName())) { - throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + throw new IllegalArgumentException("Document type " + type + " is already registered"); } if (typeById.containsKey(type.getFullName().getId())) { - throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + throw new IllegalArgumentException("Document type " + type + " is already registered"); } typeByName.put(type.getFullName(), type); typeById.put(type.getFullName().getId(), type); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index fb76c945344..2dda670f07c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -3,10 +3,20 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.searchdefinition.derived.SearchOrderer; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; /** * A collection of objects representing the content of an application package. @@ -18,28 +28,79 @@ import java.util.Map; public class Application { private final ApplicationPackage applicationPackage; - private final Map<String, Schema> schemas = new LinkedHashMap<>(); + private final Map<String, Schema> schemas; + private final DocumentModel documentModel; - public Application(ApplicationPackage applicationPackage) { + public Application(ApplicationPackage applicationPackage, + List<Schema> schemas, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, + ModelContext.Properties properties, + boolean documentsOnly, + boolean validate, + Set<Class<? extends Processor>> processorsToSkip, + DeployLogger logger) { this.applicationPackage = applicationPackage; - } - public ApplicationPackage applicationPackage() { return applicationPackage; } + Map<String, Schema> schemaMap = new LinkedHashMap<>(); + for (Schema schema : schemas) { + if (schemaMap.containsKey(schema.getName())) + throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); + schemaMap.put(schema.getName(), schema); + } + this.schemas = Collections.unmodifiableMap(schemaMap); + + schemas.forEach(schema -> schema.setOwner(this)); + if (validate) + schemas.forEach(schema -> schema.validate(logger)); + + new TemporarySDTypeResolver(schemas, logger).process(); + + List<SDDocumentType> sdocs = new ArrayList<>(); + sdocs.add(SDDocumentType.VESPA_DOCUMENT); + for (Schema schema : schemas) { + if (schema.hasDocument()) { + sdocs.add(schema.getDocument()); + } + } + + var orderer = new SDDocumentTypeOrderer(sdocs, logger); + orderer.process(); + for (SDDocumentType sdoc : orderer.getOrdered()) { + new FieldOperationApplierForStructs().process(sdoc); + new FieldOperationApplier().process(sdoc); + } + var resolver = new DocumentReferenceResolver(schemas); + sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemas); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - public void add(Schema schema) { - if (schemas.containsKey(schema.getName())) - throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); - schemas.put(schema.getName(), schema); + if (validate) + new DocumentGraphValidator().validateDocumentGraph(sdocs); + + List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); + for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? + new Processing(properties).process(schema, + logger, + rankProfileRegistry, + queryProfiles, + validate, + documentsOnly, + processorsToSkip); + } + + this.documentModel = new DocumentModelBuilder().build(schemasSomewhatOrdered); } + public ApplicationPackage applicationPackage() { return applicationPackage; } + /** Returns an unmodifiable list of the schemas of this application */ - public Map<String, Schema> schemas() { return Collections.unmodifiableMap(schemas); } + public Map<String, Schema> schemas() { return schemas; } - /** Validates this. Must be called after all content is added to it. */ - public void validate(DeployLogger logger) { - schemas.values().forEach(schema -> schema.validate(logger)); - } + public DocumentModel documentModel() { return documentModel; } @Override public String toString() { return "application " + applicationPackage.getApplicationId(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index 51466a5dbfa..533546b4d39 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -12,15 +12,13 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.searchdefinition.processing.Processor; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -28,28 +26,26 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; +import java.io.Reader; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; /** - * Helper class for building {@link Schema}s. The pattern for using this is to 1) Import - * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank - * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built - * search objects using the {@link #getSchema(String)} method. + * Application builder. Usage: + * 1) Add all schemas, using the addXXX() methods, + * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, + * 3) invoke the {@link #build} method */ -// NOTE: Since this was created we have added Application, and much of the content in this should migrate there. -public class SchemaBuilder { +public class ApplicationBuilder { - private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private final DocumentModel model = new DocumentModel(); - private final Application application; + private final ApplicationPackage applicationPackage; + private final List<Schema> schemas = new ArrayList<>(); + private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; private final FileRegistry fileRegistry; @@ -58,221 +54,224 @@ public class SchemaBuilder { /** True to build the document aspect only, skipping instantiation of rank profiles */ private final boolean documentsOnly; - private boolean isBuilt = false; + private Application application; private final Set<Class<? extends Processor>> processorsToSkip = new HashSet<>(); /** For testing only */ - public SchemaBuilder() { + public ApplicationBuilder() { this(new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger) { + public ApplicationBuilder(DeployLogger deployLogger) { this(MockApplicationPackage.createEmpty(), deployLogger); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); } /** Used for generating documents for typed access to document fields in Java */ - public SchemaBuilder(boolean documentsOnly) { + public ApplicationBuilder(boolean documentsOnly) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry) { this(rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); } - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } - public SchemaBuilder(ApplicationPackage app, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); } - private SchemaBuilder(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { - this.application = new Application(applicationPackage); + private ApplicationBuilder(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { + this.applicationPackage = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; this.fileRegistry = fileRegistry; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; + for (NamedReader reader : applicationPackage.getSchemas()) + addSchema(reader); } /** - * Import search definition. + * Adds a schema to this application. * * @param fileName the name of the file to import * @return the name of the imported object * @throws IOException thrown if the file can not be read for some reason * @throws ParseException thrown if the file does not contain a valid search definition */ - public String importFile(String fileName) throws IOException, ParseException { + public Schema addSchemaFile(String fileName) throws IOException, ParseException { File file = new File(fileName); - return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); - } - - private String importFile(Path file) throws IOException, ParseException { - return importFile(file.toString()); + return addSchema(IOUtils.readFile(file)); } /** - * Reads and parses the search definition string provided by the given reader. Once all search definitions have been - * imported, call {@link #build()}. + * Reads and parses the schema string provided by the given reader. Once all schemas have been + * imported, call {@link #build}. * - * @param reader the reader whose content to import - * @param searchDefDir the path to use when resolving file references - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition + * @param reader the reader whose content to import */ - public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir); + public void addSchema(NamedReader reader) { + try { + String schemaName = addSchema(IOUtils.readAll(reader)).getName(); + String schemaFileName = stripSuffix(reader.getName(), ApplicationPackage.SD_NAME_SUFFIX); + if ( ! schemaFileName.equals(schemaName)) { + throw new IllegalArgumentException("The file containing schema '" + schemaName + "' must be named '" + + schemaName + ApplicationPackage.SD_NAME_SUFFIX + "', not " + reader.getName()); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); + } finally { + closeIgnoreException(reader.getReader()); + } + } + + private static String stripSuffix(String readerName, String suffix) { + if ( ! readerName.endsWith(suffix)) + throw new IllegalArgumentException("Schema '" + readerName + "' does not end with " + suffix); + return readerName.substring(0, readerName.length() - suffix.length()); } /** - * Import search definition. + * Adds a schema to this * - * @param str the string to parse - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition + * @param schemaString the content of the schema */ - public String importString(String str) throws ParseException { - return importString(str, null); - } - - private String importString(String str, String searchDefDir) throws ParseException { - SimpleCharStream stream = new SimpleCharStream(str); - try { - return importRawSchema(new SDParser(stream, fileRegistry, deployLogger, properties, application, - rankProfileRegistry, documentsOnly) - .schema(docTypeMgr, searchDefDir)); - } catch (TokenMgrException e) { - throw new ParseException("Unknown symbol: " + e.getMessage()); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); - } + public Schema addSchema(String schemaString) throws ParseException { + return add(createSchema(schemaString)); } /** - * Registers the given schema to the application to be built during {@link #build()}. A + * Registers the given schema to the application to be built during {@link #build}. A * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed schemas used in unit tests. * * @param schema the object to import - * @return the name of the imported object * @throws IllegalArgumentException if the given search object has already been processed */ - public String importRawSchema(Schema schema) { + public Schema add(Schema schema) { if (schema.getName() == null) throw new IllegalArgumentException("Schema has no name"); - String rawName = schema.getName(); - application.add(schema); - return rawName; + schemas.add(schema); + return schema; } - /** - * Processes and finalizes the schemas of this. - * Only for testing. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public void build() { - build(true); + private Schema createSchema(String schemaString) throws ParseException { + Schema schema = parseSchema(schemaString); + addRankProfileFiles(schema); + return schema; } - /** - * Processes and finalizes the schemas of this. - * - * @throws IllegalStateException thrown if this method has already been called - */ - public void build(boolean validate) { - if (isBuilt) throw new IllegalStateException("Application already built"); + private Schema parseSchema(String schemaString) throws ParseException { + SimpleCharStream stream = new SimpleCharStream(schemaString); + try { + return parserOf(stream).schema(documentTypeManager); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } + } + + private void addRankProfileFiles(Schema schema) { + if (applicationPackage == null) return; - new TemporarySDTypeResolver(application.schemas().values(), deployLogger).process(); + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schema.getName()); + for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) + parseRankProfile(reader, schema); - if (validate) - application.validate(deployLogger); + Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schema.getName()); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) + parseRankProfile(reader, schema); + } - List<SDDocumentType> sdocs = new ArrayList<>(); - sdocs.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : application.schemas().values()) { - if (schema.hasDocument()) { - sdocs.add(schema.getDocument()); + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ + private void parseRankProfile(NamedReader reader, Schema schema) { + try { + SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + try { + parserOf(stream).rankProfile(schema); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); } } - - var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); - orderer.process(); - for (SDDocumentType sdoc : orderer.getOrdered()) { - new FieldOperationApplierForStructs().process(sdoc); - new FieldOperationApplier().process(sdoc); + catch (IOException e) { + throw new IllegalArgumentException("Could not read rank profile " + reader.getName(), e); } - - var resolver = new DocumentReferenceResolver(application.schemas().values()); - sdocs.forEach(resolver::resolveReferences); - sdocs.forEach(resolver::resolveInheritedReferences); - var importedFieldsEnumerator = new ImportedFieldsEnumerator(application.schemas().values()); - sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - - if (validate) - new DocumentGraphValidator().validateDocumentGraph(sdocs); - - var builder = new DocumentModelBuilder(model); - List<Schema> schemasSomewhatOrdered = new ArrayList<>(application.schemas().values()); - for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? - process(schema, new QueryProfiles(queryProfileRegistry, deployLogger), validate); + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse rank profile " + reader.getName(), e); } - builder.addToModel(schemasSomewhatOrdered); - isBuilt = true; } - /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ - public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + private SDParser parserOf(SimpleCharStream stream) { + return new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + rankProfileRegistry, documentsOnly); + } /** - * Processes and returns the given {@link Schema} object. This method has been factored out of the {@link - * #build()} method so that subclasses can choose not to build anything. + * Processes and finalizes the schemas of this. + * + * @throws IllegalStateException thrown if this method has already been called */ - private void process(Schema schema, QueryProfiles queryProfiles, boolean validate) { - new Processing(properties).process(schema, deployLogger, - rankProfileRegistry, queryProfiles, - validate, documentsOnly, - processorsToSkip); + public Application build(boolean validate) { + if (application != null) throw new IllegalStateException("Application already built"); + + application = new Application(applicationPackage, + schemas, + rankProfileRegistry, + new QueryProfiles(queryProfileRegistry, deployLogger), + properties, + documentsOnly, + validate, + processorsToSkip, + deployLogger); + return application; } + /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ + public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + /** * Convenience method to call {@link #getSchema(String)} when there is only a single {@link Schema} object * built. This method will never return null. @@ -281,7 +280,7 @@ public class SchemaBuilder { * @throws IllegalStateException if there is not exactly one search. */ public Schema getSchema() { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (application.schemas().size() != 1) throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + application.schemas().values()); @@ -289,9 +288,7 @@ public class SchemaBuilder { return application.schemas().values().stream().findAny().get(); } - public DocumentModel getModel() { - return model; - } + public DocumentModel getModel() { return application.documentModel(); } /** * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply @@ -300,10 +297,10 @@ public class SchemaBuilder { * @param name the name of the schema to return, * or null to return the only one or throw an exception if there are multiple to choose from * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build()} has not been called. + * @throws IllegalStateException if {@link #build} has not been called. */ public Schema getSchema(String name) { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (name == null) return getSchema(); return application.schemas().get(name); } @@ -323,24 +320,24 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a string. * * @param sd the string to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws ParseException thrown if there is a problem parsing the string */ - public static SchemaBuilder createFromString(String sd) throws ParseException { + public static ApplicationBuilder createFromString(String sd) throws ParseException { return createFromString(sd, new BaseDeployLogger()); } - public static SchemaBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(sd); + public static ApplicationBuilder createFromString(String sd, DeployLogger logger) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(sd); builder.build(true); return builder; } - public static SchemaBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); + public static ApplicationBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); for (var schema : schemas) - builder.importString(schema); + builder.addSchema(schema); builder.build(true); return builder; } @@ -349,26 +346,26 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. * * @param fileName the file to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - public static SchemaBuilder createFromFile(String fileName) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName) throws IOException, ParseException { return createFromFile(fileName, new BaseDeployLogger()); } /** * Convenience factory methdd to create a SearchBuilder from multiple SD files. Only for testing. */ - public static SchemaBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { + public static ApplicationBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { return createFromFiles(fileNames, new BaseDeployLogger()); } - public static SchemaBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); } - private static SchemaBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } @@ -378,14 +375,14 @@ public class SchemaBuilder { * @param fileName the file to build from. * @param deployLogger logger for deploy messages. * @param rankProfileRegistry registry for rank profiles. - * @return the built {@link SchemaBuilder} object. + * @return the built {@link ApplicationBuilder} object. * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - private static SchemaBuilder createFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, queryprofileRegistry); @@ -394,62 +391,62 @@ public class SchemaBuilder { /** * Convenience factory methdd to create a SearchBuilder from multiple SD files.. */ - private static SchemaBuilder createFromFiles(Collection<String> fileNames, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(MockApplicationPackage.createEmpty(), - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryprofileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryprofileRegistry); for (String fileName : fileNames) { - builder.importFile(fileName); + builder.addSchemaFile(fileName); } builder.build(true); return builder; } - public static SchemaBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); } - public static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); } - private static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + private static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, rankProfileRegistry, queryProfileRegistry); } - private static SchemaBuilder createFromDirectory(String dir, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryProfileRegistry); - for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.importFile(i.next()); + private static ApplicationBuilder createFromDirectory(String dir, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + ApplicationBuilder builder = new ApplicationBuilder(applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryProfileRegistry); + for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next().toString()); } builder.build(true); return builder; @@ -513,15 +510,15 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a raw object. * * @param rawSchema the raw object to build from - * @return the built {@link SchemaBuilder} object - * @see #importRawSchema(Schema) + * @return the built {@link ApplicationBuilder} object + * @see #add(Schema) */ - public static SchemaBuilder createFromRawSchema(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfileRegistry); - builder.importRawSchema(rawSchema); - builder.build(); + public static ApplicationBuilder createFromRawSchema(Schema rawSchema, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); + builder.add(rawSchema); + builder.build(true); return builder; } @@ -530,7 +527,7 @@ public class SchemaBuilder { * * @param rawSchema the raw object to build from * @return the built {@link Schema} object - * @see #importRawSchema(Schema) + * @see #add(Schema) */ public static Schema buildFromRawSchema(Schema rawSchema, RankProfileRegistry rankProfileRegistry, @@ -550,4 +547,10 @@ public class SchemaBuilder { public DeployLogger getDeployLogger() { return deployLogger; } + @SuppressWarnings("EmptyCatchBlock") + private static void closeIgnoreException(Reader reader) { + try { + reader.close(); + } catch(Exception e) {} + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index 56a739ced8b..812726a609e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -4,52 +4,48 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.ImmutableSDField; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; /** * The rank profile containing default settings. This is derived from the fields * whenever this is accessed. * - * @author bratseth + * @author bratseth */ public class DefaultRankProfile extends RankProfile { /** * Creates a new rank profile * - * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles. + * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} + * to use for storing and looking up rank profiles */ public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { super("default", schema, rankProfileRegistry, rankingConstants); } - /** - * Does nothing, the default rank profile can not inherit anything - */ - // TODO: Why not? If that's the case, then fail attempts at it - public void setInherited(String inheritedName) { - } - - /** Returns null, the default rank profile can not inherit anything */ - public String getInheritedName() { - return null; + /** Ignore self inheriting of default as some applications may use that for historical reasons. */ + public void inherit(String inheritedName) { + if (inheritedName.equals("default")) return; + super.inherit(inheritedName); } - /** Returns the rank boost value of the given field */ - public RankSetting getRankSetting(String fieldOrIndex,RankSetting.Type type) { - RankSetting setting = super.getRankSetting(fieldOrIndex,type); + @Override + public RankSetting getRankSetting(String fieldOrIndex, RankSetting.Type type) { + RankSetting setting = super.getRankSetting(fieldOrIndex, type); if (setting != null) return setting; - ImmutableSDField field = getSearch().getConcreteField(fieldOrIndex); + ImmutableSDField field = schema().getConcreteField(fieldOrIndex); if (field != null) { - setting = toRankSetting(field,type); + setting = toRankSetting(field, type); if (setting != null) return setting; } - Index index = getSearch().getIndex(fieldOrIndex); + Index index = schema().getIndex(fieldOrIndex); if (index != null) { - setting = toRankSetting(index,type); + setting = toRankSetting(index, type); if (setting != null) return setting; } @@ -57,13 +53,13 @@ public class DefaultRankProfile extends RankProfile { return null; } - private RankSetting toRankSetting(ImmutableSDField field,RankSetting.Type type) { - if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100) - return new RankSetting(field.getName(),type,field.getWeight()); + private RankSetting toRankSetting(ImmutableSDField field, RankSetting.Type type) { + if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight() > 0 && field.getWeight() != 100) + return new RankSetting(field.getName(), type, field.getWeight()); if (type.equals(RankSetting.Type.RANKTYPE)) - return new RankSetting(field.getName(),type,field.getRankType()); - if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost()>0) - return new RankSetting(field.getName(),type,field.getLiteralBoost()); + return new RankSetting(field.getName(), type, field.getRankType()); + if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost() > 0) + return new RankSetting(field.getName(), type, field.getLiteralBoost()); // Index level setting really if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { @@ -86,18 +82,19 @@ public class DefaultRankProfile extends RankProfile { * Returns the names of the fields which have a rank boost setting * explicitly in this profile or in fields */ + @Override public Set<RankSetting> rankSettings() { Set<RankSetting> settings = new LinkedHashSet<>(20); settings.addAll(this.rankSettings); - for (ImmutableSDField field : getSearch().allConcreteFields() ) { + for (ImmutableSDField field : schema().allConcreteFields() ) { addSetting(field, RankSetting.Type.WEIGHT, settings); addSetting(field, RankSetting.Type.RANKTYPE, settings); addSetting(field, RankSetting.Type.LITERALBOOST, settings); addSetting(field, RankSetting.Type.PREFERBITVECTOR, settings); } - // Foer settings that really pertains to indexes do the explicit indexes too - for (Index index : getSearch().getExplicitIndices()) { + // For settings that really pertains to indexes do the explicit indexes too + for (Index index : schema().getExplicitIndices()) { addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); } return settings; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 55f24123940..8b068381e07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -32,14 +32,13 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; - -import static java.util.Collections.emptySet; -import static java.util.stream.Collectors.toSet; +import java.util.stream.Collectors; /** * @author baldersheim @@ -48,12 +47,12 @@ public class DocumentModelBuilder { private final DocumentModel model; - public DocumentModelBuilder(DocumentModel model) { - this.model = model; - model.getDocumentManager().add(VespaDocumentType.INSTANCE); + public DocumentModelBuilder() { + this.model = new DocumentModel(); + this.model.getDocumentManager().add(VespaDocumentType.INSTANCE); } - public void addToModel(Collection<Schema> schemaList) { + public DocumentModel build(Collection<Schema> schemaList) { List<SDDocumentType> docList = new LinkedList<>(); for (Schema schema : schemaList) { docList.add(schema.getDocument()); @@ -65,6 +64,7 @@ public class DocumentModelBuilder { toAdd = tryAdd(schemaList)) { schemaList = toAdd; } + return model; } private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) { @@ -124,7 +124,7 @@ public class DocumentModelBuilder { return left; } - public void addToModel(Schema schema) { + private void addToModel(Schema schema) { // Then we add the search specific stuff SearchDef searchDef = new SearchDef(schema.getName()); addSearchFields(schema.extraFieldList(), searchDef); @@ -232,8 +232,7 @@ public class DocumentModelBuilder { @SuppressWarnings("deprecation") private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, Collection<NewDocumentType> docs, - Set<TypeReplacement> replacements) - { + Set<TypeReplacement> replacements) { DataType original = type; if (type instanceof TemporaryStructuredDataType) { DataType other = repo.getDataType(type.getId()); @@ -298,8 +297,7 @@ public class DocumentModelBuilder { } private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, - DataType dataType) - { + DataType dataType) { if (dataType instanceof TemporaryAnnotationReferenceDataType) { TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; if (refType.getId() != 0) { @@ -359,6 +357,7 @@ public class DocumentModelBuilder { addType(dt, s); return s; } + private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { if (sa.getInherits() != null) { AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); @@ -367,7 +366,7 @@ public class DocumentModelBuilder { } return false; } - @SuppressWarnings("deprecation") + private NewDocumentType convert(SDDocumentType sdoc) { Map<AnnotationType, String> annotationInheritance = new HashMap<>(); Map<StructDataType, String> structInheritance = new HashMap<>(); @@ -439,17 +438,17 @@ public class DocumentModelBuilder { private static Set<NewDocumentType.Name> convertDocumentReferencesToNames(Optional<DocumentReferences> documentReferences) { if (!documentReferences.isPresent()) { - return emptySet(); + return Set.of(); } return documentReferences.get().referenceMap().values().stream() .map(documentReference -> documentReference.targetSearch().getDocument()) .map(documentType -> new NewDocumentType.Name(documentType.getName())) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { if (importedFields == null) { - return emptySet(); + return Set.of(); } return Collections.unmodifiableSet(importedFields.fields().keySet()); } @@ -463,6 +462,7 @@ public class DocumentModelBuilder { } } } + private static void extractNestedTypes(NewDocumentType dt, DataType type) { if (type instanceof StructDataType) { StructDataType tmp = (StructDataType) type; @@ -484,8 +484,11 @@ public class DocumentModelBuilder { throw new IllegalArgumentException(type.toString()); } } + private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } + private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } + private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { DataType oldType = dt.getDataTypeRecursive(type.getId()); if (oldType == null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java index c672b662874..1d71a9f1494 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -14,8 +15,11 @@ import com.yahoo.searchdefinition.document.SDDocumentType; */ public class DocumentOnlySchema extends Schema { - public DocumentOnlySchema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - super(application, fileRegistry, deployLogger, properties); + public DocumentOnlySchema(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + super(applicationPackage, fileRegistry, deployLogger, properties); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 793fcc59f9d..14f07f224ab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -44,7 +44,7 @@ public class Index implements Cloneable, Serializable { private boolean prefix; /** The list of aliases (Strings) to this index name */ - private Set<String> aliases=new java.util.LinkedHashSet<>(1); + private Set<String> aliases = new java.util.LinkedHashSet<>(1); /** * The stemming setting of this field, or null to use the default. @@ -138,19 +138,18 @@ public class Index implements Cloneable, Serializable { } public String toString() { - String rankTypeName=rankType==null ? "(none)" : rankType.name(); - return - "index '" + name + - "' [ranktype: " + rankTypeName + - ", prefix: " + prefix + "]"; + String rankTypeName = rankType == null ? "(none)" : rankType.name(); + return "index '" + name + + "' [ranktype: " + rankTypeName + + ", prefix: " + prefix + "]"; } /** Makes a deep copy of this index */ @Override public Object clone() { try { - Index copy=(Index)super.clone(); - copy.aliases=new LinkedHashSet<>(this.aliases); + Index copy = (Index)super.clone(); + copy.aliases = new LinkedHashSet<>(this.aliases); return copy; } catch (CloneNotSupportedException e) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index fef7ff56763..08475813317 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -104,7 +104,6 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement @Override public TensorType getType(Reference reference) { // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: - boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); TensorType resolvedType = resolvedTypes.get(reference); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java index d40fe975721..4b849af9662 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java @@ -18,8 +18,8 @@ import java.util.Optional; public class OnnxModel extends DistributableResource { private OnnxModelInfo modelInfo = null; - private Map<String, String> inputMap = new HashMap<>(); - private Map<String, String> outputMap = new HashMap<>(); + private final Map<String, String> inputMap = new HashMap<>(); + private final Map<String, String> outputMap = new HashMap<>(); private String statelessExecutionMode = null; private Integer statelessInterOpThreads = null; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index d484d32b02f..49c52b21907 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -43,6 +43,7 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -57,15 +58,17 @@ public class RankProfile implements Cloneable { public final static String FIRST_PHASE = "firstphase"; public final static String SECOND_PHASE = "secondphase"; + /** The search definition-unique name of this rank profile */ private final String name; - /** The search definition owning this profile, or null if global (owned by a model) */ - private final ImmutableSchema search; + /** The schema owning this profile, or null if global (owned by a model) */ + private final ImmutableSchema schema; /** The name of the rank profile inherited by this */ - private String inheritedName = null; - private RankProfile inherited = null; + private final List<String> inheritedNames = new ArrayList<>(); + /** Stores the resolved inherited profiles, or null when not resolved. */ + private List<RankProfile> inherited; /** The match settings of this profile */ private MatchPhaseSettings matchPhaseSettings = null; @@ -95,10 +98,10 @@ public class RankProfile implements Cloneable { private double rankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; - private String inheritedSummaryFeatures; + private String inheritedSummaryFeaturesProfileName; private Set<ReferenceNode> matchFeatures; - private String inheritedMatchFeatures; + private String inheritedMatchFeaturesProfileName; private Set<ReferenceNode> rankFeatures; @@ -132,20 +135,6 @@ public class RankProfile implements Cloneable { private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; - private static class CachedFunctions { - private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; - private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; - CachedFunctions(Map<String, RankingExpressionFunction> functions) { - allRankingExpressionFunctions = functions; - ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); - for (var entry : functions.entrySet()) { - ExpressionFunction function = entry.getValue().function(); - mapBuilder.put(function.getName(), function); - } - allExpressionFunctions = mapBuilder.build(); - } - } - /** * Creates a new rank profile for a particular search definition * @@ -156,7 +145,7 @@ public class RankProfile implements Cloneable { */ public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = Objects.requireNonNull(schema, "search cannot be null"); + this.schema = Objects.requireNonNull(schema, "search cannot be null"); this.onnxModels = null; this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; @@ -172,7 +161,7 @@ public class RankProfile implements Cloneable { public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = null; + this.schema = null; this.rankProfileRegistry = rankProfileRegistry; this.rankingConstants = rankingConstants; this.onnxModels = onnxModels; @@ -180,10 +169,10 @@ public class RankProfile implements Cloneable { this.deployLogger = deployLogger; } - public String getName() { return name; } + public String name() { return name; } /** Returns the search definition owning this, or null if it is global */ - public ImmutableSchema getSearch() { return search; } + public ImmutableSchema schema() { return schema; } /** Returns the application this is part of */ public ApplicationPackage applicationPackage() { @@ -196,69 +185,76 @@ public class RankProfile implements Cloneable { } public Map<String, OnnxModel> onnxModels() { - return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); + return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); } private Stream<ImmutableSDField> allFields() { - if (search == null) return Stream.empty(); + if (schema == null) return Stream.empty(); if (allFieldsList == null) { - allFieldsList = search.allFieldsList(); + allFieldsList = schema.allFieldsList(); } return allFieldsList.stream(); } private Stream<ImmutableSDField> allImportedFields() { - return search != null ? search.allImportedFields() : Stream.empty(); + return schema != null ? schema.allImportedFields() : Stream.empty(); } /** - * Sets the name of the rank profile this inherits. Both rank profiles must be present in the same search - * definition + * Adds a profile to those inherited by this. + * The profile must belong to this schema (directly or by inheritance). */ - public void setInherited(String inheritedName) { - this.inheritedName = inheritedName; - } - - /** Returns the name of the profile this one inherits, or null if none is inherited */ - public String getInheritedName() { return inheritedName; } - - /** Returns the inherited rank profile, or null if there is none */ - private RankProfile getInherited() { - if (inheritedName == null) return null; - if (inherited == null) { - inherited = resolveInherited(); - if (inherited == null) { - String msg = "rank-profile '" + getName() + "' inherits '" + inheritedName + - "', but it does not exist anywhere in the inheritance of search '" + - ((getSearch() != null) ? getSearch().getName() : " global rank profiles") + "'."; - throw new IllegalArgumentException(msg); - } else { - List<String> children = new ArrayList<>(); - children.add(createFullyQualifiedName()); - verifyNoInheritanceCycle(children, inherited); - } - } + public void inherit(String inheritedName) { + inherited = null; + inheritedNames.add(inheritedName); + } + + /** Returns the names of the profiles this inherits, if any. */ + public List<String> inheritedNames() { return Collections.unmodifiableList(inheritedNames); } + + /** Returns the rank profiles inherited by this. */ + private List<RankProfile> inherited() { + if (inheritedNames.isEmpty()) return List.of(); + if (inherited != null) return inherited; + + inherited = resolveInheritedProfiles(schema); + List<String> children = new ArrayList<>(); + children.add(createFullyQualifiedName()); + inherited.forEach(profile -> verifyNoInheritanceCycle(children, profile)); return inherited; } private String createFullyQualifiedName() { - return (search != null) - ? (search.getName() + "." + getName()) - : getName(); + return (schema != null) + ? (schema.getName() + "." + name()) + : name(); } private void verifyNoInheritanceCycle(List<String> children, RankProfile parent) { children.add(parent.createFullyQualifiedName()); String root = children.get(0); - if (root.equals(parent.createFullyQualifiedName())) { + if (root.equals(parent.createFullyQualifiedName())) throw new IllegalArgumentException("There is a cycle in the inheritance for rank-profile '" + root + "' = " + children); + for (RankProfile parentInherited : parent.inherited()) + verifyNoInheritanceCycle(children, parentInherited); + } + + private List<RankProfile> resolveInheritedProfiles(ImmutableSchema schema) { + List<RankProfile> inherited = new ArrayList<>(); + for (String inheritedName : inheritedNames) { + RankProfile inheritedProfile = schema == null + ? rankProfileRegistry.getGlobal(inheritedName) + : resolveInheritedProfile(schema, inheritedName); + if (inheritedProfile == null) + throw new IllegalArgumentException("rank-profile '" + name() + "' inherits '" + inheritedName + + "', but this is not found in " + + ((schema() != null) ? schema() : " global rank profiles")); + inherited.add(inheritedProfile); } - if (parent.getInherited() != null) { - verifyNoInheritanceCycle(children, parent.getInherited()); - } + return inherited; } - private RankProfile resolveInherited(ImmutableSchema schema) { + private RankProfile resolveInheritedProfile(ImmutableSchema schema, String inheritedName) { SDDocumentType documentType = schema.getDocument(); if (documentType != null) { if (name.equals(inheritedName)) { @@ -273,25 +269,11 @@ public class RankProfile implements Cloneable { return rankProfileRegistry.get(schema.getName(), inheritedName); } - private RankProfile resolveInherited() { - if (inheritedName == null) return null; - return (getSearch() != null) - ? resolveInherited(search) - : rankProfileRegistry.getGlobal(inheritedName); - } - - /** - * Returns whether this profile inherits (directly or indirectly) the given profile - * - * @param name the profile name to compare this to. - * @return whether or not this inherits from the named profile. - */ + /** Returns whether this profile inherits (directly or indirectly) the given profile name. */ public boolean inherits(String name) { - RankProfile parent = getInherited(); - while (parent != null) { - if (parent.getName().equals(name)) - return true; - parent = parent.getInherited(); + for (RankProfile inheritedProfile : inherited()) { + if (inheritedProfile.name().equals(name)) return true; + if (inheritedProfile.inherits(name)) return true; } return false; } @@ -302,10 +284,20 @@ public class RankProfile implements Cloneable { } public MatchPhaseSettings getMatchPhaseSettings() { - MatchPhaseSettings settings = this.matchPhaseSettings; - if (settings != null) return settings; - if (getInherited() != null) return getInherited().getMatchPhaseSettings(); - return null; + if (matchPhaseSettings != null) return matchPhaseSettings; + return inheritedWith(p -> p.getMatchPhaseSettings() != null,"match phase settings") + .map(p -> p.getMatchPhaseSettings()).orElse(null); + } + + /** Returns the single profile having the property checked by the given filter, or empty if none */ + private Optional<RankProfile> inheritedWith(Predicate<RankProfile> property, String propertyDescription) { + List<RankProfile> matchingInherited = + inherited().stream().filter(profile -> property.test(profile)).collect(Collectors.toList()); + if (matchingInherited.isEmpty()) return Optional.empty(); + if (matchingInherited.size() == 1) return Optional.of(matchingInherited.get(0)); + throw new IllegalArgumentException("Only one of the profiles inherited by " + this + " can contain " + + propertyDescription + ", but it is present in all of " + matchingInherited); + } public void addRankSetting(RankSetting rankSetting) { @@ -319,15 +311,14 @@ public class RankProfile implements Cloneable { /** * Returns the a rank setting of a field, or null if there is no such rank setting in this profile * - * @param field the field whose settings to return. - * @param type the type that the field is required to be. - * @return the rank setting found, or null. + * @param field the field whose settings to return + * @param type the type that the field is required to be + * @return the rank setting found, or null */ RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) { RankSetting setting = i.next(); - if (setting.getFieldName().equals(field) && - setting.getType().equals(type)) { + if (setting.getFieldName().equals(field) && setting.getType() == type) { return setting; } } @@ -346,9 +337,8 @@ public class RankProfile implements Cloneable { RankSetting rankSetting = getDeclaredRankSetting(field, type); if (rankSetting != null) return rankSetting; - if (getInherited() != null) return getInherited().getRankSetting(field, type); - - return null; + return inheritedWith(p -> p.getRankSetting(field, type) != null, "rank setting " + type + " on " + field) + .map(p -> p.getRankSetting(field, type)).orElse(null); } /** @@ -374,12 +364,20 @@ public class RankProfile implements Cloneable { * Changes to the returned set will not be reflected in this rank profile. */ public Set<RankSetting> rankSettings() { - Set<RankSetting> allSettings = new LinkedHashSet<>(rankSettings); - RankProfile parent = getInherited(); - if (parent != null) - allSettings.addAll(parent.rankSettings()); + Set<RankSetting> settings = new LinkedHashSet<>(); + for (RankProfile inheritedProfile : inherited()) { + for (RankSetting setting : inheritedProfile.rankSettings()) { + if (settings.contains(setting)) + throw new IllegalArgumentException(setting + " is present in " + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + settings.add(setting); + } + } - return allSettings; + // TODO: Here we do things in the wrong order to not break tests. Reverse this. + Set<RankSetting> finalSettings = new LinkedHashSet<>(rankSettings); + finalSettings.addAll(settings); + return finalSettings; } public void addConstant(String name, Value value) { @@ -398,14 +396,20 @@ public class RankProfile implements Cloneable { /** Returns an unmodifiable view of the constants available in this */ public Map<String, Value> getConstants() { - if (constants.isEmpty()) - return getInherited() != null ? getInherited().getConstants() : Collections.emptyMap(); - if (getInherited() == null || getInherited().getConstants().isEmpty()) - return Collections.unmodifiableMap(constants); - - Map<String, Value> combinedConstants = new HashMap<>(getInherited().getConstants()); - combinedConstants.putAll(constants); - return combinedConstants; + if (inherited().isEmpty()) return new HashMap<>(constants); + + Map<String, Value> allConstants = new HashMap<>(); + for (var inheritedProfile : inherited()) { + for (var constant : inheritedProfile.getConstants().entrySet()) { + if (allConstants.containsKey(constant.getKey())) + throw new IllegalArgumentException("Constant '" + constant.getKey() + "' is present in " + + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + allConstants.put(constant.getKey(), constant.getValue()); + } + } + allConstants.putAll(constants); + return allConstants; } public void addAttributeType(String attributeName, String attributeType) { @@ -436,9 +440,8 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - RankProfile inherited = getInherited(); - if (inherited != null) return inherited.getFirstPhase(); - return null; + return inheritedWith(p -> p.getFirstPhase() != null, "first-phase expression") + .map(p -> p.getFirstPhase()).orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -465,9 +468,8 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - RankProfile inherited = getInherited(); - if (inherited != null) return inherited.getSecondPhase(); - return null; + return inheritedWith(p -> p.getSecondPhase() != null, "second-phase expression") + .map(p -> p.getSecondPhase()).orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -479,75 +481,80 @@ public class RankProfile implements Cloneable { } } - /** Returns a read-only view of the summary features to use in this profile. This is never null */ - public Set<ReferenceNode> getSummaryFeatures() { - if (inheritedSummaryFeatures != null && summaryFeatures != null) { - Set<ReferenceNode> combined = new HashSet<>(); - combined.addAll(getInherited().getSummaryFeatures()); - combined.addAll(summaryFeatures); - return Collections.unmodifiableSet(combined); - } - if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - if (getInherited() != null) return getInherited().getSummaryFeatures(); - return Set.of(); - } - - private void addSummaryFeature(ReferenceNode feature) { - if (summaryFeatures == null) - summaryFeatures = new LinkedHashSet<>(); - summaryFeatures.add(feature); - } - - /** Adds the content of the given feature list to the internal list of summary features. */ - public void addSummaryFeatures(FeatureList features) { - for (ReferenceNode feature : features) { - addSummaryFeature(feature); - } - } + // TODO: Below we have duplicate methods for summary and match features: Encapsulate this in a single parametrized + // class instead (and probably make rank features work the same). /** * Sets the name this should inherit the summary features of. - * Without setting this, this will either have the summary features of the parent, + * Without setting this, this will either have the summary features of the single parent setting them, * or if summary features are set in this, only have the summary features in this. * With this set the resulting summary features of this will be the superset of those defined in this and * the final (with inheritance included) summary features of the given parent. - * The profile must be the profile which is directly inherited by this. - * + * The profile must be one which is directly inherited by this. */ public void setInheritedSummaryFeatures(String parentProfile) { - if ( ! parentProfile.equals(inheritedName)) - throw new IllegalArgumentException("This can only inherit the summary features of its parent, '" + - inheritedName + ", but attempting to inherit '" + parentProfile); - this.inheritedSummaryFeatures = parentProfile; + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the summary features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedSummaryFeaturesProfileName = parentProfile; } /** * Sets the name of a profile this should inherit the match features of. - * Without setting this, this will either have the match features of the parent, + * Without setting this, this will either have the match features of the single parent setting them, * or if match features are set in this, only have the match features in this. * With this set the resulting match features of this will be the superset of those defined in this and * the final (with inheritance included) match features of the given parent. - * The profile must be the profile which is directly inherited by this. + * The profile must be one which which is directly inherited by this. * */ public void setInheritedMatchFeatures(String parentProfile) { - if ( ! parentProfile.equals(inheritedName)) - throw new IllegalArgumentException("This rank profile ("+name+") can only inherit the match features of its parent, '" + - inheritedName + ", but attemtping to inherit '" + parentProfile); - this.inheritedMatchFeatures = parentProfile; + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the match features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedMatchFeaturesProfileName = parentProfile; + } + + /** Returns a read-only view of the summary features to use in this profile. This is never null */ + public Set<ReferenceNode> getSummaryFeatures() { + if (inheritedSummaryFeaturesProfileName != null && summaryFeatures != null) { + Set<ReferenceNode> combined = new HashSet<>(); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedSummaryFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getSummaryFeatures()); + combined.addAll(summaryFeatures); + return Collections.unmodifiableSet(combined); + } + if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); + return inheritedWith(p -> ! p.getSummaryFeatures().isEmpty(), "summary features") + .map(p -> p.getSummaryFeatures()) + .orElse(Set.of()); } /** Returns a read-only view of the match features to use in this profile. This is never null */ public Set<ReferenceNode> getMatchFeatures() { - if (inheritedMatchFeatures != null && matchFeatures != null) { + if (inheritedMatchFeaturesProfileName != null && matchFeatures != null) { Set<ReferenceNode> combined = new HashSet<>(); - combined.addAll(getInherited().getMatchFeatures()); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedMatchFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getMatchFeatures()); combined.addAll(matchFeatures); return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - if (getInherited() != null) return getInherited().getMatchFeatures(); - return Set.of(); + return inheritedWith(p -> ! p.getMatchFeatures().isEmpty(), "match features") + .map(p -> p.getMatchFeatures()) + .orElse(Set.of()); + } + + private void addSummaryFeature(ReferenceNode feature) { + if (summaryFeatures == null) + summaryFeatures = new LinkedHashSet<>(); + summaryFeatures.add(feature); } private void addMatchFeature(ReferenceNode feature) { @@ -556,6 +563,13 @@ public class RankProfile implements Cloneable { matchFeatures.add(feature); } + /** Adds the content of the given feature list to the internal list of summary features. */ + public void addSummaryFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addSummaryFeature(feature); + } + } + /** Adds the content of the given feature list to the internal list of match features. */ public void addMatchFeatures(FeatureList features) { for (ReferenceNode feature : features) { @@ -566,8 +580,8 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - if (getInherited() != null) return getInherited().getRankFeatures(); - return Collections.emptySet(); + return inheritedWith(p -> ! p.getRankFeatures().isEmpty(), "summary-features") + .map(p -> p.getRankFeatures()).orElse(Set.of()); } private void addRankFeature(ReferenceNode feature) { @@ -598,12 +612,15 @@ public class RankProfile implements Cloneable { /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ public Map<String, List<RankProperty>> getRankPropertyMap() { - if (rankProperties.size() == 0 && getInherited() == null) return Collections.emptyMap(); - if (rankProperties.size() == 0) return getInherited().getRankPropertyMap(); - if (getInherited() == null) return Collections.unmodifiableMap(rankProperties); + if (rankProperties.size() == 0 && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); + + var inheritedProperties = inheritedWith(p -> ! p.getRankPropertyMap().isEmpty(), "rank-properties") + .map(p -> p.getRankPropertyMap()).orElse(Map.of()); + if (rankProperties.isEmpty()) return inheritedProperties; // Neither is null - Map<String, List<RankProperty>> combined = new LinkedHashMap<>(getInherited().getRankPropertyMap()); + Map<String, List<RankProperty>> combined = new LinkedHashMap<>(inheritedProperties); combined.putAll(rankProperties); // Don't combine values across inherited properties return Collections.unmodifiableMap(combined); } @@ -617,57 +634,44 @@ public class RankProfile implements Cloneable { rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty); } - @Override - public String toString() { - return "rank profile '" + getName() + "'"; - } + public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } public int getRerankCount() { - return (rerankCount < 0 && (getInherited() != null)) - ? getInherited().getRerankCount() - : rerankCount; + if (rerankCount >= 0) return rerankCount; + return inheritedWith(p -> p.getRerankCount() >= 0, "rerank-count") + .map(p -> p.getRerankCount()).orElse(-1); } + public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } + public int getNumThreadsPerSearch() { - return (numThreadsPerSearch < 0 && (getInherited() != null)) - ? getInherited().getNumThreadsPerSearch() - : numThreadsPerSearch; + if (numThreadsPerSearch >= 0) return numThreadsPerSearch; + return inheritedWith(p -> p.getNumThreadsPerSearch() >= 0, "num-threads-per-search") + .map(p -> p.getNumThreadsPerSearch()).orElse(-1); } - public void setNumThreadsPerSearch(int numThreads) { - this.numThreadsPerSearch = numThreads; - } + public void setMinHitsPerThread(int minHits) { this.minHitsPerThread = minHits; } public int getMinHitsPerThread() { - return (minHitsPerThread < 0 && (getInherited() != null)) - ? getInherited().getMinHitsPerThread() - : minHitsPerThread; - } - - public void setMinHitsPerThread(int minHits) { - this.minHitsPerThread = minHits; + if (minHitsPerThread >= 0) return minHitsPerThread; + return inheritedWith(p -> p.getMinHitsPerThread() >= 0, "min-hits-per-search") + .map(p -> p.getMinHitsPerThread()).orElse(-1); } - public void setNumSearchPartitions(int numSearchPartitions) { - this.numSearchPartitions = numSearchPartitions; - } + public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { - return (numSearchPartitions < 0 && (getInherited() != null)) - ? getInherited().getNumSearchPartitions() - : numSearchPartitions; + if (numSearchPartitions >= 0) return numSearchPartitions; + return inheritedWith(p -> p.getNumSearchPartitions() >= 0, "num-search-partitions") + .map(p -> p.getNumSearchPartitions()).orElse(-1); } - public OptionalDouble getTermwiseLimit() { - return ((termwiseLimit == null) && (getInherited() != null)) - ? getInherited().getTermwiseLimit() - : (termwiseLimit != null) ? OptionalDouble.of(termwiseLimit) : OptionalDouble.empty(); - } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } - /** Sets the rerank count. Set to -1 to use inherited */ - public void setRerankCount(int rerankCount) { - this.rerankCount = rerankCount; + public OptionalDouble getTermwiseLimit() { + if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); + return inheritedWith(p -> p.getTermwiseLimit().isPresent(), "termwise-limit") + .map(p -> p.getTermwiseLimit()).orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -677,10 +681,26 @@ public class RankProfile implements Cloneable { public boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); + return inheritedWith(p -> p.ignoreDefaultRankFeatures != null, "ignore-default-rank-features") + .map(p -> p.getIgnoreDefaultRankFeatures()).orElse(false); + } + + public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } + + public int getKeepRankCount() { + if (keepRankCount >= 0) return keepRankCount; + return inheritedWith(p -> p.getKeepRankCount() >= 0, "keep-rank-count") + .map(p -> p.getKeepRankCount()).orElse(-1); + } + + public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } + + public double getRankScoreDropLimit() { + if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; + return inheritedWith(p -> p.getRankScoreDropLimit() > -Double.MAX_VALUE, "rank.score-drop-limit") + .map(p -> p.getRankScoreDropLimit()).orElse(rankScoreDropLimit); } - /** Adds a function */ public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -742,10 +762,11 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction findFunction(String name) { RankingExpressionFunction function = functions.get(name); - return ((function == null) && (getInherited() != null)) - ? getInherited().findFunction(name) - : function; + if (function != null) return function; + return inheritedWith(p -> p.findFunction(name) != null, "function '" + name + "'") + .map(p -> p.findFunction(name)).orElse(null); } + /** Returns an unmodifiable snapshot of the functions in this */ public Map<String, RankingExpressionFunction> getFunctions() { updateCachedFunctions(); @@ -762,75 +783,58 @@ public class RankProfile implements Cloneable { } private Map<String, RankingExpressionFunction> gatherAllFunctions() { - if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap(); - if (functions.isEmpty()) return getInherited().getFunctions(); - if (getInherited() == null) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); - - // Neither is null - Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + if (functions.isEmpty() && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); + + // Combine + Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + for (var function : inheritedProfile.getFunctions().entrySet()) { + if (allFunctions.containsKey(function.getKey())) + throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + + function.getValue() + ", but this function is already " + + "defined in another profile this inherits"); + allFunctions.put(function.getKey(), function.getValue()); + } + } allFunctions.putAll(functions); return Collections.unmodifiableMap(allFunctions); } private boolean needToUpdateFunctionCache() { - if (getInherited() != null) - return (allFunctionsCached == null) || getInherited().needToUpdateFunctionCache(); + if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; return allFunctionsCached == null; } - public int getKeepRankCount() { - if (keepRankCount >= 0) return keepRankCount; - if (getInherited() != null) return getInherited().getKeepRankCount(); - return -1; - } - - public void setKeepRankCount(int rerankArraySize) { - this.keepRankCount = rerankArraySize; - } + public Set<String> filterFields() { return filterFields; } - public double getRankScoreDropLimit() { - if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited() != null) return getInherited().getRankScoreDropLimit(); - return rankScoreDropLimit; - } + /** Returns all filter fields in this profile and any profile it inherits. */ + public Set<String> allFilterFields() { + Set<String> inheritedFilterFields = + inheritedWith(p -> ! p.allFilterFields().isEmpty(), "filter fields") + .map(p -> p.allFilterFields()).orElse(Set.of()); - public void setRankScoreDropLimit(double rankScoreDropLimit) { - this.rankScoreDropLimit = rankScoreDropLimit; - } + if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); - public Set<String> filterFields() { - return filterFields; - } - - /** - * Returns all filter fields in this profile and any profile it inherits. - * - * @return the set of all filter fields - */ - public Set<String> allFilterFields() { - RankProfile parent = getInherited(); - Set<String> retval = new LinkedHashSet<>(); - if (parent != null) { - retval.addAll(parent.allFilterFields()); - } - retval.addAll(filterFields()); - return retval; + Set<String> combined = new LinkedHashSet<>(inheritedFilterFields); + combined.addAll(filterFields()); + return combined; } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); + throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + name + "."); + "' in " + name() + ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + name + "'"); + throw new RuntimeException("IOException parsing ranking expression '" + name + "'", e); } } @@ -848,10 +852,10 @@ public class RankProfile implements Cloneable { String fileName = extractFileName(expression); File file = new File(fileName); if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + - "' in subdirectory, which is not supported."); + throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); - return search.getRankingExpression(fileName); + return schema.getRankingExpression(fileName); } /** Shallow clones this */ @@ -888,7 +892,7 @@ public class RankProfile implements Cloneable { return compiled; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e); + throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); } } @@ -905,7 +909,7 @@ public class RankProfile implements Cloneable { secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions - // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. + // TODO: This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); allFunctionsCached = null; } @@ -960,6 +964,7 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (function == null) return null; + RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, featureTypes, @@ -993,7 +998,8 @@ public class RankProfile implements Cloneable { return featureTypes; } - public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes) { + public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); // Add small and large constants, respectively @@ -1076,6 +1082,11 @@ public class RankProfile implements Cloneable { }); } + @Override + public String toString() { + return "rank profile '" + name() + "'"; + } + /** * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. @@ -1118,8 +1129,9 @@ public class RankProfile implements Cloneable { return name; } + @Override public String toString() { - return "type: " + name; + return "type " + name; } } @@ -1232,7 +1244,7 @@ public class RankProfile implements Cloneable { @Override public String toString() { - return "function " + function; + return function.toString(); } } @@ -1321,4 +1333,22 @@ public class RankProfile implements Cloneable { } + private static class CachedFunctions { + + private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; + + private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; + + CachedFunctions(Map<String, RankingExpressionFunction> functions) { + allRankingExpressionFunctions = functions; + ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); + for (var entry : functions.entrySet()) { + ExpressionFunction function = entry.getValue().function(); + mapBuilder.put(function.getName(), function); + } + allExpressionFunctions = mapBuilder.build(); + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 08ae3d838ec..75c3aa313c0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -6,7 +6,6 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -44,22 +43,22 @@ public class RankProfileRegistry { /** Adds a rank profile to this registry */ public void add(RankProfile rankProfile) { - String searchName = extractName(rankProfile.getSearch()); - if ( ! rankProfiles.containsKey(searchName)) { - rankProfiles.put(searchName, new LinkedHashMap<>()); + String schemaName = extractName(rankProfile.schema()); + if ( ! rankProfiles.containsKey(schemaName)) { + rankProfiles.put(schemaName, new LinkedHashMap<>()); } checkForDuplicate(rankProfile); - rankProfiles.get(searchName).put(rankProfile.getName(), rankProfile); + rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); } private void checkForDuplicate(RankProfile rankProfile) { - String rankProfileName = rankProfile.getName(); - RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.getSearch())).get(rankProfileName); + String rankProfileName = rankProfile.name(); + RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); if (existingRankProfileWithSameName == null) return; if ( ! overridableRankProfileNames.contains(rankProfileName)) { throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + - rankProfile.getSearch()); + rankProfile.schema()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index aa43c00f461..ba34045e7de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -76,12 +76,12 @@ public class SDDocumentTypeOrderer { SDDocumentType inherited; if (type.isStruct()) { inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); - if (inherited == null) throw new IllegalStateException("Struct '" + name + "' not found in " + owningDocument); + if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); process(inherited, owningDocument); } else { inherited = createdSDTypes.get(name); - if (inherited == null) throw new IllegalStateException("Document type '" + name + "' not found"); + if (inherited == null) throw new IllegalArgumentException("Document type '" + name + "' not found"); process(inherited, inherited); } type.inherit(inherited); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index ddad67324ba..12b0394b090 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -55,6 +55,9 @@ public class Schema implements ImmutableSchema { /** The unique name of this schema */ private String name; + /** The application package this is constructed from */ + private final ApplicationPackage applicationPackage; + /** The name of the schema this should inherit all the content of, if any */ private final Optional<String> inherited; @@ -93,21 +96,22 @@ public class Schema implements ImmutableSchema { /** The resulting processed field */ private Optional<ImportedFields> importedFields = Optional.empty(); - private final Application owner; private final DeployLogger deployLogger; private final ModelContext.Properties properties; + private Application owner; + /** Testing only */ - public Schema(String name) { - this(name, Optional.empty(), null, null, new BaseDeployLogger(), new TestProperties()); + public Schema(String name, ApplicationPackage applicationPackage) { + this(name, applicationPackage, Optional.empty(), null, new BaseDeployLogger(), new TestProperties()); } public Schema(String name, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(name, Optional.empty(), application, fileRegistry, deployLogger, properties); + this(name, applicationPackage, Optional.empty(), fileRegistry, deployLogger, properties); } /** @@ -115,30 +119,30 @@ public class Schema implements ImmutableSchema { * * @param name of the schema * @param inherited the schema this inherits, if any - * @param application the application containing this */ public Schema(String name, + ApplicationPackage applicationPackage, Optional<String> inherited, - Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(inherited, application, fileRegistry, deployLogger, properties, false); - this.name = name; + this(inherited, applicationPackage, fileRegistry, deployLogger, properties, false); + this.name = Objects.requireNonNull(name, "A schema must have a name"); } - protected Schema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(Optional.empty(), application, fileRegistry, deployLogger, properties, true); + protected Schema(ApplicationPackage applicationPackage, FileRegistry fileRegistry, + DeployLogger deployLogger, ModelContext.Properties properties) { + this(Optional.empty(), applicationPackage, fileRegistry, deployLogger, properties, true); } private Schema(Optional<String> inherited, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties, boolean documentsOnly) { this.inherited = inherited; - this.owner = application; + this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; @@ -147,14 +151,21 @@ public class Schema implements ImmutableSchema { onnxModels = new OnnxModels(fileRegistry, Optional.of(this)); } - protected void setName(String name) { - this.name = name; + /** + * Assigns the owner of this + * + * @throws IllegalStateException if an owner is already assigned + */ + public void setOwner(Application owner) { + if (this.owner != null) + throw new IllegalStateException("Cannot reassign the owner of " + this); + this.owner = owner; } + protected void setName(String name) { this.name = name; } + @Override - public String getName() { - return name; - } + public String getName() {return name; } /** Returns true if this only defines a document type, not a full schema */ public boolean isDocumentsOnly() { @@ -174,11 +185,12 @@ public class Schema implements ImmutableSchema { */ public boolean isRawAsBase64() { if (rawAsBase64 != null) return rawAsBase64; + // TODO Vespa 8: flip default: if (inherited.isEmpty()) return false; return requireInherited().isRawAsBase64(); } - public void enableRawAsBase64() { rawAsBase64 = true; } + public void enableRawAsBase64(boolean value) { rawAsBase64 = value; } /** * Sets the stemming default of fields. Default is ALL @@ -311,16 +323,13 @@ public class Schema implements ImmutableSchema { */ @Override public Reader getRankingExpression(String fileName) { - return owner.applicationPackage().getRankingExpression(fileName); + return applicationPackage.getRankingExpression(fileName); } public Application application() { return owner; } @Override - public ApplicationPackage applicationPackage() { - if (owner == null) return null; - return owner.applicationPackage(); - } + public ApplicationPackage applicationPackage() { return applicationPackage; } @Override public DeployLogger getDeployLogger() { return deployLogger; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 04ef85856cd..dad8385e0e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -179,6 +179,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return "attributes"; } + @SuppressWarnings("removal") // TODO Vespa 8: remove private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) { Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>(); for (AttributesConfig.Attribute.Builder builder : ls) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 30cb236984d..3a8268029d0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -76,7 +76,7 @@ public abstract class Derived implements Exportable { protected abstract String getDerivedName(); /** Returns the value of getName if true, the given number as a string otherwise */ - protected String getIndex(int number,boolean labels) { + protected String getIndex(int number, boolean labels) { return labels ? getName() : String.valueOf(number); } @@ -87,12 +87,12 @@ public abstract class Derived implements Exportable { * @param toDirectory the directory to export to, or null * */ - public final void export(String toDirectory) - throws IOException { - Writer writer=null; + public final void export(String toDirectory) throws IOException { + Writer writer = null; try { - String fileName=getDerivedName() + ".cfg"; - if (toDirectory!=null) writer=IOUtils.createWriter(toDirectory + "/" + fileName,false); + String fileName = getDerivedName() + ".cfg"; + if (toDirectory != null) + writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); try { exportBuilderConfig(writer); } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { @@ -100,7 +100,7 @@ public abstract class Derived implements Exportable { } } finally { - if (writer!=null) IOUtils.closeWriter(writer); + if (writer != null) IOUtils.closeWriter(writer); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java index 14e303522e0..bcdc9e656e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -11,22 +11,21 @@ import java.util.Collections; import java.util.List; /** - * Auxiliary facade for deriving configs from search definitions + * Facade for deriving configs from schemas * * @author bratseth */ public class Deriver { - public static SchemaBuilder getSearchBuilder(List<String> sds) { - SchemaBuilder builder = new SchemaBuilder(); + public static ApplicationBuilder getSchemaBuilder(List<String> schemas) { + ApplicationBuilder builder = new ApplicationBuilder(); try { - for (String s : sds) { - builder.importFile(s); - } + for (String schema : schemas) + builder.addSchemaFile(schema); } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } - builder.build(); + builder.build(true); return builder; } @@ -34,22 +33,22 @@ public class Deriver { return getDocumentManagerConfig(Collections.singletonList(sd)); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) { - return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas) { + return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds, boolean useV8DocManagerCfg) { + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas, boolean useV8DocManagerCfg) { return new DocumentManager() .useV8DocManagerCfg(useV8DocManagerCfg) - .produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + .produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) { - return getDocumentTypesConfig(Collections.singletonList(sd)); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { + return getDocumentTypesConfig(Collections.singletonList(schema)); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) { - return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder()); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> schemas) { + return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java index c6f59fbe596..2a2950f71e9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -4,11 +4,9 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.collections.Pair; import java.util.ArrayList; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Logger; /** * The rank settings of a field used for native rank features. @@ -17,9 +15,7 @@ import java.util.logging.Logger; */ public class FieldRankSettings { - private static final Logger logger = Logger.getLogger(FieldRankSettings.class.getName()); - - private String fieldName; + private final String fieldName; private final Map<String, NativeTable> tables = new LinkedHashMap<>(); @@ -30,7 +26,7 @@ public class FieldRankSettings { public void addTable(NativeTable table) { NativeTable existing = tables.get(table.getType().getName()); if (existing != null) { - logger.info("Using already specified rank table " + existing + " for field " + fieldName + ", not " + table); + // TODO: Throw? return; } tables.put(table.getType().getName(), table); @@ -60,8 +56,7 @@ public class FieldRankSettings { public List<Pair<String, String>> deriveRankProperties() { List<Pair<String, String>> properties = new ArrayList<>(); - for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) { - NativeTable table = i.next(); + for (NativeTable table : tables.values()) { if (isFieldMatchTable(table)) properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); if (isAttributeMatchTable(table)) @@ -72,6 +67,7 @@ public class FieldRankSettings { return properties; } + @Override public String toString() { return "rank settings of field " + fieldName; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 3081637c975..bcebdf3a916 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -73,9 +73,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { - return (rank.getInheritedName() == null) || - rankProfiles.containsKey(rank.getInheritedName()) || - (rank.getSearch() != null && registry.resolve(rank.getSearch().getDocument(), rank.getInheritedName()) != null); + return rank.inheritedNames().isEmpty() || + rankProfiles.keySet().containsAll(rank.inheritedNames()) || + (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -92,7 +92,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } Map<String, RankProfile> remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.getName(), rank)); + rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); remaining.remove("default"); while (!remaining.isEmpty()) { List<RankProfile> ready = new ArrayList<>(); @@ -100,7 +100,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); }); processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - ready.forEach(rank -> remaining.remove(rank.getName())); + ready.forEach(rank -> remaining.remove(rank.name())); } } private void processRankProfiles(List<RankProfile> ready, @@ -116,8 +116,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ onnxModels.add(rank.onnxModels()); } - futureRawRankProfiles.put(rank.getName(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); + futureRawRankProfiles.put(rank.name(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); } try { for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index f166e533ab3..f024eb6eb77 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -63,9 +63,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), - attributeFields, deployProperties).derive(largeExpressions)); + this.name = rankProfile.name(); + compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties, queryProfiles) + .derive(largeExpressions)); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -159,9 +159,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) - { - rankprofileName = compiled.getName(); + Deriver(RankProfile compiled, + AttributeFields attributeFields, + ModelContext.Properties deployProperties, + QueryProfileRegistry queryProfiles) { + rankprofileName = compiled.name(); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); firstPhaseRanking = compiled.getFirstPhaseRanking(); @@ -184,7 +186,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); Map<String, String> functionProperties = new LinkedHashMap<>(); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, + Map.of(), + compiled.typeContext(queryProfiles)); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); @@ -206,8 +210,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void derivePropertiesAndFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, - Map<String, String> functionProperties, - SerializationContext functionContext) { + Map<String, String> functionProperties, + SerializationContext functionContext) { if (functions.isEmpty()) return; replaceFunctionFeatures(summaryFeatures, functionContext); @@ -287,14 +291,15 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) { for (Iterator<RankProfile.RankSetting> i = rankProfile.rankSettingIterator(); i.hasNext(); ) { RankProfile.RankSetting setting = i.next(); - if (!setting.getType().equals(RankProfile.RankSetting.Type.RANKTYPE)) continue; + if (setting.getType() != RankProfile.RankSetting.Type.RANKTYPE) continue; deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields, - hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); + hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); } } - private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) { + private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, + boolean isDefaultSetting) { if (isDefaultSetting) return; NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); @@ -422,7 +427,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } - if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); + if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); return properties; } @@ -461,8 +466,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { - if (rankProfile.getSearch() == null) return; - if (rankProfile.getSearch().onnxModels().asMap().isEmpty()) return; + if (rankProfile.schema() == null) return; + if (rankProfile.schema().onnxModels().asMap().isEmpty()) return; replaceOnnxFunctionInputs(rankProfile); replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); @@ -471,7 +476,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void replaceOnnxFunctionInputs(RankProfile rankProfile) { Set<String> functionNames = rankProfile.getFunctions().keySet(); if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.getSearch().onnxModels().asMap().values()) { + for (OnnxModel onnxModel: rankProfile.schema().onnxModels().asMap().values()) { for (Map.Entry<String, String> mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java index 2fd90d6c87e..7310a3fbd88 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java @@ -14,6 +14,7 @@ import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.searchdefinition.FieldSets; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -53,15 +54,19 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { protected void derive(StreamingDocumentType document, SDField field) { if (field.usesStructOrMap()) { + if (GeoPos.isAnyPos(field)) { + StreamingField streamingField = new StreamingField(field); + addField(streamingField.getName(), streamingField); + addFieldToIndices(document, field.getName(), streamingField); + } for (SDField structField : field.getStructFields()) { derive(document, structField); // Recursion } } else { - if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) return; - StreamingField streamingField=new StreamingField(field); + StreamingField streamingField = new StreamingField(field); addField(streamingField.getName(),streamingField); deriveIndices(document, field, streamingField); } @@ -132,14 +137,15 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { public static Type STRING = new Type("string","AUTOUTF8"); public static Type BOOL = new Type("bool","BOOL"); public static Type UNSEARCHABLESTRING = new Type("string","NONE"); + public static Type GEO_POSITION = new Type("position", "GEOPOS"); private String name; private String searchMethod; - private Type(String name,String searchMethod) { - this.name=name; - this.searchMethod=searchMethod; + private Type(String name, String searchMethod) { + this.name = name; + this.searchMethod = searchMethod; } @Override @@ -166,10 +172,10 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } public StreamingField(SDField field) { - this(field.getName(),field.getDataType(),field.getMatching(), field.doesAttributing()); + this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing()); } - private StreamingField(String name,DataType sourceType, Matching matching, boolean isAttribute) { + private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute) { this.name = name; this.type = convertType(sourceType); this.matching = matching; @@ -191,6 +197,8 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { return Type.BOOL; } else if (fieldType.equals(DataType.BYTE)) { return Type.INT8; + } else if (GeoPos.isAnyPos(fieldType)) { + return Type.GEO_POSITION; } else if (fieldType instanceof NumericDataType) { return Type.INT32; } else if (fval instanceof StringFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java index 03b9e795317..998343bc467 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java @@ -37,7 +37,7 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { if (doMapField(schema, summaryField)) { SDField sdField = schema.getConcreteField(summaryField.getName()); - if (sdField != null && GeoPos.isPos(sdField)) { + if (sdField != null && GeoPos.isAnyPos(sdField)) { summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); } else { summaryMap.put(summaryField, from); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 5ac6dd46102..fce128d80f3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -337,8 +337,8 @@ public final class Attribute implements Cloneable, Serializable { @SuppressWarnings("deprecation") private DataType createReferenceDataType() { - if (!referenceDocumentType.isPresent()) { - throw new IllegalStateException("Referenced document type is not set!"); + if (referenceDocumentType.isEmpty()) { + throw new IllegalStateException("Referenced document type is not set"); } StructuredDataType type = referenceDocumentType.get(); if (type instanceof DocumentType) { @@ -350,9 +350,9 @@ public final class Attribute implements Cloneable, Serializable { public DataType getDataType() { DataType dataType = toDataType(type); - if (collectionType.equals(Attribute.CollectionType.ARRAY)) { + if (collectionType == Attribute.CollectionType.ARRAY) { return DataType.getArray(dataType); - } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) { + } else if (collectionType == Attribute.CollectionType.WEIGHTEDSET) { return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); } else { return dataType; @@ -371,7 +371,7 @@ public final class Attribute implements Cloneable, Serializable { public boolean equals(Object object) { if (! (object instanceof Attribute)) return false; - Attribute other=(Attribute)object; + Attribute other = (Attribute)object; if (!this.name.equals(other.name)) return false; return isCompatible(other); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java index 691426ee413..d506b22297d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java @@ -102,8 +102,8 @@ public class Matching implements Cloneable, Serializable { } if (m.isTypeUserSet()) { this.setType(m.getType()); - if (m.getType()==Type.GRAM) - gramSize=m.gramSize; + if (m.getType() == Type.GRAM) + gramSize = m.gramSize; } if (m.getExactMatchTerminator() != null) { this.setExactMatchTerminator(m.getExactMatchTerminator()); @@ -125,8 +125,10 @@ public class Matching implements Cloneable, Serializable { this.exactMatchTerminator = exactMatchTerminator; } + @Override public String toString() { - return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]"; + return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + + "], [exact-terminator "+exactMatchTerminator+"]"; } public Matching clone() { @@ -145,10 +147,10 @@ public class Matching implements Cloneable, Serializable { Matching other=(Matching)o; if ( ! other.type.equals(this.type)) return false; if ( ! other.algorithm.equals(this.algorithm)) return false; - if ( this.exactMatchTerminator==null && other.exactMatchTerminator!=null) return false; - if ( this.exactMatchTerminator!=null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) + if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; + if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) return false; - if ( gramSize!=other.gramSize) return false; + if ( gramSize != other.gramSize) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java index 7bfff2d56ee..235677cd05e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java @@ -69,7 +69,7 @@ public class Ranking implements Cloneable, Serializable { return (Ranking)super.clone(); } catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error",e); + throw new RuntimeException("Programming error", e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 566342403d4..edb317f4a99 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -435,7 +435,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, config.setInputStream(new IndexingInput(script)); setIndexingScript(ScriptExpression.newInstance(config)); } catch (ParseException e) { - throw new RuntimeException("Failed to parser script '" + script + "'.", e); + throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java index 96f2f2f1d24..b4d76445507 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java @@ -29,11 +29,7 @@ public class TemporaryImportedFields { } public boolean hasField(String fieldName) { - boolean has = fields.get(fieldName) != null; - if (has) return true; - if (owner.inherited().isEmpty()) return false; - if (owner.inherited().get().temporaryImportedFields().isEmpty()) return false; - return owner.inherited().get().temporaryImportedFields().get().hasField(fieldName); + return fields.get(fieldName) != null; } public Map<String, TemporaryImportedField> fields() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index 166255799bd..f6ed5abaa7f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -63,8 +63,8 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile TensorValue tensorValue = (TensorValue)value; String tensorType = tensorValue.asTensor().type().toString(); - context.rankProperties().put(constantReference.toString() + ".value", tensorValue.toString()); - context.rankProperties().put(constantReference.toString() + ".type", tensorType); + context.rankProperties().put(constantReference + ".value", tensorValue.toString()); + context.rankProperties().put(constantReference + ".type", tensorType); return new ReferenceNode(constantReference); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java index ae8f918dd4b..6944a1f9dd1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java @@ -39,7 +39,6 @@ public class OnnxFeatureConverter extends ExpressionTransformer<RankProfileTrans private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { if ( ! feature.getName().equals("onnx_vespa")) return feature; - try { FeatureArguments arguments = asFeatureArguments(feature.getArguments()); ConvertedModel convertedModel = diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java index 6d9f4cdec92..35ee9ddb9ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java @@ -49,12 +49,12 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { if (context.rankProfile() == null) return feature; - if (context.rankProfile().getSearch() == null) return feature; + if (context.rankProfile().schema() == null) return feature; return transformFeature(feature, context.rankProfile()); } public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile rankProfile) { - ImmutableSchema search = rankProfile.getSearch(); + ImmutableSchema search = rankProfile.schema(); final String featureName = feature.getName(); if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java index cde172d00b9..0b275c6a722 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java @@ -6,11 +6,11 @@ import com.yahoo.javacc.FastCharStream; /** * @author Simon Thoresen Hult */ -@SuppressWarnings("deprecation") public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, - com.yahoo.vespa.indexinglanguage.parser.CharStream -{ + com.yahoo.vespa.indexinglanguage.parser.CharStream { + public SimpleCharStream(String input) { super(input); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 57d5a9c8e75..51defffa00b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,11 +26,6 @@ public class AddExtraFieldsToDocument extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } - //TODO This is a tempoarry hack to avoid producing illegal code for fields not wanted anyway. - private boolean dirtyLegalFieldNameCheck(String fieldName) { - return ! fieldName.contains(".") && !"rankfeatures".equals(fieldName) && !"summaryfeatures".equals(fieldName); - } - @Override public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = schema.getDocument(); @@ -38,10 +33,21 @@ public class AddExtraFieldsToDocument extends Processor { for (SDField field : schema.extraFieldList()) { addSdField(schema, document, field, validate); } - //TODO Vespa 8 or sooner we should avoid the dirty addition of fields from dirty 'default' summary to document at all - for (SummaryField field : schema.getSummary("default").getSummaryFields().values()) { - if (dirtyLegalFieldNameCheck(field.getName())) { - addSummaryField(schema, document, field, validate); + for (var docsum : schema.getSummaries().values()) { + for (var summaryField : docsum.getSummaryFields().values()) { + switch (summaryField.getTransform()) { + case NONE: + case BOLDED: + case DYNAMICBOLDED: + case DYNAMICTEASER: + case TEXTEXTRACTOR: + addSummaryField(schema, document, summaryField, validate); + break; + default: + // skip: generated from attribute or similar, + // so does not need to be included as an extra + // field in the document type + } } } } @@ -52,14 +58,7 @@ public class AddExtraFieldsToDocument extends Processor { return; } for (Attribute atr : field.getAttributes().values()) { - // TODO Vespa 8 or before: Check if this should be removed or changed to _zcurve. - if (atr.getName().equals(field.getName() + "_position")) { - DataType type = PositionDataType.INSTANCE; - if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) { - type = DataType.getArray(type); - } - addField(schema, document, new SDField(document, atr.getName(), type), validate); - } else if (!atr.getName().equals(field.getName())) { + if (!atr.getName().equals(field.getName())) { addField(schema, document, new SDField(document, atr.getName(), atr.getDataType()), validate); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java index 766b6ed3fec..254b3743f52 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; @@ -26,6 +27,14 @@ public class AdjustPositionSummaryFields extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + @Override public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : schema.getSummaries().values()) { @@ -80,6 +89,7 @@ public class AdjustPositionSummaryFields extends Processor { private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) { SummaryField oldField = schema.getSummaryField(fieldName); if (oldField == null) { + if (useV8GeoPositions) return; SummaryField newField = new SummaryField(fieldName, dataType, transform); newField.addSource(source); summary.add(newField); @@ -94,6 +104,7 @@ public class AdjustPositionSummaryFields extends Processor { if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) { fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'"); } + if (useV8GeoPositions) return; summary.add(oldField); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index f5c1d8d8197..0bb1b7da769 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; @@ -38,6 +39,14 @@ public class CreatePositionZCurve extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { @@ -63,14 +72,16 @@ public class CreatePositionZCurve extends Processor { // configure summary Collection<String> summaryTo = removeSummaryTo(field); - ensureCompatibleSummary(field, zName, - PositionDataType.getPositionSummaryFieldName(fieldName), - DataType.getArray(DataType.STRING), // will become "xmlstring" - SummaryTransform.POSITIONS, summaryTo, validate); - ensureCompatibleSummary(field, zName, - PositionDataType.getDistanceSummaryFieldName(fieldName), - DataType.INT, - SummaryTransform.DISTANCE, summaryTo, validate); + if (! useV8GeoPositions) { + ensureCompatibleSummary(field, zName, + PositionDataType.getPositionSummaryFieldName(fieldName), + DataType.getArray(DataType.STRING), // will become "xmlstring" + SummaryTransform.POSITIONS, summaryTo, validate); + ensureCompatibleSummary(field, zName, + PositionDataType.getDistanceSummaryFieldName(fieldName), + DataType.INT, + SummaryTransform.DISTANCE, summaryTo, validate); + } // clear indexing script field.setIndexingScript(null); SDField posX = field.getStructField(PositionDataType.FIELD_X); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 5643bb660f1..3759fc453df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -30,7 +30,7 @@ public class DiversitySettingsValidator extends Processor { } private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { String attributeName = settings.getAttribute(); - new AttributeValidator(schema.getName(), rankProfile.getName(), + new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index d89f83c333f..3f97bf83565 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -63,7 +63,7 @@ public class FilterFieldNames extends Processor { filterFields.add(fieldName); } } else { - deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); + deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.name() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 9eb8b921e81..0db6f4f05ba 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -48,7 +48,7 @@ public class ImplicitSummaries extends Processor { sdField.addSummaryFieldSources(summaryField); } - private void collectSummaries(SDField field , Schema schema, boolean validate) { + private void collectSummaries(SDField field, Schema schema, boolean validate) { SummaryField addedSummaryField = null; // Implicit @@ -91,9 +91,16 @@ public class ImplicitSummaries extends Processor { if (field.doesSummarying()) { for (Attribute attribute : field.getAttributes().values()) { if ( ! attribute.isPosition()) continue; - DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); - attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); - attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)); + if (distField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(distField); + } + var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)); + if (posField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(posField); + } } } @@ -104,7 +111,6 @@ public class ImplicitSummaries extends Processor { if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } - if (isValid(summaryField, schema, validate)) { addToDestinations(summaryField, schema); } @@ -203,8 +209,9 @@ public class ImplicitSummaries extends Processor { addToDestination("default", summaryField, schema); } else { - for (String destinationName : summaryField.getDestinations()) + for (String destinationName : summaryField.getDestinations()) { addToDestination(destinationName, summaryField, schema); + } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index e11f7d370c5..df103dcb096 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -31,8 +31,8 @@ public class ImplicitSummaryFields extends Processor { private void addField(DocumentSummary docsum, SummaryField field, boolean validate) { if (validate && docsum.getSummaryField(field.getName()) != null) { - throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" + - field.getName() + "'."); + throw new IllegalArgumentException("Summary class '" + docsum.getName() + "' uses reserved field name '" + + field.getName() + "'."); } docsum.add(field); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index b697c584ece..4eae6b47833 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -35,7 +35,7 @@ public class MatchPhaseSettingsValidator extends Processor { private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), - rankProfile.getName(), + rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java index 2a4f4f18759..2ca4abae2c4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java @@ -9,6 +9,8 @@ import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.Optional; + /** * Validates the 'paged' attribute setting and throws if specified on unsupported types. * @@ -38,11 +40,23 @@ public class PagedAttributeValidator extends Processor { } private void validatePagedSetting(Field field, Attribute attribute) { - var tensorType = attribute.tensorType(); - if (tensorType.isEmpty() - || !isDenseTensorType(tensorType.get())) { - fail(schema, field, "The 'paged' attribute setting is only supported for dense tensor types"); + if (!isSupportedType(attribute)) { + fail(schema, field, "The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types"); + } + } + + private boolean isSupportedType(Attribute attribute) { + var type = attribute.getType(); + return (type != Attribute.Type.PREDICATE) && + (type != Attribute.Type.REFERENCE) && + (isSupportedTensorType(attribute.tensorType())); + } + + private boolean isSupportedTensorType(Optional<TensorType> tensorType) { + if (tensorType.isPresent()) { + return isDenseTensorType(tensorType.get()); } + return true; } private boolean isDenseTensorType(TensorType type) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index f5c3fa35e34..e6adb8b683b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -89,10 +89,10 @@ public class RankingExpressionTypeResolver extends Processor { if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty() && ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.getName() + - "' are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + - context.queryFeaturesNotDeclared()); + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.name() + + "' are not declared in query profile " + + "types and will be interpreted as scalars, not tensors: " + + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index 8194b9f9e06..f4f920d9ec8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -35,9 +35,9 @@ public class ReservedFunctionNames extends Processor { for (String functionName : rp.getFunctions().keySet()) { if (reservedNames.contains(functionName)) { deployLogger.logApplicationPackage(Level.WARNING, "Function '" + functionName + "' " + - "in rank profile '" + rp.getName() + "' " + - "has a reserved name. This might mean that the function shadows " + - "the built-in function with the same name." + "in rank profile '" + rp.name() + "' " + + "has a reserved name. This might mean that the function shadows " + + "the built-in function with the same name." ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index bea7c95d412..15599c567ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -9,23 +9,15 @@ import com.yahoo.documentmodel.DocumentTypeRepo; * It contains a search manager managing all specified search definitions. * It contains a storage manager managing all specified storage definitions. * - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class DocumentModel { - private DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private SearchManager searchMan = new SearchManager(); - /** - * - * @return Returns the DocumentManager - */ + private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); + private final SearchManager searchMan = new SearchManager(); + public DocumentTypeRepo getDocumentManager() { return documentMan; } - /** - * - * @return Returns the SearchManager - */ - public SearchManager getSearchManager() { return searchMan; } + public SearchManager getSearchManager() { return searchMan; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java index e146d3c8798..acfdd4f8671 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java @@ -8,10 +8,10 @@ import java.util.ArrayList; import java.util.List; /** - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class SearchField extends Field { + /// Indicate if field shall be stored in memory for attribute usage. private boolean attribute = false; /// Indicate if the field is Vespa indexed. @@ -45,7 +45,6 @@ public class SearchField extends Field { validate(); } - @SuppressWarnings({ "deprecation" }) private void validate() { if (attribute || !indexed) { return; @@ -55,17 +54,14 @@ public class SearchField extends Field { if (DataType.STRING.equals(primiType) || DataType.URI.equals(primiType)) { return; } - throw new IllegalStateException("Expected type " + DataType.STRING.getName() + " for indexed field '" + - getName() + "', got " + fieldType.getName() + "."); + throw new IllegalArgumentException("Expected type " + DataType.STRING.getName() + " for indexed field '" + + getName() + "', got " + fieldType.getName() + "."); } - public SearchField setIndexed() { indexed = true; validate(); return this; } + public SearchField setIndexed() { indexed = true; validate(); return this; } public SearchField setAttribute() { attribute = true; validate(); return this; } public boolean isAttribute() { return attribute; } - /** - * True if field is Vespa indexed - * @return true if indexed - */ + public boolean isIndexed() { return indexed; } public SearchField addFeature(Feature feature) { featureList.add(feature); validate(); return this; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 0e7ee02d7ba..75a2a808a89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -197,32 +197,30 @@ public class SummaryField extends Field implements Cloneable, TypedKey { * Returns a summary field which merges the settings in the given field * into this field * - * @param merge the field to merge with this, if null, the merged field is - * <code>this</code> field + * @param merge the field to merge with this, if null, the merged field is *this* field * @throws RuntimeException if the two fields can not be merged */ public SummaryField mergeWith(SummaryField merge) { - if (merge==null) return this; + if (merge == null) return this; if (this.isImplicit()) return merge; if (merge.isImplicit()) return this; if (!merge.getName().equals(getName())) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different names"); - if (!merge.getTransform().equals(getTransform())) + if (merge.getTransform() != getTransform()) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different transforms"); if (!merge.getDataType().equals(getDataType())) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different types"); - if (!merge.isImplicit()) - setImplicit(false); + setImplicit(false); - if (isHeadOf(this.sourceIterator(),merge.sourceIterator())) { + if (isHeadOf(this.sourceIterator(), merge.sourceIterator())) { // Ok } - else if (isHeadOf(merge.sourceIterator(),this.sourceIterator())) { - sources=new LinkedHashSet<>(merge.sources); + else if (isHeadOf(merge.sourceIterator(), this.sourceIterator())) { + sources = new LinkedHashSet<>(merge.sources); } else { throw new IllegalArgumentException(merge + " conflicts with " + this + @@ -277,13 +275,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { return "'summary " + getName() + " type " + toLowerCase(getDataType().getName()) + "' in '" + getDestinationString() + "'"; } + @Override public SummaryField clone() { try { - SummaryField clone=(SummaryField)super.clone(); - if (this.sources!=null) - clone.sources=new LinkedHashSet<>(this.sources); - if (this.destinations!=null) - clone.destinations=new LinkedHashSet<>(destinations); + SummaryField clone = (SummaryField)super.clone(); + if (this.sources != null) + clone.sources = new LinkedHashSet<>(this.sources); + if (this.destinations != null) + clone.destinations = new LinkedHashSet<>(destinations); return clone; } catch (CloneNotSupportedException e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 73b106ed393..6777e2fb741 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; @@ -16,6 +17,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -54,19 +57,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon // Please keep non-null, as passed to command line in service startup private String preload = null; - // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. - private long mmapNoCoreLimit = -1L; - - // If this is true it will dump core when OOM - private boolean coreOnOOM = false; - - // If greater than 0, controls the number of threads used by open mp - private int ompNumThreads = 0; - - private String noVespaMalloc = ""; - private String vespaMalloc = ""; - private String vespaMallocDebug = ""; - private String vespaMallocDebugStackTrace = ""; + private final Map<String, Object> environmentVariables = new TreeMap<>(); /** The ports metainfo object */ protected PortsMeta portsMeta = new PortsMeta(); @@ -97,6 +88,8 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon */ public AbstractService(AbstractConfigProducer<?> parent, String name) { super(parent, name); + environmentVariables.put("OMP_NUM_THREADS", 1); + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); } /** @@ -105,7 +98,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param name the name of this service. */ public AbstractService(String name) { - super(name); + this(null, name); } @Override @@ -139,25 +132,28 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param hostResource The physical host on which this service should run. * @param userPort The wanted port given by the user. */ - private void initService(DeployLogger deployLogger, HostResource hostResource, int userPort) { + private void initService(DeployState deployState, HostResource hostResource, int userPort) { if (initialized) { throw new IllegalStateException("Service '" + getConfigId() + "' already initialized."); } if (hostResource == null) { - throw new RuntimeException("No host found for service '" + getServiceName() + "'. " + - "The hostalias is probably missing from hosts.xml."); + throw new IllegalArgumentException("No host found for service '" + getServiceName() + "'. " + + "The hostalias is probably missing from hosts.xml."); } id = getIndex(hostResource); - ports = hostResource.allocateService(deployLogger, this, getInstanceWantedPort(userPort)); + ports = hostResource.allocateService(deployState.getDeployLogger(), this, getInstanceWantedPort(userPort)); initialized = true; + for(String envVar : deployState.getProperties().environmentVariables()) { + addEnvironmentVariable(envVar); + } } /** * Called by builder class which has not given the host or port in a constructor, hence * initService is not yet run for this. */ - public void initService(DeployLogger deployLogger) { - initService(deployLogger, this.hostResource, this.basePort); + public void initService(DeployState deployState) { + initService(deployState, this.hostResource, this.basePort); } /** @@ -381,60 +377,44 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public void setPreLoad(String preload) { this.preload = preload; } - public long getMMapNoCoreLimit() { return mmapNoCoreLimit; } - public void setMMapNoCoreLimit(long noCoreLimit) { this.mmapNoCoreLimit = noCoreLimit; } - public boolean getCoreOnOOM() { return coreOnOOM; } - public void setCoreOnOOM(boolean coreOnOOM) { this.coreOnOOM = coreOnOOM; } - public int getOmpNumThreads() { return ompNumThreads; } - public void setOmpNumThreads(int value) { ompNumThreads = value; } - - public String getNoVespaMalloc() { return noVespaMalloc; } - public String getVespaMalloc() { return vespaMalloc; } - public String getVespaMallocDebug() { return vespaMallocDebug; } - public String getVespaMallocDebugStackTrace() { return vespaMallocDebugStackTrace; } - public void setNoVespaMalloc(String s) { noVespaMalloc = s; } - public void setVespaMalloc(String s) { vespaMalloc = s; } - public void setVespaMallocDebug(String s) { vespaMallocDebug = s; } - public void setVespaMallocDebugStackTrace(String s) { vespaMallocDebugStackTrace = s; } - - public String getMMapNoCoreEnvVariable() { - return (getMMapNoCoreLimit() >= 0L) - ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " " - : ""; - } - - public String getCoreOnOOMEnvVariable() { - return getCoreOnOOM() ? "" : "VESPA_SILENCE_CORE_ON_OOM=true "; - } - public String getOmpNumThreadsEnvVariable() { - return (getOmpNumThreads() == 0) - ? "" - : "OMP_NUM_THREADS=" + getOmpNumThreads() + " "; - } - public String getNoVespaMallocEnvVariable() { - return "".equals(getNoVespaMalloc()) - ? "" - : "VESPA_USE_NO_VESPAMALLOC=\"" + getNoVespaMalloc() + "\" "; - } - public String getVespaMallocEnvVariable() { - return "".equals(getVespaMalloc()) - ? "" - : "VESPA_USE_VESPAMALLOC=\"" + getVespaMalloc() + "\" "; - } - public String getVespaMallocDebugEnvVariable() { - return "".equals(getVespaMallocDebug()) - ? "" - : "VESPA_USE_VESPAMALLOC_D=\"" + getVespaMallocDebug() + "\" "; - } - public String getVespaMallocDebugStackTraceEnvVariable() { - return "".equals(getVespaMallocDebugStackTrace()) - ? "" - : "VESPA_USE_VESPAMALLOC_DST=\"" + getVespaMallocDebugStackTrace() + "\" "; - } - - public String getEnvVariables() { - return getCoreOnOOMEnvVariable() + getOmpNumThreadsEnvVariable() + getMMapNoCoreEnvVariable() + getNoVespaMallocEnvVariable() + - getVespaMallocEnvVariable() + getVespaMallocDebugEnvVariable() + getVespaMallocDebugStackTraceEnvVariable(); + /** If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump.*/ + public void setMMapNoCoreLimit(long noCoreLimit) { + if (noCoreLimit >= 0) { + environmentVariables.put("VESPA_MMAP_NOCORE_LIMIT", noCoreLimit); + } else { + environmentVariables.remove("VESPA_MMAP_NOCORE_LIMIT"); + } + } + public void setCoreOnOOM(boolean coreOnOOM) { + if ( ! coreOnOOM) { + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); + } else { + environmentVariables.remove("VESPA_SILENCE_CORE_ON_OOM"); + } + } + + public void setNoVespaMalloc(String s) { environmentVariables.put("VESPA_USE_NO_VESPAMALLOC", s); } + public void setVespaMalloc(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC", s); } + public void setVespaMallocDebug(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_D", s); } + public void setVespaMallocDebugStackTrace(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_DST", s); } + + private static String toEnvValue(Object o) { + if (o instanceof Number || o instanceof Boolean) { + return o.toString(); + } + return '"' + o.toString() + '"'; + } + + public void addEnvironmentVariable(String nameAndValue) { + int pos = nameAndValue.indexOf('='); + environmentVariables.put(nameAndValue.substring(0, pos), nameAndValue.substring(pos+1)); + } + public void addEnvironmentVariable(String name, Object value) { + environmentVariables.put(name, value); + } + + public String getEnv() { + return environmentVariables.entrySet().stream().map(e -> e.getKey() + '=' + toEnvValue(e.getValue())).collect(Collectors.joining(" ")); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 07dc0a5f9e9..b31e6028c05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -193,9 +193,9 @@ public class HostPorts { String msg = (service.getClass().equals(otherService.getClass()) && service.requiresWantedPort()) ? "You must set port explicitly for all instances of this service type, except the first one. " : ""; - throw new RuntimeException(service.getServiceName() + " cannot reserve port " + port + - " on " + hostname + ": Already reserved for " + otherService.getServiceName() + - ". " + msg + "Next available port is: " + nextAvailablePort + " ports used: " + portDB); + throw new IllegalArgumentException(service.getServiceName() + " cannot reserve port " + port + + " on " + hostname + ": Already reserved for " + otherService.getServiceName() + + ". " + msg + "Next available port is: " + nextAvailablePort + " ports used: " + portDB); } private void noMoreAvailablePorts() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 8f804229dd0..05aa125b808 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -103,7 +104,7 @@ public class HostResource implements Comparable<HostResource> { public HostInfo getHostInfo() { return new HostInfo(getHostname(), services.values().stream() .map(Service::getServiceInfo) - .collect(Collectors.toSet())); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); } /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java index 70a238b4a0c..3420ab26e20 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java @@ -14,7 +14,7 @@ import java.util.List; public class PortsMeta implements Serializable { /** A list of all ports. The list elements are lists of strings. */ - private List<LinkedList<String>> ports; + private final List<LinkedList<String>> ports; /** Remember the rpc admin port offset. */ private Integer rpcAdminOffset = null; @@ -35,8 +35,9 @@ public class PortsMeta implements Serializable { /** * Set up the port to tag, for chained usage. - * @param offset The relative port to tag. - * @return this portsmeta, to allow .tag calls. + * + * @param offset the relative port to tag + * @return this portsmeta, to allow .tag calls */ public PortsMeta on(int offset) { this.currentOffset = offset; @@ -45,8 +46,9 @@ public class PortsMeta implements Serializable { /** * Tag a previously setup port (using 'on') with the specified tag. - * @param meta The tag to apply to the current port. - * @return this portsmeta, to allow further .tag calls. + * + * @param meta the tag to apply to the current port + * @return this portsmeta, to allow further .tag calls */ public PortsMeta tag(String meta) { if (currentOffset == null) { @@ -57,9 +59,10 @@ public class PortsMeta implements Serializable { /** * Register a given metainfo string to the port at offset. + * * @param offset 0-based index to identify the port - * @param meta a String to be added to the given ports meta set. - * @return this for convenient chaining. + * @param meta a String to be added to the given ports meta set + * @return this for convenient chaining */ private PortsMeta register(int offset, String meta) { // Allocate new LinkedLists on each element up-to-and-including offset @@ -73,8 +76,9 @@ public class PortsMeta implements Serializable { /** * Check if the port at a specific offset contains a particular meta attribute. - * @param offset The relative port offset - * @param meta The meta info we want to check for + * + * @param offset the relative port offset + * @param meta the meta info we want to check for * @return boolean true if the specific port has registered the meta */ public boolean contains(int offset, String meta) { @@ -83,7 +87,8 @@ public class PortsMeta implements Serializable { /** * Get the number of ports with registered meta. - * @return the number of ports that have been registered. + * + * @return the number of ports that have been registered */ public int getNumPorts() { return ports.size(); @@ -91,21 +96,23 @@ public class PortsMeta implements Serializable { /** * Get an iterator of the Strings registered at the specific point. - * @param offset The relative offset to inquire about tags. + * + * @param offset the relative offset to inquire about tags * @return List of tags. */ public List<String> getTagsAt(int offset) { try { return ports.get(offset); } catch (IndexOutOfBoundsException e) { - throw new RuntimeException("Trying to get ports meta with offset " + offset + - ", which is outside the range 0 to " + ports.size(), e); + throw new IllegalArgumentException("Trying to get ports meta with offset " + offset + + ", which is outside the range 0 to " + ports.size(), e); } } /** * Get the offset to the rpc port used for admin. - * @return Integer the offset, or null if none set. + * + * @return the offset, or null if none set */ public Integer getRpcAdminOffset() { if (rpcAdminOffset == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 13b0f6216b2..8390cc59b6f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -87,9 +87,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static com.yahoo.text.StringUtilities.quote; import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toUnmodifiableMap; -import static java.util.stream.Collectors.toUnmodifiableSet; /** * <p> @@ -229,7 +227,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static Set<String> documentTypesWithIndex(ContentCluster content) { Set<String> typesWithIndexMode = content.getSearch().getDocumentTypesWithIndexedCluster().stream() .map(type -> type.getFullName().getName()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); Set<String> typesWithIndexedFields = content.getSearch().getIndexed() == null ? Set.of() @@ -239,9 +237,10 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .allConcreteFields() .stream().anyMatch(SDField::doesIndexing)) .map(database -> database.getInputDocType()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); - return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains).collect(toUnmodifiableSet()); + return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private void propagateRestartOnDeploy() { @@ -324,7 +323,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri for (var futureConvertedModel : futureModels) { try { futureConvertedModel.get(); - } catch (ExecutionException |InterruptedException e) { + } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } } @@ -620,7 +619,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param configId the id to register with, not necessarily equal to descendant.getConfigId(). * @param descendant The configProducer descendant to add */ - public void addDescendant(String configId, AbstractConfigProducer descendant) { + public void addDescendant(String configId, AbstractConfigProducer<?> descendant) { if (id2producer.containsKey(configId)) { throw new RuntimeException ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + @@ -695,7 +694,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .map(HostResource::spec) .filter(spec -> spec.membership().isPresent()) .map(spec -> spec.membership().get().cluster().id()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index bd8fefdda2e..f2d0ab03e27 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -24,6 +24,7 @@ import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.VespaVersion; import com.yahoo.vespa.model.application.validation.Validation; +import com.yahoo.vespa.model.application.validation.Validator; import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; @@ -50,11 +51,13 @@ public class VespaModelFactory implements ModelFactory { private final Zone zone; private final Clock clock; private final Version version; + private final List<Validator> additionalValidators; /** Creates a factory for Vespa models for this version of the source */ @Inject public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, ComponentRegistry<MlModelImporter> modelImporters, + ComponentRegistry<Validator> additionalValidators, Zone zone) { this.version = new Version(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); @@ -66,6 +69,7 @@ public class VespaModelFactory implements ModelFactory { this.configModelRegistry = new MapConfigModelRegistry(modelBuilders); this.modelImporters = modelImporters.allComponents(); this.zone = zone; + this.additionalValidators = List.copyOf(additionalValidators.allComponents()); this.clock = Clock.systemUTC(); } @@ -91,6 +95,7 @@ public class VespaModelFactory implements ModelFactory { this.configModelRegistry = configModelRegistry; } this.modelImporters = Collections.emptyList(); + this.additionalValidators = List.of(); this.zone = zone; this.clock = clock; } @@ -158,7 +163,7 @@ public class VespaModelFactory implements ModelFactory { try { return new VespaModel(configModelRegistry, deployState); } catch (IOException | SAXException e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } @@ -197,7 +202,7 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { - return Validation.validate(model, validationParameters, deployState); + return new Validation(additionalValidators).validate(model, validationParameters, deployState); } catch (ValidationOverrides.ValidationException e) { if (deployState.isHosted() && zone.environment().isManuallyDeployed()) deployState.getDeployLogger().logApplicationPackage(Level.WARNING, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 2692d676d2b..e299689d1c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -168,7 +167,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable Slobrok slobrok = new Slobrok(this, clusterController.index(), deployState.featureFlags()); slobrok.setHostResource(clusterController.getHostResource()); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } return slobroks; } @@ -237,16 +236,16 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable // between config changes, even when the set of hosts is unchanged. var container = new MetricsProxyContainer(metricsProxyCluster, host, index, deployState); - addAndInitializeService(deployState.getDeployLogger(), host, container); + addAndInitializeService(deployState, host, container); metricsProxyCluster.addContainer(container); } } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + addConfigSentinel(deployState, host, deployState.getProperties().applicationId(), deployState.zone(), deployState.featureFlags()); - addLogd(deployState.getDeployLogger(), host); - addConfigProxy(deployState.getDeployLogger(), host); + addLogd(deployState, host); + addConfigProxy(deployState, host); addFileDistribution(host); if (logForwarderConfig != null) { boolean actuallyAdd = true; @@ -259,34 +258,34 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } if (actuallyAdd) { - addLogForwarder(deployState.getDeployLogger(), host); + addLogForwarder(deployState, host); } } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + private void addConfigSentinel(DeployState deployState, HostResource host, ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) { ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); - addAndInitializeService(deployLogger, host, configSentinel); + addAndInitializeService(deployState, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } - private void addLogForwarder(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new LogForwarder(host.getHost(), logForwarderConfig)); + private void addLogForwarder(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new LogForwarder(host.getHost(), logForwarderConfig)); } - private void addLogd(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new Logd(host.getHost())); + private void addLogd(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new Logd(host.getHost())); } - private void addConfigProxy(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new ConfigProxy(host.getHost())); + private void addConfigProxy(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new ConfigProxy(host.getHost())); } - public void addAndInitializeService(DeployLogger deployLogger, HostResource host, AbstractService service) { + public void addAndInitializeService(DeployState deployState, HostResource host, AbstractService service) { service.setHostResource(host); - service.initService(deployLogger); + service.initService(deployState); } private void addFileDistribution(HostResource host) { @@ -300,7 +299,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable List<Slobrok> slobs = new ArrayList<>(); if (logserver != null) { Slobrok slobrok = new Slobrok(this, 0, deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), logserver.getHostResource(), slobrok); + addAndInitializeService(deployState, logserver.getHostResource(), slobrok); slobs.add(slobrok); } @@ -309,7 +308,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable HostResource host = hosts.get(n); if ((logserver== null || host != logserver.getHostResource()) && ! host.getHost().runsConfigServer()) { Slobrok newSlobrok = new Slobrok(this, slobs.size(), deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), host, newSlobrok); + addAndInitializeService(deployState, host, newSlobrok); slobs.add(newSlobrok); } n++; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 43f66f2c727..75b13a89e83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -17,7 +17,7 @@ import java.util.Optional; public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> { public LogserverContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { - super(parent, name, name, deployState, true); + super(parent, name, name, deployState, true, deployState.featureFlags().defaultPoolNumThreads()); addDefaultHandlersWithVip(); addLogHandler(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java index 6e6f027b520..bf3ba084091 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java @@ -9,19 +9,18 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; -import com.yahoo.net.HostName; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.container.Container; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toUnmodifiableSet; - /** * Used if clustercontroller is run standalone (not as part of the config server ZooKeeper cluster) * to provide common configs to container components. @@ -38,26 +37,24 @@ public class ClusterControllerCluster extends AbstractConfigProducer<ClusterCont public ClusterControllerCluster(AbstractConfigProducer<?> parent, String subId, DeployState deployState) { super(parent, subId); - this.previousHosts = deployState.getPreviousModel().stream() - .map(Model::allocatedHosts) - .map(AllocatedHosts::getHosts) - .flatMap(Collection::stream) - .map(HostSpec::hostname) - .collect(toUnmodifiableSet()); + this.previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() + .map(Model::allocatedHosts) + .map(AllocatedHosts::getHosts) + .flatMap(Collection::stream) + .map(HostSpec::hostname) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); } @Override public void getConfig(ZookeeperServerConfig.Builder builder) { builder.clientPort(ZK_CLIENT_PORT); builder.juteMaxBuffer(1024 * 1024); // 1 Mb should be more than enough for cluster controller - boolean oldQuorumExists = containerCluster.getContainers().stream() // More than half the previous hosts must be present in the new config for quorum to persist. - .filter(container -> previousHosts.contains(container.getHostName())) // Set intersection is symmetric. - .count() > previousHosts.size() / 2; for (ClusterControllerContainer container : containerCluster.getContainers()) { ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()); serverBuilder.id(container.index()); - serverBuilder.joining(oldQuorumExists && ! previousHosts.contains(container.getHostName())); + serverBuilder.joining( ! previousHosts.isEmpty() && ! previousHosts.contains(container.getHostName())); + serverBuilder.retired(container.isRetired()); builder.server(serverBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index eafc460bfb5..20c3e007e3b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -40,9 +40,11 @@ public class ClusterControllerContainer extends Container implements private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); private static final ComponentSpecification ZOOKEEPER_SERVER_BUNDLE = new ComponentSpecification("zookeeper-server"); private static final ComponentSpecification REINDEXING_CONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-reindexer"); - // The below adjustments to default netty settings reduces default chunkSize from 16M to 1M + // The below adjustments to default netty settings reduces default chunkSize from 16M to 128K private static final int DEFAULT_NETTY_PAGE_SIZE = 4096; // Reduced from nettys default of 8192 - private static final int DEFAULT_NETTY_MAX_ORDER = 8; // Reduced from nettys default of 11 + private static final int DEFAULT_NETTY_MAX_ORDER = 5; // Reduced from nettys default of 11 + private static final int DEFAULT_NETTY_NUM_DIRECT_ARENAS = 1; // Reduced from nettys default of 2*cores + private static final int DEFAULT_NETTY_NUM_HEAP_ARENAS = 1; // Reduced from nettys default of 2*cores private final Set<String> bundles = new TreeSet<>(); @@ -72,12 +74,16 @@ public class ClusterControllerContainer extends Container implements addFileBundle("zookeeper-server"); configureReindexing(); configureZooKeeperServer(runStandaloneZooKeeper); - prependJvmOptions(defaultNettyBufferSize(DEFAULT_NETTY_PAGE_SIZE, DEFAULT_NETTY_MAX_ORDER)); + prependJvmOptions(defaultNettyBufferSize(DEFAULT_NETTY_NUM_HEAP_ARENAS, DEFAULT_NETTY_NUM_DIRECT_ARENAS, + DEFAULT_NETTY_PAGE_SIZE, DEFAULT_NETTY_MAX_ORDER)); } - private static String defaultNettyBufferSize(int pageSize, int maxOrder) { + private static String defaultNettyBufferSize(int numHeapArenas, int numDirectArenas, int pageSize, int maxOrder) { return new StringBuffer("-Dio.netty.allocator.pageSize=").append(pageSize) - .append(" -Dio.netty.allocator.maxOrder=").append(maxOrder).toString(); + .append(" -Dio.netty.allocator.maxOrder=").append(maxOrder) + .append(" -Dio.netty.allocator.numHeapArenas=").append(numHeapArenas) + .append(" -Dio.netty.allocator.numDirectArenas=").append(numDirectArenas) + .toString(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 728e46f2ff7..a7f3a6224f2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -22,7 +22,7 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC public ClusterControllerContainerCluster( AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { - super(parent, subId, name, deployState, false); + super(parent, subId, name, deployState, false, deployState.featureFlags().defaultPoolNumThreads()); addDefaultHandlersWithVip(); this.reindexingContext = createReindexingContext(deployState); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index e894fba079d..ae2139be43c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -8,9 +8,11 @@ import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; /** @@ -71,7 +73,7 @@ class ConsumersConfigGenerator { static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { Consumer.Builder builder = new Consumer.Builder().name(consumer.id()); - consumer.metrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); + consumer.metrics().values().stream().sorted(Comparator.comparing(a -> a.name)).forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 88d224863c6..b6ddeb57cd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -159,12 +159,12 @@ public class MetricsProxyContainer extends Container implements cluster.getConfig(builder); if (clusterMembership.isPresent()) { - int maxHeapSize = clusterMembership.get().cluster().type() == ClusterSpec.Type.admin - ? 128 - : 256; + boolean adminCluster = clusterMembership.get().cluster().type() == ClusterSpec.Type.admin; + int maxHeapSize = adminCluster ? 96 : 256; builder.jvm .gcopts(jvmGCOptions) .heapsize(maxHeapSize); + if (adminCluster) builder.jvm.minHeapsize(maxHeapSize); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index dd6f77ed093..a29647b062a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -87,7 +87,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private final ApplicationId applicationId; public MetricsProxyContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { - super(parent, name, name, deployState, true); + super(parent, name, name, deployState, true, deployState.featureFlags().defaultPoolNumThreads()); this.parent = parent; applicationId = deployState.getProperties().applicationId(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 50f3a21a4fb..26e3d952080 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -48,7 +49,7 @@ public class AutoscalingMetrics { } private static Set<Metric> toMetrics(List<String> metrics) { - return metrics.stream().map(Metric::new).collect(Collectors.toSet()); + return metrics.stream().map(Metric::new).collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index d4b4a747717..5e10001baf6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -21,7 +21,7 @@ public class DefaultMetrics { public static final String defaultMetricSetId = "default"; - public static MetricSet defaultMetricSet = createMetricSet(); + public static final MetricSet defaultMetricSet = createMetricSet(); private static MetricSet createMetricSet() { return new MetricSet(defaultMetricSetId, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java index e5e5a30e34f..c2b0a358a54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java @@ -25,11 +25,11 @@ public class Metric { this.name = name; this.outputName = outputName; this.description = description; - this.dimensions = Collections.unmodifiableMap(dimensions); + this.dimensions = Map.copyOf(dimensions); } public Metric(String name, String outputName, String description) { - this(name, outputName, description, new HashMap<>()); + this(name, outputName, description, new LinkedHashMap<>()); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java index cd6035204e2..41204ce6e9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import javax.annotation.concurrent.Immutable; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; @@ -14,10 +13,10 @@ import static java.util.Collections.unmodifiableMap; /** * Models a metric set containing a set of metrics and child metric sets. + * This should be immutable. * * @author gjoranv */ -@Immutable public class MetricSet { private final String id; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index d2294402cd2..59fb541158e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -144,6 +144,11 @@ public class VespaMetricSet { metrics.add(new Metric("serverNumConnections.max")); metrics.add(new Metric("serverNumConnections.last")); + metrics.add(new Metric("serverBytesReceived.sum")); + metrics.add(new Metric("serverBytesReceived.count")); + metrics.add(new Metric("serverBytesSent.sum")); + metrics.add(new Metric("serverBytesSent.count")); + { List<String> suffixes = List.of("sum", "count", "last", "min", "max"); addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes); @@ -330,6 +335,7 @@ public class VespaMetricSet { metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); addMetric(metrics, "jdisc.render.latency", Set.of("min", "max", "count", "sum", "last", "average")); + addMetric(metrics, "query_item_count", Set.of("max", "sum", "count")); metrics.add(new Metric("totalhits_per_query.max")); metrics.add(new Metric("totalhits_per_query.sum")); @@ -474,11 +480,17 @@ public class VespaMetricSet { // resource usage metrics.add(new Metric("content.proton.resource_usage.disk.average")); - metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.total.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.total_utilization.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.transient.max")); metrics.add(new Metric("content.proton.resource_usage.memory.average")); - metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); - metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); - metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.total.max")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.total_utilization.max")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.transient.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); @@ -487,6 +499,23 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.max")); + // CPU util + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.count")); + // transaction log metrics.add(new Metric("content.proton.transactionlog.entries.average")); metrics.add(new Metric("content.proton.transactionlog.disk_usage.average")); @@ -555,9 +584,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); // TODO: Consider remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.max")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); @@ -581,8 +608,6 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); // TODO: Consider remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); @@ -603,8 +628,7 @@ public class VespaMetricSet { // TODO: For the purpose of this file and likely elsewhere, all but the last aggregate specifier, // TODO: such as 'average' and 'sum' in the metric names below are just confusing and can be mentally - // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 8. - // TODO Vespa 8 all metrics with .sum in the name should have that removed. + // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 9. metrics.add(new Metric("vds.datastored.alldisks.docs.average")); metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.max")); @@ -636,6 +660,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.count")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.max")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.sum")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count")); @@ -651,6 +678,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_rpc_direct_dispatches.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_persistence_thread_polls.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.timeouts_waiting_for_throttle_token.rate")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java new file mode 100644 index 00000000000..1642a8cb3cc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import aQute.bnd.header.Parameters; +import aQute.bnd.osgi.Domain; +import aQute.bnd.version.VersionRange; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.regex.Pattern; + +/** + * Base class for OSGi bundle validator. Uses BND library for some of the validation. + * + * @author bjorncs + */ +public abstract class AbstractBundleValidator extends Validator { + + protected abstract void validateManifest(DeployState state, JarFile jar, Manifest mf); + protected abstract void validatePomXml(DeployState state, JarFile jar, Document pom); + + @Override + public final void validate(VespaModel model, DeployState state) { + ApplicationPackage app = state.getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(state.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); + try { + state.getDeployLogger() + .log(Level.FINE, String.format("Validating bundle at '%s'", path)); + JarFile jarFile = new JarFile(app.getFileReference(path)); + validateJarFile(state, jarFile); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to validate JAR file '" + path.last() + "'", e); + } + } + } + + final void validateJarFile(DeployState state, JarFile jar) throws IOException { + Manifest manifest = jar.getManifest(); + if (manifest == null) { + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename(jar)); + } + validateManifest(state, jar, manifest); + getPomXmlContent(state.getDeployLogger(), jar) + .ifPresent(pom -> validatePomXml(state, jar, pom)); + } + + protected final String filename(JarFile jarFile) { return Paths.get(jarFile.getName()).getFileName().toString(); } + + protected final void forEachPomXmlElement(Document pom, String xpath, Consumer<Element> consumer) throws XPathExpressionException { + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/" + xpath) + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element element = (Element) dependencies.item(i); + consumer.accept(element); + } + } + + protected final void forEachImportPackage(Manifest mf, BiConsumer<String, VersionRange> consumer) { + Parameters importPackage = Domain.domain(mf).getImportPackage(); + importPackage.forEach((packageName, attrs) -> { + VersionRange versionRange = attrs.getVersion() != null + ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) + : null; + consumer.accept(packageName, versionRange); + }); + } + + protected final void log(DeployState state, Level level, String fmt, Object... args) { + state.getDeployLogger().logApplicationPackage(level, String.format(fmt, args)); + } + + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + private Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { + return jar.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + String text = new String(jar.getInputStream(f).readAllBytes()); + return DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(text))); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (SAXException e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", filename(jar))); + return null; + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jar.getName())); + return null; + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java deleted file mode 100644 index a07e07169d1..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator.needsAccessControlValidation; -import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; - -/** - * @author gjoranv - */ -public class AwsAccessControlValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - - if (! needsAccessControlValidation(model, deployState)) return; - if(! deployState.zone().getCloud().requireAccessControl()) return; - - List<String> offendingClusters = new ArrayList<>(); - for (var cluster : model.getContainerClusters().values()) { - var http = cluster.getHttp(); - if (http == null - || ! http.getAccessControl().isPresent() - || ! http.getAccessControl().get().writeEnabled - || ! http.getAccessControl().get().readEnabled) - - if (hasHandlerThatNeedsProtection(cluster)) - offendingClusters.add(cluster.getName()); - } - if (! offendingClusters.isEmpty()) - deployState.validationOverrides() - .invalid(ValidationId.accessControl, - "Access-control must be enabled for read/write operations to container clusters in AWS production zones: " + - mkString(offendingClusters, "[", ", ", "]"), deployState.now()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index b6b9190fedf..a1024425124 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -1,17 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import aQute.bnd.header.Parameters; -import aQute.bnd.osgi.Domain; import aQute.bnd.version.VersionRange; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.path.Path; -import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -25,44 +18,21 @@ import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** - * A validator for bundles. Uses BND library for some of the validation. + * A validator for bundles. * * @author hmusum * @author bjorncs */ -public class BundleValidator extends Validator { +public class BundleValidator extends AbstractBundleValidator { public BundleValidator() {} @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage app = deployState.getApplicationPackage(); - for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { - try { - Path path = Path.fromString(info.getPathRelativeToAppDir()); - DeployLogger deployLogger = deployState.getDeployLogger(); - deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); - JarFile jarFile = new JarFile(app.getFileReference(path)); - validateJarFile(deployLogger, jarFile); - } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to validate JAR file '" + info.getPathRelativeToAppDir() + "'", e); - } - } - } - - void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException { - Manifest manifest = jarFile.getManifest(); - String jarPath = jarFile.getName(); - if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarPath); - } - validateManifest(deployLogger, jarPath, manifest); - } - - void validateManifest(DeployLogger deployLogger, String jarPath, Manifest mf) { + protected void validateManifest(DeployState state, JarFile jar, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -74,69 +44,65 @@ public class BundleValidator extends Validator { for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + jarPath + "'"); + "' was not found in manifest in '" + filename(jar) + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarPath + - ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); + log(state, Level.WARNING, + "Deploying snapshot bundle " + filename(jar) + ".\nTo use this bundle, you must include the " + + "qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(deployLogger, jarPath, mf); + validateImportedPackages(state, jar, mf); } } - private static void validateImportedPackages(DeployLogger deployLogger, String jarPath, Manifest manifest) { - Domain osgiHeaders = Domain.domain(manifest); - Parameters importPackage = osgiHeaders.getImportPackage(); - Map<DeprecatedArtifact, List<String>> deprecatedPackagesInUse = new HashMap<>(); - - importPackage.forEach((packageName, attrs) -> { - VersionRange versionRange = attrs.getVersion() != null - ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) - : null; - - for (DeprecatedArtifact deprecatedArtifact : DeprecatedArtifact.values()) { - if (deprecatedArtifact.javaPackages.contains(packageName) - && (versionRange == null || deprecatedArtifact.versionDiscriminator.test(versionRange))) { - deprecatedPackagesInUse.computeIfAbsent(deprecatedArtifact, __ -> new ArrayList<>()) - .add(packageName); + @Override protected void validatePomXml(DeployState state, JarFile jar, Document pom) {} + + private void validateImportedPackages(DeployState state, JarFile jar, Manifest manifest) { + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); + forEachImportPackage(manifest, (packageName, versionRange) -> { + for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { + for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { + if (matcher.test(packageName) + && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) + .add(packageName); + } } } }); - deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> { - deployLogger.logApplicationPackage(Level.WARNING, - String.format("For JAR file '%s': \n" + - "Manifest imports the following Java packages from '%s': %s. \n" + - "%s", - jarPath, artifact.name, packagesInUse, artifact.description)); + log(state, Level.WARNING, "JAR file '%s' imports the packages %s from '%s'. \n%s", + filename(jar), packagesInUse, artifact.name, artifact.description); }); } - private enum DeprecatedArtifact { + private enum DeprecatedProvidedBundle { ORG_JSON("org.json:json", - "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", - Set.of("org.json")); + "This bundle is no longer provided on Vespa 8 - " + + "see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", + Set.of("org\\.json")); final String name; - final Collection<String> javaPackages; + final Collection<Predicate<String>> javaPackageMatchers; final Predicate<VersionRange> versionDiscriminator; final String description; - DeprecatedArtifact(String name, String description, Collection<String> javaPackages) { - this(name, description, __ -> true, javaPackages); + DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { + this(name, description, __ -> true, javaPackagePatterns); } - DeprecatedArtifact(String name, - String description, - Predicate<VersionRange> versionDiscriminator, - Collection<String> javaPackages) { + DeprecatedProvidedBundle(String name, + String description, + Predicate<VersionRange> versionDiscriminator, + Collection<String> javaPackagePatterns) { this.name = name; - this.javaPackages = javaPackages; + this.javaPackageMatchers = javaPackagePatterns.stream() + .map(s -> Pattern.compile(s).asMatchPredicate()) + .collect(Collectors.toList()); this.versionDiscriminator = versionDiscriminator; this.description = description; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index e2b08a621d1..b977e4c657e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -34,7 +34,7 @@ public class ComplexAttributeFieldsValidator extends Validator { } SearchCluster searchCluster = (SearchCluster) cluster; for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { - validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); + validateComplexFields(searchCluster.getClusterName(), spec.getSchema()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index ef0a8ddda8e..66da43856b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -90,7 +90,7 @@ public class ConstantTensorJsonValidator { validateTensorValue(); } } else { - throw new InvalidConstantTensor(parser, "Only 'address' or 'value' fields are permitted within a cell object"); + throw new InvalidConstantTensorException(parser, "Only 'address' or 'value' fields are permitted within a cell object"); } } @@ -110,11 +110,11 @@ public class ConstantTensorJsonValidator { String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensor(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensor(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); } cellDimensions.remove(dimensionName); @@ -122,7 +122,7 @@ public class ConstantTensorJsonValidator { } if (!cellDimensions.isEmpty()) { - throw new InvalidConstantTensor(parser, String.format("Tensor address missing dimension(s) %s", Joiner.on(", ").join(cellDimensions))); + throw new InvalidConstantTensorException(parser, String.format("Tensor address missing dimension(s) %s", Joiner.on(", ").join(cellDimensions))); } } @@ -134,7 +134,7 @@ public class ConstantTensorJsonValidator { private void validateLabel(TensorType.Dimension dimension) throws IOException { JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_STRING) - throw new InvalidConstantTensor(parser, String.format("Tensor label is not a string (%s)", token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Tensor label is not a string (%s)", token.toString())); if (dimension instanceof TensorType.IndexedBoundDimension) { validateBoundIndex((TensorType.IndexedBoundDimension) dimension); @@ -148,7 +148,7 @@ public class ConstantTensorJsonValidator { try { int value = Integer.parseInt(parser.getValueAsString()); if (value >= dimension.size().get()) - throw new InvalidConstantTensor(parser, String.format("Index %s not within limits of bound dimension '%s'", value, dimension.name())); + throw new InvalidConstantTensorException(parser, String.format("Index %s not within limits of bound dimension '%s'", value, dimension.name())); } catch (NumberFormatException e) { throwCoordinateIsNotInteger(parser.getValueAsString(), dimension.name()); } @@ -166,14 +166,14 @@ public class ConstantTensorJsonValidator { } private void throwCoordinateIsNotInteger(String value, String dimensionName) { - throw new InvalidConstantTensor(parser, String.format("Index '%s' for dimension '%s' is not an integer", value, dimensionName)); + throw new InvalidConstantTensorException(parser, String.format("Index '%s' for dimension '%s' is not an integer", value, dimensionName)); } private void validateTensorValue() throws IOException { JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_NUMBER_FLOAT && token != JsonToken.VALUE_NUMBER_INT) { - throw new InvalidConstantTensor(parser, String.format("Tensor value is not a number (%s)", token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Tensor value is not a number (%s)", token.toString())); } } @@ -187,7 +187,7 @@ public class ConstantTensorJsonValidator { private void assertTokenIs(JsonToken token, JsonToken wantedToken) { if (token != wantedToken) { - throw new InvalidConstantTensor(parser, String.format("Expected JSON token %s, but got %s", wantedToken.toString(), token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Expected JSON token %s, but got %s", wantedToken.toString(), token.toString())); } } @@ -195,17 +195,17 @@ public class ConstantTensorJsonValidator { String actualFieldName = parser.getCurrentName(); if (!actualFieldName.equals(wantedFieldName)) { - throw new InvalidConstantTensor(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); + throw new InvalidConstantTensorException(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); } } - static class InvalidConstantTensor extends RuntimeException { + static class InvalidConstantTensorException extends IllegalArgumentException { - InvalidConstantTensor(JsonParser parser, String message) { + InvalidConstantTensorException(JsonParser parser, String message) { super(message + " " + parser.getCurrentLocation().toString()); } - InvalidConstantTensor(JsonParser parser, Exception base) { + InvalidConstantTensorException(JsonParser parser, Exception base) { super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); } @@ -220,7 +220,7 @@ public class ConstantTensorJsonValidator { try { lambda.invoke(); } catch (IOException e) { - throw new InvalidConstantTensor(parser, e); + throw new InvalidConstantTensorException(parser, e); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 841dbb204fd..e4a64e8d476 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.model.VespaModel; import java.math.BigDecimal; +import java.util.LinkedHashSet; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -61,7 +62,7 @@ public class QuotaValidator extends Validator { .map(hostSpec -> hostSpec.membership().orElseThrow().cluster()) .filter(cluster -> cluster.type() == ClusterSpec.Type.admin) .map(ClusterSpec::id) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } /** Check that all clusters in the application do not exceed the quota max cluster size. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index e9b23461527..82c793af70a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -7,9 +7,9 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; -import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; import java.io.FileNotFoundException; @@ -36,8 +36,8 @@ public class RankingConstantsValidator extends Validator { } } - static class TensorValidationFailed extends RuntimeException { - TensorValidationFailed(String message) { + static class TensorValidationException extends IllegalArgumentException { + TensorValidationException(String message) { super(message); } } @@ -47,18 +47,18 @@ public class RankingConstantsValidator extends Validator { ApplicationPackage applicationPackage = deployState.getApplicationPackage(); ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - for (NamedSchema sd : deployState.getSchemas()) { - for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) { + for (Schema schema : deployState.getSchemas()) { + for (RankingConstant rc : schema.rankingConstants().asMap().values()) { try { validateRankingConstant(rc, applicationPackage); - } catch (InvalidConstantTensor | FileNotFoundException ex) { + } catch (InvalidConstantTensorException | FileNotFoundException ex) { exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName()); } } } if (exceptionMessageCollector.exceptionsOccurred) { - throw new TensorValidationFailed(exceptionMessageCollector.combinedMessage); + throw new TensorValidationException(exceptionMessageCollector.combinedMessage); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 14f5c345025..976ed0a0219 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -11,11 +11,11 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.NamedSchema; import java.util.List; @@ -35,26 +35,26 @@ public class SearchDataTypeValidator extends Validator { continue; } for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) { - SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument(); + SDDocumentType docType = spec.getSchema().getDocument(); if (docType == null) { continue; } - validateDocument(cluster, spec.getSearchDefinition(), docType); + validateDocument(cluster, spec.getSchema(), docType); } } } - private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) { + private void validateDocument(AbstractSearchCluster cluster, Schema schema, SDDocumentType doc) { for (SDDocumentType child : doc.getTypes()) { - validateDocument(cluster, def, child); + validateDocument(cluster, schema, child); } for (Field field : doc.fieldSet()) { DataType fieldType = field.getDataType(); - disallowIndexingOfMaps(cluster, def, field); + disallowIndexingOfMaps(cluster, schema, field); if ( ! isSupportedInSearchClusters(fieldType)) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " + - "clusters (field '" + field.getName() + "' in definition '" + - def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + "clusters (field '" + field.getName() + "' in schema '" + + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } } @@ -84,12 +84,12 @@ public class SearchDataTypeValidator extends Validator { } } - private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) { + private void disallowIndexingOfMaps(AbstractSearchCluster cluster, Schema schema, Field field) { DataType fieldType = field.getDataType(); if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + "clusters (field '" + field.getName() + "' in definition '" + - def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 08dc73a1bd0..4ba809a58ba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseVal import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; -import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; import com.yahoo.vespa.model.application.validation.first.RedundancyOnFirstDeploymentValidator; import java.time.Instant; @@ -50,6 +49,13 @@ import static java.util.stream.Collectors.toList; */ public class Validation { + private final List<Validator> additionalValidators; + + public Validation() { this(List.of()); } + + /** Create instance taking additional validators (e.g for cloud applications) */ + public Validation(List<Validator> additionalValidators) { this.additionalValidators = additionalValidators; } + /** * Validates the model supplied, and if there already exists a model for the application validates changes * between the previous and current model @@ -57,7 +63,7 @@ public class Validation { * @return a list of required changes needed to make this configuration live * @throws ValidationOverrides.ValidationException if the change fails validation */ - public static List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { + public List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { if (validationParameters.checkRouting()) { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); @@ -76,10 +82,11 @@ public class Validation { new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); new CloudWatchValidator().validate(model, deployState); - new AwsAccessControlValidator().validate(model, deployState); new QuotaValidator().validate(model, deployState); new UriBindingsValidator().validate(model, deployState); + additionalValidators.forEach(v -> v.validate(model, deployState)); + List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { validateFirstTimeDeployment(model, deployState); @@ -126,7 +133,6 @@ public class Validation { } private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); new RedundancyOnFirstDeploymentValidator().validate(model, deployState); } @@ -134,7 +140,7 @@ public class Validation { Set<ClusterSpec.Id> clustersToBeRestarted = actions.stream() .filter(action -> action.getType() == ConfigChangeAction.Type.RESTART) .map(action -> action.clusterId()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); for (var clusterToRestart : clustersToBeRestarted) { var containerCluster = model.getContainerClusters().get(clusterToRestart.value()); if (containerCluster != null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index 30abb9b445f..a3872d9354b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -13,13 +13,12 @@ import com.yahoo.vespa.model.search.SearchNode; import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toSet; - /** * Returns any change to the indexing mode of a cluster. * @@ -91,7 +90,7 @@ public class IndexingModeChangeValidator implements ChangeValidator { private static Set<String> toDocumentTypeNames(List<NewDocumentType> types) { return types.stream() .map(type -> type.getFullName().getName()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java deleted file mode 100644 index dd6e6ad590d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.first; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.ConfigModelContext.ApplicationType; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.Validator; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.ContainerCluster; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.config.provision.InstanceName.defaultName; -import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; - -/** - * Validates that hosted applications in prod zones have write protection enabled. - * - * @author gjoranv - */ -public class AccessControlOnFirstDeploymentValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - - if (! needsAccessControlValidation(model, deployState)) return; - - List<String> offendingClusters = new ArrayList<>(); - for (ContainerCluster<? extends Container> c : model.getContainerClusters().values()) { - if (! (c instanceof ApplicationContainerCluster)) continue; - ApplicationContainerCluster cluster = (ApplicationContainerCluster)c; - if (cluster.getHttp() == null - || ! cluster.getHttp().getAccessControl().isPresent() - || ! cluster.getHttp().getAccessControl().get().writeEnabled) - - if (hasHandlerThatNeedsProtection(cluster)) - offendingClusters.add(cluster.getName()); - } - if (! offendingClusters.isEmpty()) - deployState.validationOverrides().invalid(ValidationId.accessControl, - "Access-control must be enabled for write operations to container clusters in production zones: " + - mkString(offendingClusters, "[", ", ", "]"), deployState.now()); - } - - public static boolean needsAccessControlValidation(VespaModel model, DeployState deployState) { - if (! deployState.isHosted()) return false; - if (! deployState.zone().environment().isProduction()) return false; - if (deployState.zone().system().isPublic()) return false; - if (! deployState.getApplicationPackage().getApplicationId().instance().equals(defaultName())) return false; - if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return false; - - return true; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java index b163f9f0699..06747bdbd2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java @@ -11,8 +11,9 @@ import java.util.regex.Pattern; * @author Tony Vaagenes */ public class BinaryScaledAmountParser { - //The pattern must match the one given in the schema - private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + // The pattern must match the one given in the schema + private static final Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); public static BinaryScaledAmount parse(String valueString) { Matcher matcher = pattern.matcher(valueString); @@ -34,4 +35,5 @@ public class BinaryScaledAmountParser { return BinaryPrefix.fromSymbol(binaryPrefixString.toUpperCase().charAt(0)); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java index 1317f96e109..4a46b62d096 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java @@ -10,8 +10,9 @@ import static com.yahoo.text.Lowercase.toLowerCase; * @author Tony Vaagenes */ public class BinaryUnit { - //The pattern must match the one given in the schema - private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + // The pattern must match the one given in the schema + private static final Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); public static double valueOf(String valueString) { Matcher matcher = pattern.matcher(valueString); @@ -28,13 +29,13 @@ public class BinaryUnit { } private static double unitToValue(char unit) { - final char units[] = {'k', 'm', 'g'}; - for (int i=0; i<units.length; ++i) { + char[] units = {'k', 'm', 'g'}; + for (int i = 0; i < units.length; ++i) { if (units[i] == unit) { return Math.pow(2, 10*(i+1)); } } - throw new RuntimeException("No such unit: '" + unit + "'"); + throw new IllegalArgumentException("No such unit: '" + unit + "'"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 56f462805bd..4c74282c061 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -45,7 +45,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu this.configServerSpecs = configServerSpecs; } - List<Configserver> getConfigServersFromSpec(DeployLogger deployLogger, AbstractConfigProducer<?> parent) { + List<Configserver> getConfigServersFromSpec(DeployState deployState, AbstractConfigProducer<?> parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.hostSystem(); @@ -54,7 +54,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort()); configserver.setHostResource(host); configserver.setBasePort(configserver.getWantedPort()); - configserver.initService(deployLogger); + configserver.initService(deployState); configservers.add(configserver); } return configservers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 5aaba9550d2..8e7c543b67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -53,7 +53,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers; if (multitenant) - configservers = getConfigServersFromSpec(deployState.getDeployLogger(), admin); + configservers = getConfigServersFromSpec(deployState, admin); else configservers = getConfigServers(deployState, admin, adminE); if (configservers.isEmpty() && ! multitenant) @@ -129,7 +129,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { SimpleConfigProducer<?> configServers = new SimpleConfigProducer<>(parent, "configservers"); Configserver configServer = new Configserver(configServers, "configserver", Configserver.defaultRpcPort); configServer.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); - configServer.initService(deployState.getDeployLogger()); + configServer.initService(deployState); return List.of(configServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 41baf2db3aa..a91a7949ad8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; @@ -44,7 +43,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element w3cAdminElement) { ModelElement adminElement = new ModelElement(w3cAdminElement); - admin.addConfigservers(getConfigServersFromSpec(deployState.getDeployLogger(), admin)); + admin.addConfigservers(getConfigServersFromSpec(deployState, admin)); // Note: These two elements only exists in admin version 4.0 // This build handles admin version 3.0 by ignoring its content (as the content is not useful) @@ -79,13 +78,13 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); + Logserver logserver = createLogserver(deployState, admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - createLogserver(deployState.getDeployLogger(), admin, hosts); + createLogserver(deployState, admin, hosts); } else { context.getDeployLogger().logApplicationPackage(Level.INFO, "No container host available to use for running logserver"); } @@ -110,9 +109,9 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { LogserverContainer container = new LogserverContainer(logServerCluster, deployState); container.setHostResource(hostResource); - container.initService(deployState.getDeployLogger()); + container.initService(deployState); logServerCluster.addContainer(container); - admin.addAndInitializeService(deployState.getDeployLogger(), hostResource, container); + admin.addAndInitializeService(deployState, hostResource, container); admin.setLogserverContainerCluster(logServerCluster); context.getConfigModelRepoAdder().add(logserverClusterModel); } @@ -169,11 +168,11 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { return hosts.subList(0, Math.min(count, hosts.size())); } - private Logserver createLogserver(DeployLogger deployLogger, Admin admin, Collection<HostResource> hosts) { + private Logserver createLogserver(DeployState deployState, Admin admin, Collection<HostResource> hosts) { Logserver logserver = new Logserver(admin); logserver.setHostResource(hosts.iterator().next()); admin.setLogserver(logserver); - logserver.initService(deployLogger); + logserver.initService(deployState); return logserver; } @@ -185,7 +184,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Slobrok slobrok = new Slobrok(admin, index++, deployState.featureFlags()); slobrok.setHostResource(host); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } admin.addSlobroks(slobroks); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 0b32c77fe58..bbc18ab2254 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -15,7 +15,9 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; +import java.util.List; import java.util.Set; +import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; @@ -49,7 +51,12 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); - for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) + List<Element> clientBindingsElements = XML.getChildren(handlerElement, "clientBinding"); + if (! clientBindingsElements.isEmpty()) { + deployState.getDeployLogger().logApplicationPackage( + Level.WARNING, "The 'clientBindings' element is deprecated for removal in Vespa 8, with no replacement"); + } + for (Element clientBinding : clientBindingsElements) handler.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(clientBinding))); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java index cecc258a3b8..40e457ba69b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java @@ -13,7 +13,6 @@ import org.w3c.dom.Element; * producer in between. This should not be used by new model plugins. * * @author Ulf Lilleengen - * @since 5.1 */ public abstract class LegacyConfigModelBuilder<MODEL extends ConfigModel> extends ConfigModelBuilder<MODEL> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 81923db15dd..0be3ac01f71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -69,20 +69,20 @@ public class VespaDomBuilder extends VespaModelBuilder { // TODO Move and change scope public static List<String> getHostAliases(NodeList hostAliases) { List<String> aliases = new LinkedList<>(); - for (int i=0; i < hostAliases.getLength(); i++) { + for (int i = 0; i < hostAliases.getLength(); i++) { Node n = hostAliases.item(i); if (! (n instanceof Element)) { continue; } Element e = (Element)n; if (! e.getNodeName().equals("alias")) { - throw new RuntimeException("Unexpected tag: '" + e.getNodeName() + "' at node " + - XML.getNodePath(e, " > ") + ", expected 'alias'."); + throw new IllegalArgumentException("Unexpected tag: '" + e.getNodeName() + "' at node " + + XML.getNodePath(e, " > ") + ", expected 'alias'."); } String alias = e.getFirstChild().getNodeValue(); if ((alias == null) || (alias.equals(""))) { - throw new RuntimeException("Missing value for the alias tag at node " + - XML.getNodePath(e, " > ") + "'."); + throw new IllegalArgumentException("Missing value for the alias tag at node " + + XML.getNodePath(e, " > ") + "'."); } aliases.add(alias); } @@ -113,7 +113,6 @@ public class VespaDomBuilder extends VespaModelBuilder { * include hostalias, baseport and user config overrides generically. * * @param <T> an {@link com.yahoo.config.model.producer.AbstractConfigProducer} - * @author vegardh */ public static abstract class DomConfigProducerBuilder<T extends AbstractConfigProducer<?>> { @@ -184,7 +183,7 @@ public class VespaDomBuilder extends VespaModelBuilder { // This depends on which constructor in AbstractService is used, but the best way // is to let this method do initialize. if (!t.isInitialized()) { - t.initService(deployState.getDeployLogger()); + t.initService(deployState); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java index 560ede554e6..0d54b7b49f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java @@ -134,15 +134,15 @@ public class ComponentsBuilder<T extends ChainedComponent<?>> { private void ensureNotDefinition(String componentName, Element componentSpec) { if (componentSpec.getAttributes().getLength() > 1 || !XML.getChildren(componentSpec).isEmpty()) - throw new RuntimeException("Expecting " + componentName + - " to be a reference to a global component with the same name," + - " so no additional attributes or nested elements are allowed"); + throw new IllegalArgumentException("Expecting " + componentName + + " to be a reference to a global component with the same name," + + " so no additional attributes or nested elements are allowed"); } private void ensureTypesMatch(ComponentType type1, ComponentType type2, String componentName) { if (!type1.equals(type2)) { - throw new RuntimeException("Two different types declared for the component with name '" + componentName + "' (" - + type1.name + " != " + type2.name + ")."); + throw new IllegalArgumentException("Two different types declared for the component with name '" + componentName + + "' (" + type1.name + " != " + type2.name + ")."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java index 92887bc3227..b62c1a01fa3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java @@ -6,9 +6,11 @@ import java.lang.reflect.InvocationTargetException; /** * Utility class for instantiating a builder using reflection. + * * @author Tony Vaagenes */ public class DomBuilderCreator { + public static <T> T create(Class<T> builderClass, Object... parameters) { try { return getConstructor(builderClass).newInstance(parameters); @@ -23,4 +25,5 @@ public class DomBuilderCreator { assert(constructors.length == 1); return (Constructor<T>) constructors[0]; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java index 9db28c83a93..19c5c489f7e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java @@ -157,7 +157,6 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { } } - @SuppressWarnings("deprecation") private Provider buildProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { @@ -166,7 +165,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { } else if (LocalProviderSpec.includesType(providerReader.type)) { return buildLocalProvider(specWithoutInnerSearchers, providerReader, federationOptions); } else { - throw new RuntimeException("Unknown provider type '" + providerReader.type + "'"); + throw new IllegalArgumentException("Unknown provider type '" + providerReader.type + "'"); } } @@ -194,7 +193,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { private void ensureEmpty(ComponentId componentId, Object... objects) { for (Object object : objects) { if (object != null) { - throw new RuntimeException("Invalid provider option in provider '" + componentId + "': value='" + object + "'"); + throw new IllegalArgumentException("Invalid provider option in provider '" + componentId + "': value='" + object + "'"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 7c386875d02..ba5e8ed5544 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -11,6 +11,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.Optional; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + /** * A container that is typically used by container clusters set up from the user application. * @@ -24,6 +28,7 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; + private final boolean useQrserverServiceName; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -33,6 +38,7 @@ public final class ApplicationContainer extends Container implements super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); + this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -56,7 +62,7 @@ public final class ApplicationContainer extends Container implements if (parent instanceof ContainerCluster) { ContainerCluster<?> cluster = (ContainerCluster<?>)parent; // TODO: The 'qrserver' name is retained for legacy reasons (e.g. system tests and log parsing). - if (cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { + if (useQrserverServiceName && cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { return ContainerServiceType.QRSERVER; } } @@ -99,4 +105,11 @@ public final class ApplicationContainer extends Container implements return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); } + @Override + public Optional<String> getPreShutdownCommand() { + int preshutdownTimeoutSeconds = 360; + int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; + String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; + return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c4d420f2d44..8f95e390b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -40,13 +40,13 @@ import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; /** @@ -94,14 +94,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private List<ApplicationClusterEndpoint> endpointList = List.of(); public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { - super(parent, configSubId, clusterId, deployState, true); + super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); - previousHosts = deployState.getPreviousModel().stream() - .map(Model::allocatedHosts) - .map(AllocatedHosts::getHosts) - .flatMap(Collection::stream) - .map(HostSpec::hostname) - .collect(Collectors.toUnmodifiableSet()); + previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() + .map(Model::allocatedHosts) + .map(AllocatedHosts::getHosts) + .flatMap(Collection::stream) + .map(HostSpec::hostname) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); addSimpleComponent("com.yahoo.language.provider.DefaultLinguisticsProvider"); addSimpleComponent("com.yahoo.language.provider.DefaultEmbedderProvider"); @@ -179,21 +179,18 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return Optional.empty(); } - /* - Create list of endpoints, these will be consumed later by the LBservicesProducer - */ + /** Create list of endpoints, these will be consumed later by LbServicesProducer */ private void createEndpointList(DeployState deployState) { if(!deployState.isHosted()) return; if(deployState.getProperties().applicationId().instance().isTester()) return; List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); - // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. - // For now support both L7 and L4 routing + // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. List<String> hosts = getContainers().stream() .map(AbstractService::getHostName) .collect(Collectors.toList()); - for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { - // L4 + + for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( deployState.zone().system(), ClusterSpec.Id.from(getName()), @@ -206,28 +203,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .hosts(hosts) .clusterId(getName()) .build()); - - // L7 - ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedRouting() - .dnsName(l7Name) - .hosts(hosts) - .clusterId(getName()) - .build()); } - // Then get all endpoints provided by controller. - Set<ApplicationClusterEndpoint.RoutingMethod> supportedRoutingMethods = Set.of(shared, sharedLayer4); + // Include all endpoints provided by controller Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); endpointsFromController.stream() .filter(ce -> ce.clusterId().equals(getName())) - .filter(ce -> supportedRoutingMethods.contains(ce.routingMethod())) + .filter(ce -> ce.routingMethod() == sharedLayer4) .forEach(ce -> ce.names().forEach( name -> endpoints.add(ApplicationClusterEndpoint.builder() .scope(ce.scope()) @@ -309,8 +291,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()) .id(container.index()) - .joining(!previousHosts.isEmpty() && - !previousHosts.contains(container.getHostName())); + .joining( ! previousHosts.isEmpty() && + ! previousHosts.contains(container.getHostName())) + .retired(container.isRetired()); builder.server(serverBuilder); builder.dynamicReconfiguration(true); } @@ -343,6 +326,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return endpointList; } + @Override + public String name() { return getName(); } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 5c5b065ac29..4b1c03a170c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -168,11 +168,11 @@ public abstract class Container extends AbstractService implements public void addBuiltinHandlers() { } @Override - public void initService(DeployLogger deployLogger) { + public void initService(DeployState deployState) { if (isInitialized()) return; // XXX: Must be called first, to set the baseport - super.initService(deployLogger); + super.initService(deployState); if (getHttp() == null) { initDefaultJettyConnector(); @@ -281,7 +281,7 @@ public abstract class Container extends AbstractService implements } protected int allocatedRpcPort = 0; - private int getRpcPort() { + protected int getRpcPort() { return allocatedRpcPort; } protected int numRpcPorts() { return rpcServerEnabled() ? 1 : 0; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index c73a3b2a676..b6ce1a7f457 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final ComponentGroup<Component<?, ?>> componentGroup; private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; + private final int numAvailableProcessors; private final Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); @@ -160,23 +161,25 @@ public abstract class ContainerCluster<CONTAINER extends Container> private boolean deferChangesUntilRestart = false; - public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity) { + public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity, int defaultPoolNumThreads) { super(parent, configSubId); this.name = clusterId; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; + numAvailableProcessors = deployState.featureFlags().availableProcessors(); componentGroup = new ComponentGroup<>(this, "component"); addCommonVespaBundles(); - // TODO Vespa 8: remove LoggingRequestHandler.Context (replaced by ThreadedHttpRequestHandler.Context) - addSimpleComponent(com.yahoo.container.jdisc.LoggingRequestHandler.Context.class); + // TODO Vespa 8: remove LoggingRequestHandler.Context component if we can break binary compatibility + // (ThreadedHttpRequestHandler.Context is source compatible.) + addSimpleComponent("com.yahoo.container.jdisc.LoggingRequestHandler$Context"); addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); + addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); @@ -331,8 +334,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public SearchChains getSearchChains() { if (containerSearch == null) - throw new IllegalStateException("Search components not found in container cluster '" + getSubId() + - "': Add <search/> to the cluster in services.xml"); + throw new IllegalArgumentException("Search components not found in container cluster '" + getSubId() + + "': Add <search/> to the cluster in services.xml"); return containerSearch.getChains(); } @@ -371,8 +374,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public DocprocChains getDocprocChains() { if (containerDocproc == null) - throw new IllegalStateException("Document processing components not found in container cluster '" + getSubId() + - "': Add <document-processing/> to the cluster in services.xml"); + throw new IllegalArgumentException("Document processing components not found in container cluster '" + getSubId() + + "': Add <document-processing/> to the cluster in services.xml"); return containerDocproc.getChains(); } @@ -493,7 +496,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void getConfig(QrStartConfig.Builder builder) { builder.jvm .verbosegc(false) - .availableProcessors(2) + .availableProcessors(numAvailableProcessors) .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(256) @@ -576,6 +579,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.system(zone.system().value()); builder.environment(zone.environment().value()); builder.region(zone.region().value()); + builder.cloud(zone.getCloud().name().value()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index e0d4f3c0692..0b37abaded9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -5,7 +5,6 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; /** @@ -16,38 +15,27 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolConfig.Producer { private final ContainerCluster<?> cluster; - private final int metricsproxyNumThreads; + private final int defaultWorkerThreads; - DefaultThreadpoolProvider(ContainerCluster<?> cluster, int metricsproxyNumThreads) { + DefaultThreadpoolProvider(ContainerCluster<?> cluster, int defaultWorkerThreads) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); this.cluster = cluster; - this.metricsproxyNumThreads = metricsproxyNumThreads; - } - - private int defaultThreadsByClusterType() { - if (cluster instanceof MetricsProxyContainerCluster) { - return metricsproxyNumThreads; - } - return 10; + this.defaultWorkerThreads = defaultWorkerThreads; } @Override public void getConfig(ThreadpoolConfig.Builder builder) { - if (!(cluster instanceof ApplicationContainerCluster)) { + if (cluster instanceof ApplicationContainerCluster) { + // Core pool size of 2xcores, and max of 100xcores and using a synchronous Q + // This is the deafault pool used by both federation and generally when you ask for an Executor. + builder.corePoolSize(-2).maxthreads(-100).queueSize(0); + } else { // Container clusters such as logserver, metricsproxy and clustercontroller - int defaultWorkerThreads = defaultThreadsByClusterType(); - builder.maxthreads(defaultWorkerThreads); - builder.corePoolSize(defaultWorkerThreads); - builder.queueSize(50); - return; + builder.corePoolSize(defaultWorkerThreads).maxthreads(defaultWorkerThreads).queueSize(50); } - - // Core pool size of 2xcores, and max of 100xcores and using a synchronous Q - // This is the deafault pool used by both federation and generally when you ask for an Executor. - builder.corePoolSize(-2).maxthreads(-100).queueSize(0); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index e58adf8d6b9..78fea787b7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -9,6 +9,8 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import java.util.Objects; + /** * @author Tony Vaagenes * @author gjoranv @@ -52,7 +54,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); - this.fileNamePattern = fileNamePattern; + this.fileNamePattern = Objects.requireNonNull(fileNamePattern, "File name pattern required when configuring access log"); this.rotationInterval = rotationInterval; this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; @@ -60,9 +62,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compressionType = compressionType; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; - - if (fileNamePattern == null) - throw new RuntimeException("File name pattern required when configuring access log."); } private static String accessLogClass(AccessLogType logType) { @@ -114,4 +113,5 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL public String getFileNamePattern() { return fileNamePattern; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 302e8eff2d8..413c6f07efd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -83,8 +83,6 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } - - builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index c61c140c05b..f342aa1a2bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -21,22 +21,20 @@ public interface CloudConfigOptions { Optional<Integer> rpcPort(); Optional<Boolean> multiTenant(); Optional<Boolean> hostedVespa(); - ConfigServer[] allConfigServers(); int[] configServerZookeeperIds(); Optional<Integer> zookeeperClientPort(); String[] configModelPluginDirs(); Optional<Long> sessionLifeTimeSecs(); - Optional<Long> zookeeperBarrierTimeout(); //in seconds Optional<Integer> zookeeperElectionPort(); Optional<Integer> zookeeperQuorumPort(); Optional<String> environment(); Optional<String> region(); Optional<String> system(); + default Optional<String> cloud() { return Optional.empty(); } Optional<Boolean> useVespaVersionInRequest(); Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); - String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java index da6da75f409..167dac4c57e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java @@ -6,13 +6,13 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig; -import com.yahoo.path.Path; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.ContainerCluster; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -41,10 +41,11 @@ public class BlockFeedGlobalEndpointsFilter extends Filter implements RuleBasedF public void getConfig(RuleBasedFilterConfig.Builder builder) { Set<String> hostNames = endpoints.stream() .flatMap(e -> e.names().stream()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if(hostNames.size() > 0) { + Collection<String> hostnamesSorted = hostNames.stream().sorted().collect(Collectors.toList()); RuleBasedFilterConfig.Rule.Builder rule = new RuleBasedFilterConfig.Rule.Builder() - .hostNames(hostNames) + .hostNames(hostnamesSorted) .pathExpressions(ContainerCluster.RESERVED_URI_PREFIX + "/{*}") .pathExpressions(ContainerDocumentApi.DOCUMENT_V1_PREFIX + "/{*}") .methods(List.of(PUT, POST, DELETE)) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 59e8c493709..f5d1d3e6afd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -98,7 +98,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl for (FilterBinding binding: bindings) { if (filters.getComponent(binding.chainId()) == null && chains.getComponent(binding.chainId()) == null) - throw new RuntimeException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); + throw new IllegalArgumentException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 7b91b4b3244..33cbb328c52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -75,6 +75,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); } configureJettyThreadpool(builder); + builder.stopTimeout(300); } private void configureJettyThreadpool(ServerConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index b7bacb34b05..e3aa5bd517f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.Collection; import java.util.List; -import java.util.Set; +import java.util.stream.Collectors; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -88,9 +88,9 @@ public class HostedSslConnectorFactory extends ConnectorFactory { connectorBuilder.ssl.enabledProtocols(List.of("TLSv1.2")); if (!tlsCiphersOverride.isEmpty()) { - connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride); + connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride.stream().sorted().collect(Collectors.toList())); } else { - connectorBuilder.ssl.enabledCipherSuites(Set.copyOf(TlsContext.ALLOWED_CIPHER_SUITES)); + connectorBuilder.ssl.enabledCipherSuites(TlsContext.ALLOWED_CIPHER_SUITES.stream().sorted().collect(Collectors.toList())); } connectorBuilder diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 1c0601915e9..13503906c04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -106,7 +106,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> .orElse(null); if (tenantDomain == null) { if (explicitDomain == null) { - throw new IllegalStateException("No Athenz domain provided for 'access-control'"); + throw new IllegalArgumentException("No Athenz domain provided for 'access-control'"); } deployState.getDeployLogger().logApplicationPackage(Level.WARNING, "Athenz tenant is not provided by deploy call. This will soon be handled as failure."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index bf5bcbff360..6532fed9abf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -84,7 +84,7 @@ public class ModelsEvaluatorTester { return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, rankingExpressionsConfig, onnxModelsConfig, files); } catch (IOException | SAXException e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } finally { if (temporaryApplicationDir != null) { IOUtils.recursiveDeleteDir(temporaryApplicationDir); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 0e53417aea9..e0b7c3b288a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -125,7 +125,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). name(sys.getClusterName()); for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) { - scB.searchdef(spec.getSearchDefinition().getName()); + scB.searchdef(spec.getSchema().getName()); } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index 55284c444e4..1abb62fedab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -208,14 +208,14 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem Target target = targetResolver.resolve(targetSpec); if (target == null) { - throw new RuntimeException("Can't find source " + targetSpec.sourceSpec + - " used as a source for federation '" + getComponentId() + "'"); + throw new IllegalArgumentException("Can't find source " + targetSpec.sourceSpec + + " used as a source for federation '" + getComponentId() + "'"); } Target duplicate = resolvedTargets.put(target.id, target); if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { - throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + - " to the federation searcher " + getComponentId()); + throw new IllegalArgumentException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index e2579d95fc0..7eefa2e5e74 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -116,7 +116,7 @@ public class LocalProvider extends Provider implements List<String> documentTypes = new ArrayList<>(); for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { - documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); + documentTypes.add(spec.getSchema().getDocument().getName()); } return documentTypes; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java index a733c2a0f30..a71e1936425 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -41,9 +41,8 @@ public class SearchChains extends Chains<SearchChain> { private void setSearchClusterForLocalProvider(Map<String, ? extends AbstractSearchCluster> clusterIndexByName) { for (LocalProvider provider : localProviders()) { AbstractSearchCluster cluster = clusterIndexByName.get(provider.getClusterName()); - if (cluster == null) { - throw new RuntimeException("No searchable content cluster with id '" + provider.getClusterName() + "'"); - } + if (cluster == null) + throw new IllegalArgumentException("No searchable content cluster with id '" + provider.getClusterName() + "'"); provider.setSearchCluster(cluster); } } @@ -53,9 +52,8 @@ public class SearchChains extends Chains<SearchChain> { sourceGroup.validate(); if (getChainGroup().getComponentMap().containsKey(sourceGroup.getComponentId())) { - throw new RuntimeException( - String.format("Same id used for a source and another search chain/provider: '%s'", - sourceGroup.getComponentId())); + throw new IllegalArgumentException("Id '" + sourceGroup.getComponentId() + + "' is used both for a source and another search chain/provider"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java index 0be6675915c..7abb3111afd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java @@ -9,25 +9,21 @@ import java.util.LinkedHashSet; import java.util.Set; /** - * A set of sources with the same name, - * each associated with a different provider, - * that fills the same role. + * A set of sources with the same name, each associated with a different provider, that fills the same role. + * * @author Tony Vaagenes */ final class SourceGroup { + private final ComponentId id; private Source leader; - private final Set<Source> participants = - new LinkedHashSet<>(); + private final Set<Source> participants = new LinkedHashSet<>(); private void setLeader(Source leader) { assert (validMember(leader)); - if (this.leader != null) { - throw new IllegalStateException( - "There can not be two default providers for the source " - + id); - } + if (this.leader != null) + throw new IllegalArgumentException("There can not be two default providers for the source '" + id + "'"); this.leader = leader; } @@ -36,10 +32,8 @@ final class SourceGroup { assert (validMember(source)); assert (!source.equals(leader)); - if (!participants.add(source)) { - throw new RuntimeException("Source added twice to the same group " - + source); - } + if (!participants.add(source)) + throw new IllegalArgumentException("Source '" + source + "' added twice to the same group"); } private boolean validMember(Source leader) { @@ -55,8 +49,8 @@ final class SourceGroup { } public void add(Source source) { - assert source.getComponentId().equals(getComponentId()): - "Ids differ: " + source.getComponentId() + " -- " + getComponentId(); + if ( ! source.getComponentId().equals(getComponentId())) + throw new IllegalStateException("Ids differ: " + source.getComponentId() + " and " + getComponentId()); if (Source.GroupOption.leader == source.groupOption) { setLeader(source); @@ -89,7 +83,8 @@ final class SourceGroup { public void validate() { if (leader == null) - throw new IllegalStateException("Missing leader for the source " + getComponentId() + - ". One of the sources must use the attribute id instead of idref."); + throw new IllegalArgumentException("Missing leader for the source " + getComponentId() + + ". One of the sources must use the attribute id instead of idref."); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 5e1cf6db770..fbdefd0afb0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -43,7 +43,7 @@ public class BundleInstantiationSpecificationBuilder { for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { - throw new RuntimeException("Setting up " + forbiddenClass + " manually is not supported."); + throw new IllegalArgumentException("Setting up " + forbiddenClass + " manually is not supported"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 986d5a21c70..c88d225f527 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -96,11 +96,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -162,6 +164,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { checkVersion(spec); checkTagName(spec, log); + checkDeprecatedAttributes(spec, log); ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); @@ -171,6 +174,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { model.setCluster(cluster); } + private void checkDeprecatedAttributes(Element spec, DeployLogger log) { + String version = spec.getAttribute("jetty"); + if (!version.isEmpty()) { + log.logApplicationPackage(WARNING, + "The 'jetty' attribute is deprecated and will be removed in Vespa 8. " + + "It has no effect - Jetty is always enabled." + + "Please remove the attribute from the 'container'/'jdisc' element in services.xml."); + } + } + private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder<ApplicationContainerCluster>() { @Override @@ -281,7 +294,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { if ( ! deployState.isHosted()) return; if ( ! cluster.getZone().system().isPublic()) - throw new RuntimeException("cloud secret store is not supported in non-public system, please see documentation"); + throw new IllegalArgumentException("Cloud secret store is not supported in non-public system, see the documentation"); CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map<String, TenantSecretStore> secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() @@ -296,10 +309,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { TenantSecretStore secretStore = secretStoresByName.get(account); if (secretStore == null) - throw new RuntimeException("No configured secret store named " + account); + throw new IllegalArgumentException("No configured secret store named " + account); if (secretStore.getExternalId().isEmpty()) - throw new RuntimeException("No external ID has been set"); + throw new IllegalArgumentException("No external ID has been set"); cloudSecretStore.addConfig(account, region, secretStore.getAwsId(), secretStore.getRole(), secretStore.getExternalId().get()); } @@ -347,7 +360,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Only consider global endpoints. .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global) .flatMap(endpoint -> endpoint.names().stream()) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); // Build the comma delimited list of endpoints this container should be known as. // Confusingly called 'rotations' for legacy reasons. @@ -358,6 +371,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (environment != Environment.prod) return; Element aliases = XML.getChild(spec, "aliases"); + if (aliases != null) { + log.logApplicationPackage(WARNING, "The 'aliases' element and its children has no effect. They have been deprecated for removal in Vespa 8"); + } for (Element alias : XML.getChildren(aliases, "service-alias")) { cluster.serviceAliases().add(XML.getValue(alias)); } @@ -389,7 +405,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { - for (Element clientSpec: XML.getChildren(spec, "client")) { + List<Element> clientElements = XML.getChildren(spec, "client"); + if (! clientElements.isEmpty()) { + log.logApplicationPackage( + Level.WARNING, "The 'client' element is deprecated for removal in Vespa 8, with no replacement"); + } + for (Element clientSpec : clientElements) { cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec)); } } @@ -447,13 +468,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { - throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane"); + throw new IllegalArgumentException("Client certificate authority security/clients.pem is missing - " + + "see: https://cloud.vespa.ai/en/security-model#data-plane"); } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); boolean enforceHandshakeClientAuth = cluster.getHttp().getAccessControl() .map(accessControl -> accessControl.clientAuthentication) - .map(clientAuth -> clientAuth.equals(AccessControl.ClientAuthentication.need)) + .map(clientAuth -> clientAuth == AccessControl.ClientAuthentication.need) .orElse(false); connectorFactory = authorizeClient @@ -632,14 +654,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void checkVersion(Element spec) { String version = spec.getAttribute("version"); - if ( ! Version.fromString(version).equals(new Version(1))) { - throw new RuntimeException("Expected container version to be 1.0, but got " + version); - } + if ( ! Version.fromString(version).equals(new Version(1))) + throw new IllegalArgumentException("Expected container version to be 1.0, but got " + version); } private void checkTagName(Element spec, DeployLogger logger) { if (spec.getTagName().equals(DEPRECATED_CONTAINER_TAG)) { - logger.logApplicationPackage(WARNING, "'" + DEPRECATED_CONTAINER_TAG + "' is deprecated as tag name. Use '" + CONTAINER_TAG + "' instead."); + logger.logApplicationPackage(WARNING, "'" + DEPRECATED_CONTAINER_TAG + + "' is deprecated as tag name. Use '" + CONTAINER_TAG + "' instead."); } } @@ -806,7 +828,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); - return createNodesFromHosts(log, hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); @@ -816,7 +838,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, context.getDeployState()); node.setHostResource(host); - node.initService(context.getDeployLogger()); + node.initService(context.getDeployState()); return List.of(node); } @@ -827,7 +849,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec.Id.from(cluster.getName()), log, hasZooKeeper(containerElement)); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -839,7 +861,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), log); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List<ApplicationContainer> createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -857,11 +879,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { referenceId, cluster.getRoot().hostSystem(), context.getDeployLogger()); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } - private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, - Map<HostResource, ClusterMembership> hosts, + private List<ApplicationContainer> createNodesFromHosts(Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster, DeployState deployState) { List<ApplicationContainer> nodes = new ArrayList<>(); @@ -869,7 +890,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String id = "container." + entry.getValue().index(); ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), deployState); container.setHostResource(entry.getKey()); - container.initService(deployLogger); + container.initService(deployState); nodes.add(container); } return nodes; @@ -1020,9 +1041,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AthenzService service = spec.instance(app.getApplicationId().instance()) .flatMap(instanceSpec -> instanceSpec.athenzService(zone.environment(), zone.region())) .or(() -> spec.athenzService()) - .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration in instance '" + app.getApplicationId().instance() + "'")); + .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + + app.getApplicationId().instance() + "'")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; - IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); + IdentityProvider identityProvider = new IdentityProvider(domain, + service, + getLoadBalancerName(loadBalancerName, configServerSpecs), + ztsUrl, + zoneDnsSuffix, + zone); cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { @@ -1152,7 +1179,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (invalidOptions.isEmpty()) return; - String message = "Invalid JVM options in services.xml: " + String.join(",", invalidOptions); + String message = "Invalid or misplaced JVM options in services.xml: " + + String.join(",", invalidOptions) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else @@ -1216,7 +1245,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (options.isEmpty()) return; Collections.sort(options); - String message = "Invalid JVM GC options in services.xml: " + String.join(",", options); + String message = "Invalid or misplaced JVM GC options in services.xml: " + + String.join(",", options) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java index 628987d5930..e15c531aa32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java @@ -8,9 +8,9 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; * Represents configuration for bucket splitting. */ public class BucketSplitting implements StorDistributormanagerConfig.Producer { - Integer maxDocuments; - Integer splitSize; - Integer minSplitCount; + private final Integer maxDocuments; + private final Integer splitSize; + private final Integer minSplitCount; public static class Builder { public BucketSplitting build(ModelElement clusterElem) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 25e00dc05dd..f0b32f9140d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -41,7 +41,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr minNodeRatioPerGroup = clusterTuning.childAsDouble("min-node-ratio-per-group"); bucketSplittingMinimumBits = clusterTuning.childAsInteger("bucket-splitting.minimum-bits"); } - boolean enableClusterFeedBlock = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); if (tuning != null) { return new ClusterControllerConfig(ancestor, clusterName, @@ -53,14 +52,13 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); + resourceLimits); } } } @@ -74,7 +72,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr private final Double minStorageUpRatio; private final Integer minSplitBits; private final Double minNodeRatioPerGroup; - private final boolean enableClusterFeedBlock; private final ResourceLimits resourceLimits; // TODO refactor; too many args @@ -88,7 +85,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - boolean enableClusterFeedBlock, ResourceLimits resourceLimits) { super(parent, "fleetcontroller"); @@ -101,7 +97,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr this.minStorageUpRatio = minStorageUpRatio; this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; - this.enableClusterFeedBlock = enableClusterFeedBlock; this.resourceLimits = resourceLimits; } @@ -144,7 +139,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr if (minNodeRatioPerGroup != null) { builder.min_node_ratio_per_group(minNodeRatioPerGroup); } - builder.enable_cluster_feed_block(enableClusterFeedBlock); resourceLimits.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 0a08f81ffca..80dd17213f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -34,7 +34,6 @@ public class ClusterResourceLimits { public static class Builder { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final double resourceLimitDisk; private final double resourceLimitMemory; @@ -42,11 +41,9 @@ public class ClusterResourceLimits { private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, - boolean hostedVespa, + public Builder(boolean hostedVespa, double resourceLimitDisk, double resourceLimitMemory) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; this.resourceLimitDisk = resourceLimitDisk; this.resourceLimitMemory = resourceLimitMemory; @@ -81,17 +78,15 @@ public class ClusterResourceLimits { } private void deriveLimits() { - if (enableFeedBlockInDistributor) { - // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), - nodeBuilder.getDiskLimit(), - ctrlBuilder::setDiskLimit, - resourceLimitDisk); - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), - nodeBuilder.getMemoryLimit(), - ctrlBuilder::setMemoryLimit, - resourceLimitMemory); - } + // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), + nodeBuilder.getDiskLimit(), + ctrlBuilder::setDiskLimit, + resourceLimitDisk); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), + nodeBuilder.getMemoryLimit(), + ctrlBuilder::setMemoryLimit, + resourceLimitMemory); deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 533364d80c3..dcf0be48f7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -243,8 +243,8 @@ public class Content extends ConfigModel { String indexingClusterName = cluster.getIndexingClusterName(); ContainerModel containerModel = findByName(indexingClusterName, containers); if (containerModel == null) - throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + - "cluster '" + indexingClusterName + "', but this cluster does not exist."); + throw new IllegalArgumentException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + "cluster '" + indexingClusterName + "', but this cluster does not exist."); addIndexingChainsTo(containerModel.getCluster(), cluster); } @@ -312,7 +312,7 @@ public class Content extends ConfigModel { index++; docprocService.useDynamicPorts(); docprocService.setHostResource(host); - docprocService.initService(modelContext.getDeployLogger()); + docprocService.initService(modelContext.getDeployState()); nodes.add(docprocService); processedHosts.add(host); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index d0cba617cfc..22b752777e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.builder.UserConfigBuilder; @@ -14,7 +15,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.search.NamedSchema; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; @@ -73,6 +73,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultFeedConcurrency; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; + private final ProtonConfig.Replay_throttling_policy.Type.Enum persistenceAsyncThrottling; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final double fractionOfMemoryReserved; @@ -202,6 +203,14 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } + private static ProtonConfig.Replay_throttling_policy.Type.Enum convertPersistenceAsyncThrottling(String value) { + try { + return ProtonConfig.Replay_throttling_policy.Type.Enum.valueOf(value); + } catch (Throwable t) { + return ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED; + } + } + private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, @@ -226,6 +235,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); + this.persistenceAsyncThrottling = convertPersistenceAsyncThrottling(featureFlags.persistenceAsyncThrottling()); } public void setVisibilityDelay(double delay) { @@ -249,27 +259,25 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); - NamedSchema searchDefinition = - schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas()); - if (searchDefinition == null) - throw new RuntimeException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + - this + " does not exist"); + Schema schema = schemaDefinitionXMLHandler.findResponsibleSchema(deployState.getSchemas()); + if (schema == null) + throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + + this + " does not exist"); // TODO: remove explicit building of user configs when the complete content model is built using builders. - sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition, + sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(schema, UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); - //need to get the document names from this sdfile - sc.addDocumentNames(searchDefinition); + sc.addDocumentNames(schema); } } private void addCluster(AbstractSearchCluster sc) { if (clusters.containsKey(sc.getClusterName())) { - throw new IllegalArgumentException("I already have registered cluster '" + sc.getClusterName() + "'"); + throw new IllegalArgumentException("Duplicate cluster '" + sc.getClusterName() + "'"); } if (sc instanceof IndexedSearchCluster) { if (indexedCluster != null) { - throw new IllegalArgumentException("I already have one indexed cluster named '" + indexedCluster.getClusterName()); + throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; } @@ -292,11 +300,11 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); - searchNode.initService(deployState.getDeployLogger()); + searchNode.initService(deployState); tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState.getDeployLogger()); + tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, fractionOfMemoryReserved) @@ -444,6 +452,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> builder.indexing.tasklimit(feedTaskLimit); builder.feeding.master_task_limit(feedMasterTaskLimit); builder.feeding.shared_field_writer_executor(sharedFieldWriterExecutor); + builder.replay_throttling_policy.type(persistenceAsyncThrottling); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 7a45689901f..3f01f5610f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -42,9 +42,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean hasIndexedDocumentType; private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; - private final int mergeBusyWait; - private final boolean enhancedMaintenanceScheduling; private final boolean unorderedMergeChaining; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -107,16 +106,13 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - int mergeBusyWait = deployState.getProperties().featureFlags().distributorMergeBusyWait(); - boolean useEnhancedMaintenanceScheduling = deployState.getProperties().featureFlags().distributorEnhancedMaintenanceScheduling(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, mergeBusyWait, - useEnhancedMaintenanceScheduling, - unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); } } @@ -124,9 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - int mergeBusyWait, - boolean enhancedMaintenanceScheduling, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean inhibitDefaultMergesWhenGlobalMergesPending) { super(parent, "distributor"); this.parent = parent; @@ -135,9 +130,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.hasIndexedDocumentType = hasIndexedDocumentType; this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.mergeBusyWait = mergeBusyWait; - this.enhancedMaintenanceScheduling = enhancedMaintenanceScheduling; this.unorderedMergeChaining = unorderedMergeChaining; + this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -151,9 +145,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.disable_bucket_activation(hasIndexedDocumentType == false); builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.inhibit_merge_sending_on_busy_node_duration_sec(mergeBusyWait); - builder.implicitly_clear_bucket_priority_on_schedule(enhancedMaintenanceScheduling); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java index 4ea07ae06de..235128f7a62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -3,12 +3,13 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toSet; /** * Performs the following validations: @@ -29,7 +30,7 @@ public class GlobalDistributionValidator { private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) .filter(name -> !documentDefinitions.containsKey(name.toString())) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if (!unknowDocuments.isEmpty()) { throw new IllegalArgumentException("The following document types are referenced from other documents, " + "but are not listed in services.xml: " + asPrintableString(unknowDocuments.stream())); @@ -41,7 +42,7 @@ public class GlobalDistributionValidator { Set<NewDocumentType> nonGlobalReferencedDocuments = getReferencedDocuments(documentDefinitions) .map(name -> documentDefinitions.get(name.toString())) .filter(documentType -> !globallyDistributedDocuments.contains(documentType)) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if (!nonGlobalReferencedDocuments.isEmpty()) { throw new IllegalArgumentException("The following document types are referenced from other documents, " + "but are not globally distributed: " + asPrintableString(toDocumentNameStream(nonGlobalReferencedDocuments))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index 26bf8a33a73..df269c7a9a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -31,13 +31,12 @@ public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonCon @Override public void getConfig(FleetcontrollerConfig.Builder builder) { - // TODO: Choose other defaults when this is default enabled. // Note: The resource categories must match the ones used in host info reporting // between content nodes and cluster controller: // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("attribute-address-space", 0.89); + builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.75)); + builder.cluster_feed_block_limit.put("attribute-address-space", 0.9); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index a07a36d800b..88739e92567 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -211,7 +211,7 @@ public class StorageGroup { Optional<ModelElement> nodes = getNodes(clusterElement); if (group.isPresent() && nodes.isPresent()) - throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal"); + throw new IllegalArgumentException("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration"); if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null)) deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); @@ -280,15 +280,13 @@ public class StorageGroup { /** The nodes explicitly specified as a nodes tag in this group, or empty if none */ private final Optional<NodesSpecification> nodeRequirement; - private final DeployLogger deployLogger; private GroupBuilder(StorageGroup storageGroup, List<GroupBuilder> subGroups, List<XmlNodeBuilder> nodeBuilders, - Optional<NodesSpecification> nodeRequirement, DeployLogger deployLogger) { + Optional<NodesSpecification> nodeRequirement) { this.storageGroup = storageGroup; this.subGroups = subGroups; this.nodeBuilders = nodeBuilders; this.nodeRequirement = nodeRequirement; - this.deployLogger = deployLogger; } /** @@ -319,11 +317,11 @@ public class StorageGroup { StorageNode searchNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), 1.0, distributionKey , false); searchNode.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); PersistenceEngine provider = parent.getPersistence().create(deployState, searchNode, storageGroup, null); - searchNode.initService(deployLogger); + searchNode.initService(deployState); Distributor distributor = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), distributionKey, null, provider); distributor.setHostResource(searchNode.getHostResource()); - distributor.initService(deployLogger); + distributor.initService(deployState); return searchNode; } @@ -339,7 +337,7 @@ public class StorageGroup { throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployLogger) : + provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -450,7 +448,7 @@ public class StorageGroup { else // Nodes or groups explicitly listed - resolve in GroupBuilder nodeRequirement = Optional.empty(); - return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, context.getDeployLogger()); + return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement); } private Optional<String> childAsString(Optional<ModelElement> element, String childTagName) { @@ -503,13 +501,13 @@ public class StorageGroup { private static StorageNode createStorageNode(DeployState deployState, ContentCluster parent, HostResource hostResource, StorageGroup parentGroup, ClusterMembership clusterMembership) { StorageNode sNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), null, clusterMembership.index(), clusterMembership.retired()); sNode.setHostResource(hostResource); - sNode.initService(deployState.getDeployLogger()); + sNode.initService(deployState); // TODO: Supplying null as XML is not very nice PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, parentGroup, null); Distributor d = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), clusterMembership.index(), null, provider); d.setHostResource(sNode.getHostResource()); - d.initService(deployState.getDeployLogger()); + d.initService(deployState); return sNode; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 628cf6bb4c7..0d8f7148758 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -120,9 +120,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); - var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - stateIsHosted(deployState), + var resourceLimits = new ClusterResourceLimits.Builder(stateIsHosted(deployState), deployState.featureFlags().resourceLimitDisk(), deployState.featureFlags().resourceLimitMemory()) .build(contentElement); @@ -142,7 +140,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) - throw new RuntimeException("Indexed search requires proton as engine"); + throw new IllegalArgumentException("Indexed search requires proton as engine"); if (documentsElement != null) { ModelElement e = documentsElement.child("document-processing"); @@ -220,14 +218,14 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce if (distribution != null) { String attr = distribution.stringAttribute("type"); if (attr != null) { - if (attr.toLowerCase().equals("strict")) { + if (attr.equalsIgnoreCase("strict")) { c.distributionMode = DistributionMode.STRICT; - } else if (attr.toLowerCase().equals("loose")) { + } else if (attr.equalsIgnoreCase("loose")) { c.distributionMode = DistributionMode.LOOSE; - } else if (attr.toLowerCase().equals("legacy")) { + } else if (attr.equalsIgnoreCase("legacy")) { c.distributionMode = DistributionMode.LEGACY; } else { - throw new IllegalStateException("Distribution type " + attr + " not supported."); + throw new IllegalArgumentException("Distribution type " + attr + " not supported."); } } } @@ -367,7 +365,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce boolean retired = host.spec().membership().map(ClusterMembership::retired).orElse(false); var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, runStandaloneZooKeeper, deployState, retired); clusterControllerContainer.setHostResource(host); - clusterControllerContainer.initService(deployState.getDeployLogger()); + clusterControllerContainer.initService(deployState); clusterControllerContainer.setProp("clustertype", "admin"); containers.add(clusterControllerContainer); ++index; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java index a4231105f42..1cab6cd462d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java @@ -5,10 +5,10 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; - -import static java.util.stream.Collectors.toSet; +import java.util.stream.Collectors; /** * Determines the set of document types that are configured to be globally distributed. @@ -32,7 +32,7 @@ public class GlobalDistributionBuilder { .filter(GlobalDistributionBuilder::isGloballyDistributed) .map(GlobalDistributionBuilder::getDocumentName) .map(this::getDocumentType) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static boolean isGloballyDistributed(ModelElement e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java index d80cb0056fa..f45bc8f4eca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -23,7 +23,7 @@ public class SearchDefinitionBuilder { if (documentType != null) { docTypes.put(documentType.getName(), documentType); } else { - throw new RuntimeException("Document type '" + name + "' not found in application package"); + throw new IllegalArgumentException("Document type '" + name + "' not found in application package"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 1f3a76b766e..f3263658717 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,9 +46,10 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; + private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; + private final double persistenceThrottlingWsDecrementFactor; + private final double persistenceThrottlingWsBackoff; private final boolean useAsyncMessageHandlingOnSchedule; - private final boolean asyncApplyBucketDiff; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -58,11 +59,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } - private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) { try { - return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType); } catch (Throwable t) { - return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED; } } @@ -72,8 +73,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); - useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); - asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); + this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); + this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); + this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); } @Override @@ -85,8 +87,16 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.async_apply_bucket_diff(asyncApplyBucketDiff); - builder.async_operation_throttler_type(asyncOperationThrottlerType); + // TODO remove deprecated throttler type config + builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC) + ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC + : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED); + + var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); + throttleBuilder.type(asyncOperationThrottlerType); + throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); + throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); + builder.async_operation_throttler(throttleBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index c7d581fca35..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,7 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; - private Boolean ignoreMergeQueueLimit; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -51,11 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); - ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -75,8 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } - if (ignoreMergeQueueLimit != null) { - builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit); - } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java index ac5c456a7b1..3962aa3d612 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java @@ -22,7 +22,10 @@ public class FileReferencesRepository { } public Set<FileReference> allFileReferences() { - return fileRegistry.export().stream().map(e -> e.reference).collect(Collectors.toUnmodifiableSet()); + return fileRegistry.export() + .stream() + .map(e -> e.reference) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 1488f5fd112..38b1d42862e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -87,7 +87,7 @@ public class ConvertedModel { ApplicationPackage applicationPackage = context.rankProfile().applicationPackage(); ImportedMlModel sourceModel = // TODO: Convert to name here, make sure its done just one way context.importedModels().get(sourceModelFile(applicationPackage, modelPath)); - ModelName modelName = new ModelName(context.rankProfile().getName(), modelPath, pathIsFile); + ModelName modelName = new ModelName(context.rankProfile().name(), modelPath, pathIsFile); if (sourceModel == null && ! new ModelStore(applicationPackage, modelName).exists()) throw new IllegalArgumentException("No model '" + modelPath + "' is available. Available models: " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index e642da6176e..754800a42e1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -317,7 +317,7 @@ public final class DocumentProtocol implements Protocol, DocprocChain defaultChain = getDefaultChain(containerCluster.getDocproc()); if (defaultChain != null) { if (result != null) - throw new RuntimeException("Only a single default docproc chain is allowed across all container clusters"); + throw new IllegalArgumentException("Only a single default docproc chain is allowed across all container clusters"); result = defaultChain; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 19b1f39c87d..93a264ae4bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; @@ -37,9 +38,8 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr this.index = index; } - public void addDocumentNames(NamedSchema searchDefinition) { - String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); - documentNames.add(dName); + public void addDocumentNames(Schema schema) { + documentNames.add(schema.getDocument().getDocumentName().getName()); } /** Returns a List with document names used in this search cluster */ @@ -51,16 +51,15 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr public String getClusterName() { return clusterName; } public final String getIndexingModeName() { return getIndexingMode().getName(); } - public final boolean isRealtime() { return getIndexingMode() == IndexingMode.REALTIME; } public final boolean isStreaming() { return getIndexingMode() == IndexingMode.STREAMING; } public final AbstractSearchCluster setQueryTimeout(Double to) { - this.queryTimeout=to; + this.queryTimeout = to; return this; } public final AbstractSearchCluster setVisibilityDelay(double delay) { - this.visibilityDelay=delay; + this.visibilityDelay = delay; return this; } @@ -103,16 +102,16 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr public static final class SchemaSpec { - private final NamedSchema searchDefinition; + private final Schema schema; private final UserConfigRepo userConfigRepo; - public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) { - this.searchDefinition = searchDefinition; + public SchemaSpec(Schema schema, UserConfigRepo userConfigRepo) { + this.schema = schema; this.userConfigRepo = userConfigRepo; } - public NamedSchema getSearchDefinition() { - return searchDefinition; + public Schema getSchema() { + return schema; } public UserConfigRepo getUserConfigs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a68993be543..fb7c6696b54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -196,10 +196,9 @@ public class IndexedSearchCluster extends SearchCluster @Override protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) { for (SchemaSpec spec : localSearches) { - Schema schema = spec.getSearchDefinition().getSearch(); - if ( ! (schema instanceof DocumentOnlySchema)) { - DocumentDatabase db = new DocumentDatabase(this, schema.getName(), - new DerivedConfiguration(schema, + if ( ! (spec.getSchema() instanceof DocumentOnlySchema)) { + DocumentDatabase db = new DocumentDatabase(this, spec.getSchema().getName(), + new DerivedConfiguration(spec.getSchema(), deployState.getDeployLogger(), deployState.getProperties(), deployState.rankProfileRegistry(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java deleted file mode 100644 index 334493b4a92..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.searchdefinition.Schema; - -import java.util.Collection; - -/** - * @author Tony Vaagenes - */ -// TODO: This class is quite pointless -public class NamedSchema { - - private final Schema schema; - private final String name; - - public static final String fileNameSuffix = ".sd"; - - public Schema getSearch() { - return schema; - } - - public String getName() { - return name; - } - - public NamedSchema(String name, Schema schema) { - this.name = name; - this.schema = schema; - } - - //Find search definition from a collection with the name specified - public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) { - for (NamedSchema candidate : schemas) { - if (candidate.getName().equals(schemaName) ) - return candidate; - } - - return null; - } - - // Used by admin interface - public String getFilename() { - return getName() + fileNameSuffix; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index 881b68e8396..b4082308195 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.io.Serializable; +import java.util.Collection; import java.util.List; /** @@ -13,19 +15,23 @@ import java.util.List; */ public class SchemaDefinitionXMLHandler implements Serializable { - private String sdName; + private String schemaName; public SchemaDefinitionXMLHandler(ModelElement elem) { - sdName = elem.stringAttribute("name"); - if (sdName == null) { - sdName = elem.stringAttribute("type"); + schemaName = elem.stringAttribute("name"); + if (schemaName == null) { + schemaName = elem.stringAttribute("type"); } } - public String getName() { return sdName; } + public String getName() { return schemaName; } - public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) { - return NamedSchema.findByName(getName(), schemas ); + public Schema findResponsibleSchema(List<Schema> schemas) { + for (Schema candidate : schemas) { + if (candidate.getName().equals(schemaName) ) + return candidate; + } + return null; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 28d1fbe72ef..6e3f3e1ebf5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -123,7 +123,6 @@ public class SearchNode extends AbstractService implements boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved) { super(parent, name); - setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.nodeSpec = nodeSpec; @@ -239,10 +238,7 @@ public class SearchNode extends AbstractService implements @Override public String getStartupCommand() { - if (getOmpNumThreads() != 1) { - throw new IllegalStateException("ompNumThreads must be 1"); - } - String startup = getEnvVariables() + "exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); + String startup = getEnv() + " exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); if (serviceLayerService != null) { startup = startup + " --serviceidentity " + serviceLayerService.getConfigId(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 7b328acdc98..42bb2a1da29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -85,14 +85,15 @@ public class StreamingSearchCluster extends SearchCluster implements @Override protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { - deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); + deriveSingleSearchDefinition(local.get(0).getSchema(), deployState); } else if (local.size() > 1){ - throw new IllegalStateException("Logical indexes are not supported: Got " + local.size() + " search definitions, expected 1"); + throw new IllegalArgumentException("Only a single schema is supported, got " + local.size()); } } private void deriveSingleSearchDefinition(Schema localSchema, DeployState deployState) { if (!localSchema.getName().equals(docTypeName)) { - throw new IllegalStateException("Mismatch between document type name (" + docTypeName + ") and name of search definition (" + localSchema.getName() + ")"); + throw new IllegalArgumentException("Document type name '" + docTypeName + + "' must be the same as the schema name '" + localSchema.getName() + "'"); } this.sdConfig = new DerivedConfiguration(localSchema, deployState.getDeployLogger(), deployState.getProperties(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index c6d94ccfea0..2f28da60e7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -294,7 +294,7 @@ public class Tuning extends AbstractConfigProducer<Tuning> implements ProtonConf public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { if (outputInt) { - if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); + if (maxSize != null) chunk.maxbytes(maxSize.intValue()); } else { throw new IllegalStateException("Fix this, chunk does not have long types"); } @@ -305,6 +305,7 @@ public class Tuning extends AbstractConfigProducer<Tuning> implements ProtonConf } public static class LogStore { + public Long maxFileSize = null; public Component chunk = null; public Double minFileSizeFactor = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java index 7dabefeec47..9f049f727d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java @@ -14,6 +14,7 @@ import java.util.Set; * @author Tony Vaagenes */ public class FreezableMap<K, V> implements Map<K, V> { + private boolean frozen = false; private Map<K, V> map; @@ -85,7 +86,7 @@ public class FreezableMap<K, V> implements Map<K, V> { public void freeze() { if (frozen) - throw new RuntimeException("The map has already been frozen."); + throw new IllegalStateException("The map has already been frozen"); frozen = true; map = Collections.unmodifiableMap(map); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java index 799107ea763..0ad28d37e93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.ConfigProducer; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -23,8 +24,7 @@ import java.util.stream.Collectors; */ public final class ReflectionUtil { - private ReflectionUtil() { - } + private ReflectionUtil() {} public static Set<ConfigKey<?>> getAllConfigsProduced(Class<? extends ConfigProducer> producerClass, String configId) { // TypeToken is @Beta in guava, so consider implementing a simple recursive method instead. @@ -32,7 +32,7 @@ public final class ReflectionUtil { return interfaces.rawTypes().stream() .filter(ReflectionUtil::isConcreteProducer) .map(i -> createConfigKeyFromInstance(i.getEnclosingClass(), configId)) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } /** @@ -64,6 +64,7 @@ public final class ReflectionUtil { /** * Compares the config instances and lists any differences that will require service restart. + * * @param from The previous config. * @param to The new config. * @return An object describing the difference. @@ -91,13 +92,9 @@ public final class ReflectionUtil { } private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) { - try { - String defName = ConfigInstance.getDefName(configInstClass); - String defNamespace = ConfigInstance.getDefNamespace(configInstClass); - return new ConfigKey<>(defName, configId, defNamespace); - } catch (IllegalArgumentException | SecurityException e) { - throw new RuntimeException(e); - } + String defName = ConfigInstance.getDefName(configInstClass); + String defNamespace = ConfigInstance.getDefNamespace(configInstClass); + return new ConfigKey<>(defName, configId, defNamespace); } private static boolean isConcreteProducer(Class<?> producerInterface) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 92633f61e67..2ccb9a82bdf 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -22,7 +22,7 @@ import com.yahoo.document.*; import com.yahoo.documentmodel.*; import com.yahoo.compress.Compressor; import com.yahoo.compress.CompressionType; -import com.yahoo.searchdefinition.Application; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.searchdefinition.DistributableResource; import com.yahoo.searchdefinition.document.*; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; @@ -73,7 +73,7 @@ import java.util.logging.Level; public class SDParser { private DocumentTypeManager docMan = null; - private Application application; + private ApplicationPackage applicationPackage; private FileRegistry fileRegistry; private DeployLogger deployLogger; private ModelContext.Properties properties; @@ -86,17 +86,17 @@ public class SDParser { * @param documentsOnly true to only parse the document aspect of a schema (e.g skip rank profiles) */ public SDParser(SimpleCharStream stream, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties, - Application application, RankProfileRegistry rankProfileRegistry, boolean documentsOnly) { this(stream); + this.applicationPackage = applicationPackage; this.fileRegistry = fileRegistry; this.deployLogger = deployLogger; this.properties = properties; - this.application = application; this.rankProfileRegistry = rankProfileRegistry; this.documentsOnly = documentsOnly; } @@ -395,16 +395,15 @@ SPECIAL_TOKEN : * The rule consumes any schema and returns the corresponding object. This is the only production that should * ever consume leading newlines. * - * @param dir the directory containing the file being parsed * @return the schema object */ -Schema schema(DocumentTypeManager docMan, String dir) : +Schema schema(DocumentTypeManager docMan) : { this.docMan = docMan; Schema schema; } { - (<NL>)* (schema = rootSchema(dir) | schema = rootDocument(dir)) + (<NL>)* (schema = rootSchema() | schema = rootDocument()) { return schema; } } @@ -412,10 +411,9 @@ Schema schema(DocumentTypeManager docMan, String dir) : * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume * trailing newline tokens. * - * @param dir the directory containing the file being parsed. * @return the schema definition object. */ -Schema rootSchema(String dir) : +Schema rootSchema() : { String name; String inherited = null; @@ -423,7 +421,7 @@ Schema rootSchema(String dir) : } { ( ( <SCHEMA> | <SEARCH> ) name = identifier() (<INHERITS> inherited = identifier() )? { - schema = new Schema(name, Optional.ofNullable(inherited), application, fileRegistry, deployLogger, properties); + schema = new Schema(name, applicationPackage, Optional.ofNullable(inherited), fileRegistry, deployLogger, properties); rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));} lbrace() (rootSchemaItem(schema) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) @@ -459,12 +457,11 @@ Object rootSchemaItem(Schema schema) : { } /** * Consumes a schema definition that contains only documents to be used for inheritance, etc. * - * @param dir the directory containing the file being parsed. * @return the schema definition object. */ -Schema rootDocument(String dir) : +Schema rootDocument() : { - Schema schema = new DocumentOnlySchema(application, fileRegistry, deployLogger, properties); + Schema schema = new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties); } { ( (rootDocumentItem(schema) (<NL>)*)*<EOF> ) @@ -552,9 +549,18 @@ Object documentBody(SDDocumentType document, Schema schema) : } void rawAsBase64(Schema schema) : -{} { - <RAW_AS_BASE64_IN_SUMMARY> { schema.enableRawAsBase64(); } + boolean enabled = false; +} +{ + <RAW_AS_BASE64_IN_SUMMARY> + { + enabled = true; + } + [ <COLON> ( <TRUE> | ( <FALSE> { enabled = false; } ) ) ] + { + schema.enableRawAsBase64(enabled); + } } /** @@ -1267,17 +1273,14 @@ Object summaryInDocument(DocumentSummary document) : summary.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(); - } lbrace() (summaryItem(op) (<NL>)*)* <RBRACE> { - if (op.destinationIterator().hasNext()) { + if (op.destinationIterator().hasNext()) { throw new ParseException("Summaries defined in a document-summary section " + "can not have a 'to' line."); } - op.applyToSummary(summary); - document.add(summary); return null; } @@ -2036,15 +2039,15 @@ Object rankProfileItem(RankProfile profile) : { } /** * This rule consumes an inherits statement of a rank-profile. * - * @param profile The profile to modify. + * @param profile the profile to modify */ void inheritsRankProfile(RankProfile profile) : { - String str; + String name; } { - <INHERITS> str = identifierWithDash() - { profile.setInherited(str); } + <INHERITS> name = identifierWithDash() { profile.inherit(name); } + ( <COMMA> name = identifierWithDash() { profile.inherit(name);; } )* } /** @@ -2561,7 +2564,7 @@ void constantTensor(RankProfile profile, String name) : { <LBRACE> (<NL>)* (( tensorString = tensorValue() | - tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.getName(), name)) ) (<NL>)* )* <RBRACE> + tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> { if (tensorType != null) { profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorType, tensorString))); diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 819e6b79fbb..3e751a379d4 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -52,6 +52,7 @@ ParallelInstances = element parallel { Upgrade = element upgrade { attribute policy { xsd:string }? & + attribute revision { xsd:string }? & attribute rollout { xsd:string }? } diff --git a/config-model/src/main/resources/schema/legacygenericcluster.rnc b/config-model/src/main/resources/schema/legacygenericcluster.rnc deleted file mode 100644 index b00d4ff097b..00000000000 --- a/config-model/src/main/resources/schema/legacygenericcluster.rnc +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Generic, application-specific service cluster -# -include "legacygenericmodule.rnc" - -LegacyGenericCluster = element cluster { - attribute name { text } & - attribute command { text } & - attribute hostservice { text }? & - attribute num-hosts { text }? & - GenericConfig* & - LegacyGenericModule* & - element node { - service.attlist & - attribute name { text }? & - LegacyGenericModule* & - GenericConfig* - }* -} - diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index c8467898639..3a8ffe30563 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -7,13 +7,11 @@ include "docproc.rnc" include "routing.rnc" include "containercluster.rnc" include "genericcluster.rnc" -include "legacygenericcluster.rnc" start = element services { attribute version { "1.0" }? & attribute application-type { "hosted-infrastructure" }? & element legacy { element v7-geo-positions { xsd:boolean } }? & - LegacyGenericCluster* & GenericCluster* & GenericConfig* & Admin? & diff --git a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF index 760a9ecf00f..760a9ecf00f 100644 --- a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF diff --git a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar b/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar Binary files differdeleted file mode 100644 index 84781c4802e..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar b/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar Binary files differdeleted file mode 100644 index f4f7dd4e127..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok.jar b/config-model/src/test/cfg/application/validation/testjars/ok.jar Binary files differdeleted file mode 100644 index fce043c6ff7..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/ok.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..53773b8b1cc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: ok +Bundle-SymbolicName: ok +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar Binary files differdeleted file mode 100644 index a395a52d17d..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..21c58490c13 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: vespa container maven plugin +Built-By: tonyv +Build-Jdk: 1.6.0_26 +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/jrt-5.1-SNAPSHOT.jar +Bundle-Version: 5.1.0.SNAPSHOT +Bundle-Name: container maven plugin test +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: TestBundle + diff --git a/config-model/src/test/cfg/application/validation/testjars/test.jar b/config-model/src/test/cfg/application/validation/testjars/test.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/test.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar Binary files differdeleted file mode 100644 index 31266f1e8f2..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar +++ /dev/null diff --git a/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg new file mode 100644 index 00000000000..2dcc2cde532 --- /dev/null +++ b/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg @@ -0,0 +1,16 @@ +cluster{mobile}.defaultRoute "mobile-direct" +cluster{mobile}.route[0].name "mobile-index" +cluster{mobile}.route[0].messageType 100004 +cluster{mobile}.route[1].name "mobile-index" +cluster{mobile}.route[1].messageType 100005 +cluster{mobile}.route[2].name "mobile-index" +cluster{mobile}.route[2].messageType 100006 +cluster{mobile}.selector "(mobile)" +cluster{music}.defaultRoute "music-direct" +cluster{music}.route[0].name "music-index" +cluster{music}.route[0].messageType 100004 +cluster{music}.route[1].name "music-index" +cluster{music}.route[1].messageType 100005 +cluster{music}.route[2].name "music-index" +cluster{music}.route[2].messageType 100006 +cluster{music}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg b/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg index 657b05d63b7..fce2a6ef2ac 100755 --- a/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg +++ b/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg @@ -1,6 +1,6 @@ -route[0].name "content/mobile" -route[0].selector "mobile or mobile_search" +route[0].name "mobile" +route[0].selector "(mobile)" route[0].feed "" -route[1].name "content/music" -route[1].selector "music or music_search" +route[1].name "music" +route[1].selector "(music)" route[1].feed "" diff --git a/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg b/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg index 3860bf71431..bff785cdf04 100755 --- a/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg +++ b/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg @@ -1,44 +1,34 @@ routingtable[0].protocol "document" -routingtable[0].hop[0].name "docproc/cluster.mobile.indexing/chain.mobile.indexing" -routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing]" +routingtable[0].hop[0].name "docproc/cluster.mobile.indexing/chain.indexing" +routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.mobile.indexing;session=chain.indexing]" routingtable[0].hop[0].ignoreresult false -routingtable[0].hop[1].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[1].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.music.indexing]" +routingtable[0].hop[1].name "docproc/cluster.music.indexing/chain.indexing" +routingtable[0].hop[1].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.indexing]" routingtable[0].hop[1].ignoreresult false routingtable[0].hop[2].name "indexing" routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].recipient[0] "content/mobile" -routingtable[0].hop[2].recipient[1] "content/music" +routingtable[0].hop[2].recipient[0] "mobile" +routingtable[0].hop[2].recipient[1] "music" routingtable[0].hop[2].ignoreresult false -routingtable[0].route[0].name "content/mobile" -routingtable[0].route[0].hop[0] "[MessageType:content/mobile]" -routingtable[0].route[1].name "content/mobile-direct" -routingtable[0].route[1].hop[0] "[Content:cluster=mobile]" -routingtable[0].route[2].name "content/mobile-index" -routingtable[0].route[2].hop[0] "docproc/cluster.mobile.indexing/chain.mobile.indexing" -routingtable[0].route[2].hop[1] "[Content:cluster=mobile]" -routingtable[0].route[3].name "content/music" -routingtable[0].route[3].hop[0] "[MessageType:content/music]" -routingtable[0].route[4].name "content/music-direct" -routingtable[0].route[4].hop[0] "[Content:cluster=music]" -routingtable[0].route[5].name "content/music-index" -routingtable[0].route[5].hop[0] "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].route[5].hop[1] "[Content:cluster=music]" -routingtable[0].route[6].name "default" -routingtable[0].route[6].hop[0] "indexing" -routingtable[0].route[7].name "mobile" -routingtable[0].route[7].hop[0] "route:content/mobile" -routingtable[0].route[8].name "mobile-direct" -routingtable[0].route[8].hop[0] "route:content/mobile-direct" -routingtable[0].route[9].name "mobile-index" -routingtable[0].route[9].hop[0] "route:content/mobile-index" -routingtable[0].route[10].name "music" -routingtable[0].route[10].hop[0] "route:content/music" -routingtable[0].route[11].name "music-direct" -routingtable[0].route[11].hop[0] "route:content/music-direct" -routingtable[0].route[12].name "music-index" -routingtable[0].route[12].hop[0] "route:content/music-index" -routingtable[0].route[13].name "storage/cluster.mobile" -routingtable[0].route[13].hop[0] "route:content/mobile" -routingtable[0].route[14].name "storage/cluster.music" -routingtable[0].route[14].hop[0] "route:content/music" +routingtable[0].route[0].name "default" +routingtable[0].route[0].hop[0] "indexing" +routingtable[0].route[1].name "default-get" +routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[2].name "mobile" +routingtable[0].route[2].hop[0] "[MessageType:mobile]" +routingtable[0].route[3].name "mobile-direct" +routingtable[0].route[3].hop[0] "[Content:cluster=mobile]" +routingtable[0].route[4].name "mobile-index" +routingtable[0].route[4].hop[0] "docproc/cluster.mobile.indexing/chain.indexing" +routingtable[0].route[4].hop[1] "[Content:cluster=mobile]" +routingtable[0].route[5].name "music" +routingtable[0].route[5].hop[0] "[MessageType:music]" +routingtable[0].route[6].name "music-direct" +routingtable[0].route[6].hop[0] "[Content:cluster=music]" +routingtable[0].route[7].name "music-index" +routingtable[0].route[7].hop[0] "docproc/cluster.music.indexing/chain.indexing" +routingtable[0].route[7].hop[1] "[Content:cluster=music]" +routingtable[0].route[8].name "storage/cluster.mobile" +routingtable[0].route[8].hop[0] "route:mobile" +routingtable[0].route[9].name "storage/cluster.music" +routingtable[0].route[9].hop[0] "route:music" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg new file mode 100644 index 00000000000..438969400fe --- /dev/null +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg @@ -0,0 +1,8 @@ +cluster{c}.defaultRoute "c-direct" +cluster{c}.route[0].name "c-index" +cluster{c}.route[0].messageType 100004 +cluster{c}.route[1].name "c-index" +cluster{c}.route[1].messageType 100005 +cluster{c}.route[2].name "c-index" +cluster{c}.route[2].messageType 100006 +cluster{c}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg index 7e7a546235c..041c6ac08d9 100755 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg @@ -1,3 +1,3 @@ -route[0].name "content/music" -route[0].selector "music or music_search" +route[0].name "c" +route[0].selector "(music)" route[0].feed "" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg index e1c87a4274d..b56b2316440 100755 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg @@ -1,25 +1,21 @@ routingtable[0].protocol "document" -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.music.indexing]" +routingtable[0].hop[0].name "docproc/cluster.c.indexing/chain.indexing" +routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.c.indexing;session=chain.indexing]" routingtable[0].hop[0].ignoreresult false routingtable[0].hop[1].name "indexing" routingtable[0].hop[1].selector "[DocumentRouteSelector]" -routingtable[0].hop[1].recipient[0] "content/music" +routingtable[0].hop[1].recipient[0] "c" routingtable[0].hop[1].ignoreresult false -routingtable[0].route[0].name "content/music" -routingtable[0].route[0].hop[0] "[MessageType:content/music]" -routingtable[0].route[1].name "content/music-direct" -routingtable[0].route[1].hop[0] "[Content:cluster=music]" -routingtable[0].route[2].name "content/music-index" -routingtable[0].route[2].hop[0] "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].route[2].hop[1] "[Content:cluster=music]" +routingtable[0].route[0].name "c" +routingtable[0].route[0].hop[0] "[MessageType:c]" +routingtable[0].route[1].name "c-direct" +routingtable[0].route[1].hop[0] "[Content:cluster=c]" +routingtable[0].route[2].name "c-index" +routingtable[0].route[2].hop[0] "docproc/cluster.c.indexing/chain.indexing" +routingtable[0].route[2].hop[1] "[Content:cluster=c]" routingtable[0].route[3].name "default" routingtable[0].route[3].hop[0] "indexing" -routingtable[0].route[4].name "music" -routingtable[0].route[4].hop[0] "route:content/music" -routingtable[0].route[5].name "music-direct" -routingtable[0].route[5].hop[0] "route:content/music-direct" -routingtable[0].route[6].name "music-index" -routingtable[0].route[6].hop[0] "route:content/music-index" -routingtable[0].route[7].name "storage/cluster.music" -routingtable[0].route[7].hop[0] "route:content/music" +routingtable[0].route[4].name "default-get" +routingtable[0].route[4].hop[0] "[Content:cluster=c]" +routingtable[0].route[5].name "storage/cluster.c" +routingtable[0].route[5].hop[0] "route:c" diff --git a/config-model/src/test/derived/advanced/advanced.sd b/config-model/src/test/derived/advanced/advanced.sd index b03db73d45d..928e74160b9 100644 --- a/config-model/src/test/derived/advanced/advanced.sd +++ b/config-model/src/test/derived/advanced/advanced.sd @@ -99,6 +99,7 @@ search advanced { fieldset titleabstract { fields: title } + raw-as-base64-in-summary : true fieldset default { fields: title } diff --git a/config-model/src/test/derived/attributerank/attributerank.sd b/config-model/src/test/derived/attributerank/attributerank.sd index 86f96e4817c..4989e1795cd 100644 --- a/config-model/src/test/derived/attributerank/attributerank.sd +++ b/config-model/src/test/derived/attributerank/attributerank.sd @@ -38,4 +38,5 @@ search attributerank { rank-type singledouble: identity rank-type singlestring: identity } + } diff --git a/config-model/src/test/derived/complex/complex.sd b/config-model/src/test/derived/complex/complex.sd index fffa9b02212..844d09a529d 100644 --- a/config-model/src/test/derived/complex/complex.sd +++ b/config-model/src/test/derived/complex/complex.sd @@ -140,6 +140,7 @@ search complex { fieldset special { fields: special1, special2, special3 } + raw-as-base64-in-summary : false fieldset all { fields: combineda, combinedb } diff --git a/config-model/src/test/derived/indexschema/vsmfields.cfg b/config-model/src/test/derived/indexschema/vsmfields.cfg index 9dcffd30313..31db622183e 100644 --- a/config-model/src/test/derived/indexschema/vsmfields.cfg +++ b/config-model/src/test/derived/indexschema/vsmfields.cfg @@ -20,6 +20,11 @@ fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "" fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX +fieldspec[].name "pos" +fieldspec[].searchmethod GEOPOS +fieldspec[].arg1 "" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype INDEX fieldspec[].name "se" fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "word" @@ -124,6 +129,8 @@ documenttype[].index[].name "sc" documenttype[].index[].field[].name "sc" documenttype[].index[].name "sd" documenttype[].index[].field[].name "sd" +documenttype[].index[].name "pos" +documenttype[].index[].field[].name "pos" documenttype[].index[].name "se" documenttype[].index[].field[].name "se" documenttype[].index[].name "sf" diff --git a/config-model/src/test/derived/multiplesummaries/attributes.cfg b/config-model/src/test/derived/multiplesummaries/attributes.cfg index d5db9a8fe9d..44c94c73781 100644 --- a/config-model/src/test/derived/multiplesummaries/attributes.cfg +++ b/config-model/src/test/derived/multiplesummaries/attributes.cfg @@ -1,15 +1,160 @@ -attribute[a].collectiontype SINGLE -attribute[a].datatype STRING -attribute[a].name "a" -attribute[abolded].collectiontype SINGLE -attribute[abolded].datatype STRING -attribute[abolded].name "abolded" -attribute[adynamic].collectiontype SINGLE -attribute[adynamic].datatype STRING -attribute[adynamic].name "adynamic" -attribute[c].collectiontype SINGLE -attribute[c].datatype STRING -attribute[c].name "c" -attribute[loc_pos_zcurve].collectiontype SINGLE -attribute[loc_pos_zcurve].datatype INT64 -attribute[loc_pos_zcurve].name "loc_pos_zcurve" +attribute[].name "a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "adynamic" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "abolded" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "c" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "loc_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index fbb546179e1..5434b0770f7 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,13 +1,27 @@ -ilscript[multiplesummaries].doctype "multiplesummaries" -ilscript[multiplesummaries].name "multiplesummaries" -ilscript[multiplesummaries].content[] "clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a }" -ilscript[multiplesummaries].content[] "clear_state | guard { input adynamic | tokenize normalize stem:\"SHORTEST\" | summary adynamic | attribute adynamic }" -ilscript[multiplesummaries].content[] "clear_state | guard { input abolded | tokenize normalize stem:\"SHORTEST\" | summary abolded | attribute abolded }" -ilscript[multiplesummaries].content[] "clear_state | guard { input b | summary b }" -ilscript[multiplesummaries].content[] "clear_state | guard { input c | summary c | attribute c }" -ilscript[multiplesummaries].content[] "clear_state | guard { input d | tokenize normalize stem:\"SHORTEST\" | summary d }" -ilscript[multiplesummaries].content[] "clear_state | guard { input e | tokenize normalize stem:\"SHORTEST\" | summary dynamice }" -ilscript[multiplesummaries].content[] "clear_state | guard { input f | summary f }" -ilscript[multiplesummaries].content[] "clear_state | guard { input g | summary g }" -ilscript[multiplesummaries].content[] "clear_state | guard { input h | summary h }" -ilscript[multiplesummaries].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve }" +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "multiplesummaries" +ilscript[].docfield[0] "a" +ilscript[].docfield[1] "adynamic" +ilscript[].docfield[2] "abolded" +ilscript[].docfield[3] "b" +ilscript[].docfield[4] "c" +ilscript[].docfield[5] "d" +ilscript[].docfield[6] "e" +ilscript[].docfield[7] "f" +ilscript[].docfield[8] "g" +ilscript[].docfield[9] "h" +ilscript[].docfield[10] "loc" +ilscript[].docfield[11] "mytags" +ilscript[].content[0] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" +ilscript[].content[1] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" +ilscript[].content[2] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" +ilscript[].content[3] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" +ilscript[].content[4] "clear_state | guard { input b | summary anotherb | summary b; }" +ilscript[].content[5] "clear_state | guard { input c | summary c | attribute c; }" +ilscript[].content[6] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" +ilscript[].content[7] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" +ilscript[].content[8] "clear_state | guard { input f | summary f; }" +ilscript[].content[9] "clear_state | guard { input g | summary g; }" +ilscript[].content[10] "clear_state | guard { input h | summary h; }" +ilscript[].content[11] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index ec2d3f06e46..d5002535761 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -1,59 +1,145 @@ -indexinfo[multiplesummaries].name "multiplesummaries" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "sddocname" -indexinfo[multiplesummaries].command[].command "word" -indexinfo[multiplesummaries].command[].indexname "sddocname" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "a" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "a" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "b" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "c" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "c" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "d" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "e" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "f" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "g" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "h" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc" -indexinfo[multiplesummaries].command[].command "default-position" -indexinfo[multiplesummaries].command[].indexname "loc_pos" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc_pos" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc_pos_zcurve" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "loc_pos_zcurve" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "d" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "adynamic2" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "abolded2" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "aboldeddynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "aboldeddynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "dynamice" +indexinfo[].name "multiplesummaries" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "b" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "b" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "e" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "e" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "type Array<int>" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "type WeightedSet<string>" +indexinfo[].command[].indexname "loc" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "default-position" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "type position" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "type long" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg index 2151e5a59c4..9b5c6a5a7ba 100644 --- a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg +++ b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg @@ -1,16 +1,43 @@ +length 256 +max_matches 3 +min_length 128 prefix true -override[abolded].fieldname "abolded" -override[abolded].length 65536 -override[abolded].max_matches 1 -override[abolded].min_length 8192 -override[abolded].surround_max 65536 -override[abolded2].fieldname "abolded2" -override[abolded2].length 65536 -override[abolded2].max_matches 1 -override[abolded2].min_length 8192 -override[abolded2].surround_max 65536 -override[d].fieldname "d" -override[d].length 65536 -override[d].max_matches 1 -override[d].min_length 8192 -override[d].surround_max 65536 +surround_max 128 +winsize 200 +winsize_fallback_multiplier 10.0 +max_match_candidates 1000 +stem_min_length 5 +stem_max_extend 3 +override[].fieldname "d" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 +override[].fieldname "abolded2" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 +override[].fieldname "abolded" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index f4e131ee0bb..ae0e2fe92bc 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -67,6 +67,10 @@ search multiplesummaries { field loc type string { } + + field mytags type array<string> { + indexing: index + } } field loc_pos type position { @@ -120,7 +124,7 @@ search multiplesummaries { } # Since a here is a dynamic summary, it will be fetched from disk - document-summary notattributesonly2 { + document-summary anothernotattributesonly2 { summary adynamic2 type string { # Should still be dynamic here source: a @@ -130,6 +134,19 @@ search multiplesummaries { summary c type string { } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } + summary anothera type string { + source: a + } + summary anotherb type string { + source: b + } } # Not attributes only because d is bolded @@ -153,6 +170,7 @@ search multiplesummaries { } summary loc_position type long { + source: loc_pos_zcurve } } @@ -166,7 +184,7 @@ search multiplesummaries { source: a } - summary loc_position type long { + summary loc_pos_zcurve type long { } } diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index 16d0024155b..1c8fc47878b 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -1,174 +1,200 @@ -defaultsummaryid 235127765 +defaultsummaryid 456145241 usev8geopositions false -classes[1156201411].id 1156201411 -classes[1156201411].name "attributeprefetch" -classes[1156201411].fields[a].name "a" -classes[1156201411].fields[a].type "longstring" -classes[1156201411].fields[c].name "c" -classes[1156201411].fields[c].type "longstring" -classes[1156201411].fields[loc_pos_zcurve].name "loc_pos_zcurve" -classes[1156201411].fields[loc_pos_zcurve].type "int64" -classes[1156201411].fields[rankfeatures].name "rankfeatures" -classes[1156201411].fields[rankfeatures].type "longstring" -classes[1156201411].fields[summaryfeatures].name "summaryfeatures" -classes[1156201411].fields[summaryfeatures].type "longstring" -classes[1277791169].id 1277791169 -classes[1277791169].name "notattributesonly2" -classes[1277791169].fields[adynamic2].name "adynamic2" -classes[1277791169].fields[adynamic2].type "longstring" -classes[1277791169].fields[c].name "c" -classes[1277791169].fields[c].type "longstring" -classes[1277791169].fields[rankfeatures].name "rankfeatures" -classes[1277791169].fields[rankfeatures].type "longstring" -classes[1277791169].fields[summaryfeatures].name "summaryfeatures" -classes[1277791169].fields[summaryfeatures].type "longstring" -classes[1280967808].id 1280967808 -classes[1280967808].name "attributesonly2" -classes[1280967808].fields[anotdynamic].name "anotdynamic" -classes[1280967808].fields[anotdynamic].type "longstring" -classes[1280967808].fields[c].name "c" -classes[1280967808].fields[c].type "longstring" -classes[1280967808].fields[loc_position].name "loc_position" -classes[1280967808].fields[loc_position].type "int64" -classes[1280967808].fields[rankfeatures].name "rankfeatures" -classes[1280967808].fields[rankfeatures].type "longstring" -classes[1280967808].fields[summaryfeatures].name "summaryfeatures" -classes[1280967808].fields[summaryfeatures].type "longstring" -classes[1334083320].id 1334083320 -classes[1334083320].name "third" -classes[1334083320].fields[a].name "a" -classes[1334083320].fields[a].type "longstring" -classes[1334083320].fields[adynamic].name "adynamic" -classes[1334083320].fields[adynamic].type "longstring" -classes[1334083320].fields[d].name "d" -classes[1334083320].fields[d].type "longstring" -classes[1334083320].fields[e].name "e" -classes[1334083320].fields[e].type "longstring" -classes[1334083320].fields[f].name "f" -classes[1334083320].fields[f].type "jsonstring" -classes[1334083320].fields[g].name "g" -classes[1334083320].fields[g].type "jsonstring" -classes[1334083320].fields[h].name "h" -classes[1334083320].fields[h].type "jsonstring" -classes[1334083320].fields[rankfeatures].name "rankfeatures" -classes[1334083320].fields[rankfeatures].type "longstring" -classes[1334083320].fields[summaryfeatures].name "summaryfeatures" -classes[1334083320].fields[summaryfeatures].type "longstring" -classes[1439192258].id 1439192258 -classes[1439192258].name "second" -classes[1439192258].fields[a].name "a" -classes[1439192258].fields[a].type "longstring" -classes[1439192258].fields[adynamic].name "adynamic" -classes[1439192258].fields[adynamic].type "longstring" -classes[1439192258].fields[c].name "c" -classes[1439192258].fields[c].type "longstring" -classes[1439192258].fields[f].name "f" -classes[1439192258].fields[f].type "jsonstring" -classes[1439192258].fields[rankfeatures].name "rankfeatures" -classes[1439192258].fields[rankfeatures].type "longstring" -classes[1439192258].fields[summaryfeatures].name "summaryfeatures" -classes[1439192258].fields[summaryfeatures].type "longstring" -classes[1653275739].id 1653275739 -classes[1653275739].name "attributesonly3" -classes[1653275739].fields[a].name "a" -classes[1653275739].fields[a].type "longstring" -classes[1653275739].fields[anotbolded].name "anotbolded" -classes[1653275739].fields[anotbolded].type "longstring" -classes[1653275739].fields[loc_position].name "loc_position" -classes[1653275739].fields[loc_position].type "int64" -classes[1653275739].fields[rankfeatures].name "rankfeatures" -classes[1653275739].fields[rankfeatures].type "longstring" -classes[1653275739].fields[summaryfeatures].name "summaryfeatures" -classes[1653275739].fields[summaryfeatures].type "longstring" -classes[182001096].id 182001096 -classes[182001096].name "notattributesonly1" -classes[182001096].fields[adynamic].name "adynamic" -classes[182001096].fields[adynamic].type "longstring" -classes[182001096].fields[c].name "c" -classes[182001096].fields[c].type "longstring" -classes[182001096].fields[rankfeatures].name "rankfeatures" -classes[182001096].fields[rankfeatures].type "longstring" -classes[182001096].fields[summaryfeatures].name "summaryfeatures" -classes[182001096].fields[summaryfeatures].type "longstring" -classes[1881063334].id 1881063334 -classes[1881063334].name "notattributesonly4" -classes[1881063334].fields[abolded2].name "abolded2" -classes[1881063334].fields[abolded2].type "longstring" -classes[1881063334].fields[c].name "c" -classes[1881063334].fields[c].type "longstring" -classes[1881063334].fields[rankfeatures].name "rankfeatures" -classes[1881063334].fields[rankfeatures].type "longstring" -classes[1881063334].fields[summaryfeatures].name "summaryfeatures" -classes[1881063334].fields[summaryfeatures].type "longstring" -classes[1988966242].id 1988966242 -classes[1988966242].name "attributesonly1" -classes[1988966242].fields[a].name "a" -classes[1988966242].fields[a].type "longstring" -classes[1988966242].fields[c].name "c" -classes[1988966242].fields[c].type "longstring" -classes[1988966242].fields[rankfeatures].name "rankfeatures" -classes[1988966242].fields[rankfeatures].type "longstring" -classes[1988966242].fields[summaryfeatures].name "summaryfeatures" -classes[1988966242].fields[summaryfeatures].type "longstring" -classes[235127765].id 235127765 -classes[235127765].name "multiplesummaries" -classes[235127765].fields[a].name "a" -classes[235127765].fields[a].type "longstring" -classes[235127765].fields[abolded].name "abolded" -classes[235127765].fields[abolded].type "longstring" -classes[235127765].fields[abolded2].name "abolded2" -classes[235127765].fields[abolded2].type "longstring" -classes[235127765].fields[aboldeddynamic].name "aboldeddynamic" -classes[235127765].fields[aboldeddynamic].type "longstring" -classes[235127765].fields[adynamic].name "adynamic" -classes[235127765].fields[adynamic].type "longstring" -classes[235127765].fields[adynamic2].name "adynamic2" -classes[235127765].fields[adynamic2].type "longstring" -classes[235127765].fields[b].name "b" -classes[235127765].fields[b].type "longstring" -classes[235127765].fields[c].name "c" -classes[235127765].fields[c].type "longstring" -classes[235127765].fields[d].name "d" -classes[235127765].fields[d].type "longstring" -classes[235127765].fields[documentid].name "documentid" -classes[235127765].fields[documentid].type "longstring" -classes[235127765].fields[dynamice].name "dynamice" -classes[235127765].fields[dynamice].type "longstring" -classes[235127765].fields[e].name "e" -classes[235127765].fields[e].type "longstring" -classes[235127765].fields[f].name "f" -classes[235127765].fields[f].type "jsonstring" -classes[235127765].fields[g].name "g" -classes[235127765].fields[g].type "jsonstring" -classes[235127765].fields[h].name "h" -classes[235127765].fields[h].type "jsonstring" -classes[235127765].fields[loc_pos.distance].name "loc_pos.distance" -classes[235127765].fields[loc_pos.distance].type "integer" -classes[235127765].fields[loc_pos.position].name "loc_pos.position" -classes[235127765].fields[loc_pos.position].type "xmlstring" -classes[235127765].fields[loc_position].name "loc_position" -classes[235127765].fields[loc_position].type "int64" -classes[235127765].fields[rankfeatures].name "rankfeatures" -classes[235127765].fields[rankfeatures].type "longstring" -classes[235127765].fields[summaryfeatures].name "summaryfeatures" -classes[235127765].fields[summaryfeatures].type "longstring" -classes[803323247].id 803323247 -classes[803323247].name "notattributesonly3" -classes[803323247].fields[a].name "a" -classes[803323247].fields[a].type "longstring" -classes[803323247].fields[d].name "d" -classes[803323247].fields[d].type "longstring" -classes[803323247].fields[rankfeatures].name "rankfeatures" -classes[803323247].fields[rankfeatures].type "longstring" -classes[803323247].fields[summaryfeatures].name "summaryfeatures" -classes[803323247].fields[summaryfeatures].type "longstring" -classes[937467944].id 937467944 -classes[937467944].name "notattributesonly5" -classes[937467944].fields[aboldeddynamic].name "aboldeddynamic" -classes[937467944].fields[aboldeddynamic].type "longstring" -classes[937467944].fields[c].name "c" -classes[937467944].fields[c].type "longstring" -classes[937467944].fields[rankfeatures].name "rankfeatures" -classes[937467944].fields[rankfeatures].type "longstring" -classes[937467944].fields[summaryfeatures].name "summaryfeatures" -classes[937467944].fields[summaryfeatures].type "longstring" +classes[].id 456145241 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "loc_pos" +classes[].fields[].type "jsonstring" +classes[].fields[].name "loc_pos.position" +classes[].fields[].type "xmlstring" +classes[].fields[].name "loc_pos.distance" +classes[].fields[].type "integer" +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "abolded" +classes[].fields[].type "longstring" +classes[].fields[].name "b" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "dynamice" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "g" +classes[].fields[].type "jsonstring" +classes[].fields[].name "h" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "e" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic2" +classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" +classes[].fields[].name "abolded2" +classes[].fields[].type "longstring" +classes[].fields[].name "aboldeddynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 783153771 +classes[].name "third" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "e" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "g" +classes[].fields[].type "jsonstring" +classes[].fields[].name "h" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 815922035 +classes[].name "attributesonly1" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1308077923 +classes[].name "notattributesonly1" +classes[].omitsummaryfeatures false +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1609068631 +classes[].name "anothernotattributesonly2" +classes[].omitsummaryfeatures false +classes[].fields[].name "adynamic2" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anothera" +classes[].fields[].type "longstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 686755772 +classes[].name "notattributesonly3" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1711750363 +classes[].name "attributesonly2" +classes[].omitsummaryfeatures false +classes[].fields[].name "anotdynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "loc_position" +classes[].fields[].type "int64" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1510953467 +classes[].name "attributesonly3" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "anotbolded" +classes[].fields[].type "longstring" +classes[].fields[].name "loc_pos_zcurve" +classes[].fields[].type "int64" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 923824943 +classes[].name "notattributesonly4" +classes[].omitsummaryfeatures false +classes[].fields[].name "abolded2" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 552611075 +classes[].name "notattributesonly5" +classes[].omitsummaryfeatures false +classes[].fields[].name "aboldeddynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 146047714 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "loc_pos_zcurve" +classes[].fields[].type "int64" +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 324773027 +classes[].name "second" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/multiplesummaries/summarymap.cfg b/config-model/src/test/derived/multiplesummaries/summarymap.cfg index 94c86ea8c80..94adc250c54 100644 --- a/config-model/src/test/derived/multiplesummaries/summarymap.cfg +++ b/config-model/src/test/derived/multiplesummaries/summarymap.cfg @@ -1,49 +1,61 @@ defaultoutputclass -1 -override[a].arguments "a" -override[a].command "attribute" -override[a].field "a" -override[abolded].arguments "abolded" -override[abolded].command "dynamicteaser" -override[abolded].field "abolded" -override[abolded2].arguments "abolded2" -override[abolded2].command "dynamicteaser" -override[abolded2].field "abolded2" -override[aboldeddynamic].arguments "aboldeddynamic" -override[aboldeddynamic].command "dynamicteaser" -override[aboldeddynamic].field "aboldeddynamic" -override[adynamic].arguments "adynamic" -override[adynamic].command "dynamicteaser" -override[adynamic].field "adynamic" -override[adynamic2].arguments "adynamic2" -override[adynamic2].command "dynamicteaser" -override[adynamic2].field "adynamic2" -override[anotbolded].arguments "a" -override[anotbolded].command "attribute" -override[anotbolded].field "anotbolded" -override[anotdynamic].arguments "adynamic" -override[anotdynamic].command "attribute" -override[anotdynamic].field "anotdynamic" -override[c].arguments "c" -override[c].command "attribute" -override[c].field "c" -override[d].arguments "d" -override[d].command "dynamicteaser" -override[d].field "d" -override[dynamice].arguments "dynamice" -override[dynamice].command "dynamicteaser" -override[dynamice].field "dynamice" -override[loc_pos.distance].arguments "loc_pos_zcurve" -override[loc_pos.distance].command "absdist" -override[loc_pos.distance].field "loc_pos.distance" -override[loc_pos.position].arguments "loc_pos_zcurve" -override[loc_pos.position].command "positions" -override[loc_pos.position].field "loc_pos.position" -override[loc_pos_zcurve].arguments "loc_pos_zcurve" -override[loc_pos_zcurve].command "attribute" -override[loc_pos_zcurve].field "loc_pos_zcurve" -override[rankfeatures].arguments "" -override[rankfeatures].command "rankfeatures" -override[rankfeatures].field "rankfeatures" -override[summaryfeatures].arguments "" -override[summaryfeatures].command "summaryfeatures" -override[summaryfeatures].field "summaryfeatures" +override[].field "a" +override[].command "attribute" +override[].arguments "a" +override[].field "adynamic" +override[].command "dynamicteaser" +override[].arguments "adynamic" +override[].field "d" +override[].command "dynamicteaser" +override[].arguments "d" +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" +override[].field "c" +override[].command "attribute" +override[].arguments "c" +override[].field "adynamic2" +override[].command "dynamicteaser" +override[].arguments "adynamic2" +override[].field "sometags" +override[].command "matchedelementsfilter" +override[].arguments "mytags" +override[].field "anothera" +override[].command "attribute" +override[].arguments "a" +override[].field "anotdynamic" +override[].command "attribute" +override[].arguments "adynamic" +override[].field "loc_position" +override[].command "attribute" +override[].arguments "loc_pos_zcurve" +override[].field "anotbolded" +override[].command "attribute" +override[].arguments "a" +override[].field "loc_pos_zcurve" +override[].command "attribute" +override[].arguments "loc_pos_zcurve" +override[].field "abolded2" +override[].command "dynamicteaser" +override[].arguments "abolded2" +override[].field "aboldeddynamic" +override[].command "dynamicteaser" +override[].arguments "aboldeddynamic" +override[].field "loc_pos" +override[].command "geopos" +override[].arguments "loc_pos_zcurve" +override[].field "loc_pos.position" +override[].command "positions" +override[].arguments "loc_pos_zcurve" +override[].field "loc_pos.distance" +override[].command "absdist" +override[].arguments "loc_pos_zcurve" +override[].field "abolded" +override[].command "dynamicteaser" +override[].arguments "abolded" +override[].field "dynamice" +override[].command "dynamicteaser" +override[].arguments "dynamice" diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index cd7c295ab11..2c46031bdad 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -1,12 +1,8 @@ -defaultsummaryid 1727020212 -usev8geopositions false -classes[].id 1727020212 +defaultsummaryid 1151071433 +usev8geopositions true +classes[].id 1151071433 classes[].name "default" classes[].omitsummaryfeatures false -classes[].fields[].name "pos.position" -classes[].fields[].type "xmlstring" -classes[].fields[].name "pos.distance" -classes[].fields[].type "integer" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/position_nosummary/summarymap.cfg b/config-model/src/test/derived/position_nosummary/summarymap.cfg index cc1c14a6614..0a36f98c6ca 100644 --- a/config-model/src/test/derived/position_nosummary/summarymap.cfg +++ b/config-model/src/test/derived/position_nosummary/summarymap.cfg @@ -1,10 +1,4 @@ defaultoutputclass -1 -override[].field "pos.position" -override[].command "positions" -override[].arguments "pos_zcurve" -override[].field "pos.distance" -override[].command "absdist" -override[].arguments "pos_zcurve" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index 7b8bf16287f..7fda1ca0c05 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -1,14 +1,10 @@ -defaultsummaryid 230670304 -usev8geopositions false -classes[].id 230670304 +defaultsummaryid 644855547 +usev8geopositions true +classes[].id 644855547 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "pos" classes[].fields[].type "jsonstring" -classes[].fields[].name "pos.position" -classes[].fields[].type "xmlstring" -classes[].fields[].name "pos.distance" -classes[].fields[].type "integer" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/position_summary/summarymap.cfg b/config-model/src/test/derived/position_summary/summarymap.cfg index de48a19b6db..2223d3380a0 100644 --- a/config-model/src/test/derived/position_summary/summarymap.cfg +++ b/config-model/src/test/derived/position_summary/summarymap.cfg @@ -2,12 +2,6 @@ defaultoutputclass -1 override[].field "pos" override[].command "geopos" override[].arguments "pos_zcurve" -override[].field "pos.position" -override[].command "positions" -override[].arguments "pos_zcurve" -override[].field "pos.distance" -override[].command "absdist" -override[].arguments "pos_zcurve" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" diff --git a/config-model/src/test/derived/position_summary/vsmsummary.cfg b/config-model/src/test/derived/position_summary/vsmsummary.cfg index d7d45782843..56405069131 100644 --- a/config-model/src/test/derived/position_summary/vsmsummary.cfg +++ b/config-model/src/test/derived/position_summary/vsmsummary.cfg @@ -2,12 +2,6 @@ outputclass "" fieldmap[].summary "pos" fieldmap[].document[].field "pos" fieldmap[].command NONE -fieldmap[].summary "pos.position" -fieldmap[].document[].field "pos_zcurve" -fieldmap[].command NONE -fieldmap[].summary "pos.distance" -fieldmap[].document[].field "pos_zcurve" -fieldmap[].command NONE fieldmap[].summary "rankfeatures" fieldmap[].command NONE fieldmap[].summary "summaryfeatures" diff --git a/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg new file mode 100644 index 00000000000..3af23fd3770 --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg @@ -0,0 +1,48 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].name "in_schema0" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" +rankprofile[].name "in_schema1" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "in_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "rankingExpression(f2).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + rankingExpression(fo2)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(f2)" +rankprofile[].name "in_schema3" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "rankingExpression(f2).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + rankingExpression(fo2)" +rankprofile[].fef.property[].name "rankingExpression(fo1).rankingScript" +rankprofile[].fef.property[].value "now" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(f2)" +rankprofile[].name "outside_schema1" +rankprofile[].fef.property[].name "rankingExpression(fo1).rankingScript" +rankprofile[].fef.property[].value "now" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "outside_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" diff --git a/config-model/src/test/derived/rankprofilemodularity/test.sd b/config-model/src/test/derived/rankprofilemodularity/test.sd new file mode 100644 index 00000000000..34414414d6c --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test.sd @@ -0,0 +1,49 @@ +schema test { + + document test { + + field title type string { + indexing: index + } + + } + + rank-profile default inherits outside_schema2 { + } + + rank-profile in_schema0 inherits outside_schema2 { + } + + rank-profile in_schema1 { + + first-phase { + expression: nativeRank + } + + } + + rank-profile in_schema2 inherits outside_schema2 { + + function f2() { + expression: fieldMatch(title) + fo2 + } + + first-phase { + expression: f2 + } + + } + + rank-profile in_schema3 inherits outside_schema1, outside_schema2 { + + function f2() { + expression: fieldMatch(title) + fo2 + } + + first-phase { + expression: f2 + } + + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile new file mode 100644 index 00000000000..a8092689b7f --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile @@ -0,0 +1,7 @@ +rank-profile outside_schema1 inherits in_schema1 { + + function fo1() { + expression: now + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile new file mode 100644 index 00000000000..8cf3fcfbb78 --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile @@ -0,0 +1,11 @@ +rank-profile outside_schema2 { + + function fo2() { + expression: random + } + + first-phase { + expression: fieldMatch(title).completeness + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/renamedfeatures/foo.sd b/config-model/src/test/derived/renamedfeatures/foo.sd index f7884dd937d..462eb0da44e 100644 --- a/config-model/src/test/derived/renamedfeatures/foo.sd +++ b/config-model/src/test/derived/renamedfeatures/foo.sd @@ -1,6 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search foo { - + document foo { field title type string { indexing: summary | index diff --git a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg index b78866fdf05..0a327575a48 100644 --- a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg +++ b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg @@ -79,4 +79,4 @@ rankprofile[3].fef.property[10].value "rankingExpression(mymul)" rankprofile[3].fef.property[11].name "vespa.feature.rename" rankprofile[3].fef.property[11].value "mymul" rankprofile[3].fef.property[12].name "vespa.type.attribute.t1" -rankprofile[3].fef.property[12].value "tensor(m{},v[3])"
\ No newline at end of file +rankprofile[3].fef.property[12].value "tensor(m{},v[3])" diff --git a/config-model/src/test/derived/slice/query-profiles/default.xml b/config-model/src/test/derived/slice/query-profiles/default.xml new file mode 100644 index 00000000000..2535ca895ed --- /dev/null +++ b/config-model/src/test/derived/slice/query-profiles/default.xml @@ -0,0 +1,3 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default" type="DefaultQueryProfileType"> +</query-profile> diff --git a/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml new file mode 100644 index 00000000000..50970d8743f --- /dev/null +++ b/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml @@ -0,0 +1,4 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="DefaultQueryProfileType" inherits="native"> + <field name="ranking.features.query(myTensor)" type="tensor<float>(key{})" /> +</query-profile-type> diff --git a/config-model/src/test/derived/slice/rank-profiles.cfg b/config-model/src/test/derived/slice/rank-profiles.cfg new file mode 100644 index 00000000000..75725b81ecf --- /dev/null +++ b/config-model/src/test/derived/slice/rank-profiles.cfg @@ -0,0 +1,25 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" +rankprofile[].name "parent" +rankprofile[].fef.property[].name "rankingExpression(mySlice@77dee0712164ce73).rankingScript" +rankprofile[].fef.property[].value "query(myTensor){key:MY_KEY2}" +rankprofile[].fef.property[].name "rankingExpression(myFunction).rankingScript" +rankprofile[].fef.property[].value "4 * query(myTensor){key:MY_KEY1} * rankingExpression(mySlice@77dee0712164ce73)" +rankprofile[].fef.property[].name "rankingExpression(myValue).rankingScript" +rankprofile[].fef.property[].value "4" +rankprofile[].fef.property[].name "rankingExpression(mySlice).rankingScript" +rankprofile[].fef.property[].value "myTensor{key:MY_KEY2}" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" diff --git a/config-model/src/test/derived/slice/test.sd b/config-model/src/test/derived/slice/test.sd new file mode 100644 index 00000000000..c2060300785 --- /dev/null +++ b/config-model/src/test/derived/slice/test.sd @@ -0,0 +1,23 @@ +search test { + + document test { + } + + rank-profile parent { + + function inline myFunction() { + expression: myValue * query(myTensor){MY_KEY1} * mySlice(query(myTensor)) + } + + function inline myValue() { + expression: 4 + } + + function inline mySlice(myTensor) { + # TODO: We are missing type resolving across function calls in serialization, + # so using the short form (without 'key') here will fail + expression: myTensor{key:MY_KEY2} + } + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/twostreamingstructs/whatever.sd b/config-model/src/test/derived/twostreamingstructs/whatever.sd index 34b0261326f..17ac30102a8 100644 --- a/config-model/src/test/derived/twostreamingstructs/whatever.sd +++ b/config-model/src/test/derived/twostreamingstructs/whatever.sd @@ -9,7 +9,6 @@ search whatever { field f1 type pair { indexing: summary - body } } diff --git a/config-model/src/test/examples/header_body.sd b/config-model/src/test/examples/header_body.sd deleted file mode 100644 index 61ddbaa7238..00000000000 --- a/config-model/src/test/examples/header_body.sd +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Search definition with (ignored) header and body statements -# TODO: Remove in Vespa 8 -search header_body { - - document header_body { - - field title type string { - indexing: summary | index - header - } - - field description type string { - indexing: summary | index - body - } - } -} diff --git a/config-model/src/test/examples/nextgen/summaryfield.sd b/config-model/src/test/examples/nextgen/summaryfield.sd index 9b3cc6862b9..99c73d1be53 100644 --- a/config-model/src/test/examples/nextgen/summaryfield.sd +++ b/config-model/src/test/examples/nextgen/summaryfield.sd @@ -5,11 +5,21 @@ search summaryfield { indexing: index | summary summary bar: full } + field mytags type array<string> { + indexing: index + } } document-summary baz { summary cox type string { source: bar } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } } } diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 0ccbec529ed..3acc44113d0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -18,7 +18,6 @@ import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.NamedSchema; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -55,19 +54,18 @@ public class ApplicationDeployTest { public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); new VespaModel(tester.app()); - List<NamedSchema> schemas = tester.getSchemas(); + List<Schema> schemas = tester.getSchemas(); assertEquals(schemas.size(), 5); - for (NamedSchema searchDefinition : schemas) { - Schema s = searchDefinition.getSearch(); - switch (s.getName()) { + for (Schema schema : schemas) { + switch (schema.getName()) { case "music": case "laptop": case "pc": case "sock": break; case "product": - assertTrue(s instanceof DocumentOnlySchema); - assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType()); + assertTrue(schema instanceof DocumentOnlySchema); + assertEquals(DataType.STRING, schema.getDocument().getField("title").getDataType()); break; default: fail(); @@ -95,8 +93,8 @@ public class ApplicationDeployTest { // Check that getFilename works ArrayList<String> sdFileNames = new ArrayList<>(); - for (NamedSchema sd : schemas) - sdFileNames.add(sd.getFilename()); + for (Schema schema : schemas) + sdFileNames.add(schema.getName() + ApplicationPackage.SD_NAME_SUFFIX); Collections.sort(sdFileNames); assertEquals("laptop.sd", sdFileNames.get(0)); assertEquals("music.sd", sdFileNames.get(1)); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 3d799375420..96186150108 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.searchdefinition.Schema; import java.io.File; import java.io.IOException; @@ -39,7 +39,7 @@ public class ApplicationPackageTester { public FilesApplicationPackage app() { return applicationPackage; } - public List<NamedSchema> getSchemas() { + public List<Schema> getSchemas() { return new DeployState.Builder().applicationPackage(app()).build().getSchemas(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 10f883bdc75..ba70b7493a2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -2054,7 +2054,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, false, 0, Optional.of(model), new DeployState.Builder()); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-10-04", "node-1-3-10-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); @@ -2067,6 +2067,14 @@ public class ModelProvisioningTest { 4, true), config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, ZookeeperServerConfig.Server::joining))); + assertEquals("Retired nodes are retired", + Map.of(0, false, + 1, true, + 2, true, + 3, false, + 4, false), + config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, + ZookeeperServerConfig.Server::retired))); } } diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index f1bebdb0a29..d4ebbe62507 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -5,12 +5,10 @@ import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; -import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; @@ -122,8 +120,8 @@ public class SDDocumentTypeTestCase extends AbstractSchemaTestCase { " }" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); - builder.importString(schemaLines); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); + builder.addSchema(schemaLines); builder.build(true); var application = builder.application(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java index aaf75f0c852..2ed1ab6892a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java @@ -22,7 +22,7 @@ public class ArraysTestCase extends AbstractSchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/arrays.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arrays.sd"); SDField tags = (SDField) schema.getDocument().getField("tags"); assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java index 1dd2c987d90..9c70fcf79f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertTrue; public class ArraysWeightedSetsTestCase extends AbstractSchemaTestCase { @Test public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); SDField tags = (SDField) schema.getDocument().getField("tags"); assertTrue(tags.getDataType() instanceof ArrayDataType); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index e4de0aef622..7b2ed2d7a7f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -30,7 +30,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void testAttributeSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); SDField f1=(SDField) schema.getDocument().getField("f1"); assertEquals(1, f1.getAttributes().size()); @@ -91,22 +91,22 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatFastAccessCanBeSet() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); SDField field = (SDField) schema.getDocument().getField("fast_access"); assertEquals(1, field.getAttributes().size()); Attribute attr = field.getAttributes().get(field.getName()); assertTrue(attr.isFastAccess()); } - private Schema getSearch(String sd) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(sd); - builder.build(); + private Schema getSchema(String sd) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(sd); + builder.build(true); return builder.getSchema(); } private Attribute getAttributeF(String sd) throws ParseException { - Schema schema = getSearch(sd); + Schema schema = getSchema(sd); SDField field = (SDField) schema.getDocument().getField("f"); return field.getAttributes().get(field.getName()); } @@ -153,14 +153,14 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatMutableCanNotbeSetInDocument() throws ParseException { try { - getSearch("search test {\n" + - " document test {\n" + - " field f type int {\n" + - " indexing: attribute\n" + - " attribute: mutable\n" + - " }\n" + - " }\n" + - "}\n"); + getSchema("search test {\n" + + " document test {\n" + + " field f type int {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " }\n" + + "}\n"); fail(); } catch (IllegalArgumentException e) { assertEquals("Field 'f' in 'test' can not be marked mutable as it is inside the document clause.", e.getMessage()); @@ -185,7 +185,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { } private Schema getSearchWithMutables() throws ParseException { - return getSearch( + return getSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java index 8b7f0885907..3e92c1df9e7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -18,7 +18,7 @@ public class CommentTestCase extends AbstractSchemaTestCase { @Test public void testComments() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/comment.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/comment.sd"); SDField field = schema.getConcreteField("a"); assertEquals("{ input a | tokenize normalize stem:\"BEST\" | summary a | index a; }", field.getIndexingScript().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java index 7c5c5c6fccc..ca7313a5c3f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java @@ -15,8 +15,8 @@ public class DiversityTestCase { @Test public void testDiversity() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + @@ -42,7 +42,7 @@ public class DiversityTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); @@ -60,10 +60,10 @@ public class DiversityTestCase { } @Test public void requireSingleNumericOrString() throws ParseException { - SchemaBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); + ApplicationBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); @@ -72,19 +72,19 @@ public class DiversityTestCase { @Test public void requireSingle() throws ParseException { - SchemaBuilder builder = getSearchBuilder("field b type array<int> { indexing: attribute }"); + ApplicationBuilder builder = getSearchBuilder("field b type array<int> { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private SchemaBuilder getSearchBuilder(String diversity) throws ParseException { + private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index fb552f5cfb0..81a44261daf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -143,7 +144,7 @@ public class DocumentGraphValidatorTest { } private static Schema createSearchWithName(String name, Schema... parents) { - Schema campaignSchema = new Schema(name); + Schema campaignSchema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); campaignSchema.addDocument(document); document.setDocumentReferences(new DocumentReferences(Collections.emptyMap())); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index cc45a2aaec0..66f1850bd10 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; @@ -32,7 +33,7 @@ public class DocumentReferenceResolverTest { @Test public void reference_from_one_document_to_another_is_resolved() { // Create bar document with no fields - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); barSchema.addDocument(barDocument); @@ -41,7 +42,7 @@ public class DocumentReferenceResolverTest { ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); @@ -64,7 +65,7 @@ public class DocumentReferenceResolverTest { SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); AttributeUtils.addAttributeAspect(fooRefToBarField); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); @@ -80,14 +81,14 @@ public class DocumentReferenceResolverTest { @Test public void throws_exception_if_reference_is_not_an_attribute() { // Create bar document with no fields - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDDocumentType barDocument = new SDDocumentType("bar", barSchema); barSchema.addDocument(barDocument); // Create foo document with document reference to bar SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java index 4057fc469bc..7d2386030da 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -19,7 +20,7 @@ public class ImportedFieldsEnumeratorTest { @Test public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { String PARENT = "parent"; - Schema parentSchema = new Schema(PARENT); + Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); var parentField = new SDField("their_field", DataType.INT); AttributeUtils.addAttributeAspect(parentField); @@ -27,7 +28,7 @@ public class ImportedFieldsEnumeratorTest { parentSchema.addDocument(parentDocument); String FOO = "foo"; - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDField fooRefToParent = new SDField( "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToParent); @@ -38,7 +39,7 @@ public class ImportedFieldsEnumeratorTest { fooSchema.addDocument(fooDocument); String BAR = "bar"; - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDField barRefToParent = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(barRefToParent); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 621e7061203..23be59e38f5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -21,9 +21,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends AbstractSchemaTes public void testIncorrectRef() throws IOException, ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", - registry, - new QueryProfileRegistry()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", + registry, + new QueryProfileRegistry()); new DerivedConfiguration(schema, registry); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java index 55593ef64e4..d81d07bf6e2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -15,7 +15,7 @@ public class IncorrectSummaryTypesTestCase extends AbstractSchemaTestCase { @Test public void testImportingIncorrect() throws ParseException { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search incorrectsummarytypes {\n" + " document incorrectsummarytypes {\n" + " field somestring type string {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java index f2e50439306..198f8b6c79e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java @@ -21,7 +21,7 @@ public class IndexSettingsTestCase extends AbstractSchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/indexsettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/indexsettings.sd"); SDField usingDefault=(SDField) schema.getDocument().getField("usingdefault"); assertEquals(Stemming.SHORTEST,usingDefault.getStemming(schema)); @@ -38,7 +38,7 @@ public class IndexSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatInterlavedFeaturesAreSetOnExtraField() throws ParseException { - SchemaBuilder builder = SchemaBuilder.createFromString(joinLines( + ApplicationBuilder builder = ApplicationBuilder.createFromString(joinLines( "search test {", " document test {", " field content type string {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java index 60728ab7d99..8ba7dfdb14f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -15,13 +15,13 @@ public class IndexingParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/indexing.sd")); + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/indexing.sd")); } @Test public void requireThatParseExceptionPositionIsCorrect() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); } catch (ParseException e) { if (!e.getMessage().contains("at line 5, column 57.")) { throw e; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java index 41321b5c81e..0fcc6e09b02 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -14,6 +14,6 @@ import java.io.IOException; public class MultipleSummariesTestCase extends AbstractSchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index fc94bbe8731..1e766b7793d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -19,7 +19,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testNameField() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search simple {\n" + " document name-check {\n" + " field title type string {\n" + @@ -40,7 +40,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testDuplicateNamesInSearchDifferentType() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search duplicatenamesinsearch {\n" + " document {\n" + " field grpphotoids64 type string { }\n" + @@ -60,7 +60,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testDuplicateNamesInDoc() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search duplicatenamesindoc {\n" + " document {\n" + " field foo type int {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java index 5663473481b..6563fb8d3f1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java @@ -17,7 +17,7 @@ public class OutsideTestCase extends AbstractSchemaTestCase { @Test public void testOutsideIndex() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); Index defaultIndex= schema.getIndex("default"); assertTrue(defaultIndex.isPrefix()); @@ -26,7 +26,7 @@ public class OutsideTestCase extends AbstractSchemaTestCase { @Test public void testOutsideSummary() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java index 456a236f293..2b49f3d67a6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java @@ -68,7 +68,7 @@ public class PredicateDataTypeTestCase { lowerBoundParameter(lowerBound) + upperBoundParameter(upperBound)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -95,7 +95,7 @@ public class PredicateDataTypeTestCase { "lower-bound: -100000000000000000L\n" + // +'L' upperBoundParameter(upperBound)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -113,7 +113,7 @@ public class PredicateDataTypeTestCase { predicateFieldSd( attributeFieldSd( arityParameter(2)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -131,7 +131,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Missing arity value in predicate field."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); fail(); } @@ -141,7 +141,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -150,7 +150,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @@ -160,7 +160,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Invalid arity value in predicate field, must be greater than 1."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -169,7 +169,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Arity parameter is used only for predicate type fields."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -181,7 +181,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Parameters lower-bound and upper-bound are used only for predicate type fields."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -193,7 +193,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Collections of predicates are not allowed."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index d2486395340..d07f4513c3c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -33,7 +34,7 @@ public class RankProfileRegistryTest { @Test(expected = IllegalArgumentException.class) public void testRankProfileDuplicateNameIsIllegal() { - Schema schema = new Schema("foo"); + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry, schema.rankingConstants()); rankProfileRegistry.add(barRankProfile); @@ -42,7 +43,7 @@ public class RankProfileRegistryTest { @Test public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() { - Schema schema = new Schema("foo"); + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 3fa129eae39..d77c6d822e3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; @@ -43,7 +44,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void testRankProfileInheritance() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); SDField a = document.addField("a", DataType.STRING); @@ -51,7 +52,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { document.addField("b", DataType.STRING); schema.addDocument(document); RankProfile child = new RankProfile("child", schema, rankProfileRegistry, schema.rankingConstants()); - child.setInherited("default"); + child.inherit("default"); rankProfileRegistry.add(child); Iterator<RankProfile.RankSetting> i = child.rankSettingIterator(); @@ -69,8 +70,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatIllegalInheritanceIsChecked() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 inherits notexist {}", @@ -78,7 +79,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail(); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'p1' inherits 'notexist', but it does not exist anywhere in the inheritance of search 'test'.", e.getMessage()); + assertEquals("rank-profile 'p1' inherits 'notexist', but this is not found in schema 'test'", e.getMessage()); } } @@ -86,8 +87,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatSelfInheritanceIsIllegal() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema test {", " document test { } ", " rank-profile self inherits self {}", @@ -102,13 +103,13 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatSelfInheritanceIsLegalWhenOverloading() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema base {", " document base { } ", " rank-profile self inherits default {}", "}")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema test {", " document test inherits base { } ", " rank-profile self inherits self {}", @@ -119,8 +120,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatSidewaysInheritanceIsImpossible() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema child1 {", " document child1 {", " field field1 type int {", @@ -141,7 +142,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " }", " }", "}\n")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema child2 {", " document child2 {", " field field1 type int {", @@ -165,15 +166,15 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail("Sideways inheritance should have been enforced"); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'child' inherits 'parent', but it does not exist anywhere in the inheritance of search 'child1'.", e.getMessage()); + assertEquals("rank-profile 'child' inherits 'parent', but this is not found in schema 'child1'", e.getMessage()); } } @Test - public void requireThatDefaultCanAlwaysBeInherited() throws ParseException { + public void requireThatDefaultInheritingDefaultIsIgnored() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema test {", " document test { } ", " rank-profile default inherits default {}", @@ -185,8 +186,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatCyclicInheritanceIsIllegal() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile a inherits b {}", @@ -204,8 +205,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatRankProfilesCanInheritNotYetSeenProfiles() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 inherits not_yet_defined {}", @@ -246,9 +247,9 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { private void verifyTermwiseLimitAndSomeMoreIncludingInheritance(ModelContext.Properties deployProperties, String sd, Double termwiseLimit) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(sd); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(sd); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); verifyRankProfile(rankProfileRegistry.get(schema, "parent"), attributeFields, deployProperties, termwiseLimit); @@ -279,8 +280,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatConfigIsDerivedForAttributeTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10]) { indexing: attribute }", @@ -290,7 +291,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " rank-profile p1 {}", " rank-profile p2 {}", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); assertEquals(4, registry.all().size()); @@ -303,14 +304,14 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatDenseDimensionsMustBeBound() throws ParseException { try { - SchemaBuilder builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[]) { indexing: attribute }", " }", "}")); - builder.build(); + builder.build(true); } catch (IllegalArgumentException e) { assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", @@ -332,8 +333,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 {}", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index d6065e74af5..6de890549c1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -23,8 +23,8 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { @Test public void testRankPropertyInheritance() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test {", " field a type string { ", @@ -48,7 +48,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); @@ -80,8 +80,8 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { @Test public void testRankProfileMutate() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test {", " field a type int { ", @@ -128,7 +128,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); RankProfile a = rankProfileRegistry.get(schema, "a"); List<RankProfile.MutateOperation> operations = a.getMutateOperations(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java index 1925b7e77b9..6eb74dede62 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java @@ -28,8 +28,8 @@ public class RankingConstantTest { final String TENSOR_FILE = "path/my-tensor-file.json"; final String TENSOR_TYPE = "tensor(x{})"; RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " rank-profile my_rank_profile {", @@ -43,7 +43,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); Iterator<RankingConstant> constantIterator = schema.rankingConstants().asMap().values().iterator(); @@ -59,10 +59,10 @@ public class RankingConstantTest { @Test public void tensor_constant_must_have_a_type() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a type"); - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -75,10 +75,10 @@ public class RankingConstantTest { @Test public void tensor_constant_must_have_a_file() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a file"); - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -91,8 +91,8 @@ public class RankingConstantTest { @Test public void constant_file_does_not_need_path_or_ending() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -101,7 +101,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals("simplename", constant.getFileName()); @@ -110,8 +110,8 @@ public class RankingConstantTest { @Test public void constant_uri_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -120,7 +120,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -130,8 +130,8 @@ public class RankingConstantTest { @Test public void constant_https_uri_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -140,7 +140,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -150,8 +150,8 @@ public class RankingConstantTest { @Test public void constant_uri_with_port_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -160,7 +160,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -170,8 +170,8 @@ public class RankingConstantTest { @Test public void constant_uri_no_dual_slashes_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -180,7 +180,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -190,12 +190,12 @@ public class RankingConstantTest { @Test public void constant_uri_only_supports_http_and_https() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); String expectedMessage = "Encountered \" <IDENTIFIER> \"ftp\"\" at line 5, column 10.\n\n" + "Was expecting:\n\n" + "<URI_PATH> ..."; try { - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 87b406c39fa..4646ff2f394 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -25,8 +25,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -66,7 +66,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(queryProfileRegistry, new ImportedMlModels()); assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); @@ -91,8 +91,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNameCollision() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -110,7 +110,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); try { rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -125,8 +125,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNegativeLiteralArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -141,7 +141,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -150,8 +150,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNegativeConstantArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -169,7 +169,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -180,8 +180,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -193,7 +193,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 1.0E8", @@ -203,8 +203,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void test3() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field rating_yelp type int {" + @@ -219,7 +219,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 0598fee538a..94d25deb16a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -28,8 +28,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type double { \n" + @@ -65,7 +65,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -79,8 +79,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -125,7 +125,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -150,8 +150,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testNonTopLevelInlining() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type double { \n" + @@ -181,7 +181,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -194,13 +194,13 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { public void testFunctionInliningWithReplacement() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); MockDeployLogger deployLogger = new MockDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - deployLogger, - new TestProperties(), - rankProfileRegistry, - new QueryProfileRegistry()); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + deployLogger, + new TestProperties(), + rankProfileRegistry, + new QueryProfileRegistry()); + builder.addSchema( "search test {\n" + " document test { }\n" + " rank-profile test {\n" + @@ -215,7 +215,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index 689b75690ae..ff8a9df4076 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -16,8 +16,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testSelfLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -36,7 +36,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -48,8 +48,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testNestedLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -71,7 +71,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -83,8 +83,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testSelfArgumentLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -106,7 +106,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -118,8 +118,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testNoLoopWithSameLocalArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -140,14 +140,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithMultipleInvocations() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -168,14 +168,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithBoundIdentifiers() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -191,14 +191,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithTheSameNestedIdentifierWhichIsUnbound() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -214,14 +214,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithTheSameAlternatingNestedIdentifierWhichIsUnbound() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -240,7 +240,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index ff146fedd88..8354eb59416 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -26,8 +26,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -45,7 +45,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -61,8 +61,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -86,7 +86,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -113,8 +113,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -132,7 +132,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -153,8 +153,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(input[1])"); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfiles); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -193,7 +193,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " file: ignored.json\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, queryProfiles, s).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 299724ea39e..ee3b4ab25e0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -32,20 +32,20 @@ public class RankingExpressionValidationTestCase extends AbstractSchemaTestCase } private Schema importWithExpression(String expression, RankProfileRegistry registry) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(registry); - builder.importString("search test {" + - " document test { " + - " field a type string { " + - " indexing: index " + - " }" + - " }" + - " rank-profile default {" + - " first-phase {" + - " expression: " + expression + - " }" + - " }" + - "}"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema("search test {" + + " document test { " + + " field a type string { " + + " indexing: index " + + " }" + + " }" + + " rank-profile default {" + + " first-phase {" + + " expression: " + expression + + " }" + + " }" + + "}"); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java index a6f76897c0b..a8aaee83938 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java @@ -15,7 +15,7 @@ public class ReservedWordsAsFieldNamesTestCase extends AbstractSchemaTestCase { @Test public void testIt() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); assertNotNull(schema.getDocument().getField("inline")); assertNotNull(schema.getDocument().getField("constants")); assertNotNull(schema.getDocument().getField("reference")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java index c2816336076..9c895b452d5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java @@ -18,6 +18,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Iterator; +import static com.google.common.collect.testing.Helpers.assertEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -36,9 +37,9 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { @SuppressWarnings("deprecation") public void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder sb = new SchemaBuilder(rankProfileRegistry, new QueryProfileRegistry()); - sb.importFile("src/test/examples/simple.sd"); - sb.build(); + ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); + sb.addSchemaFile("src/test/examples/simple.sd"); + sb.build(true); Schema schema = sb.getSchema(); assertEquals("simple", schema.getName()); assertTrue(schema.hasDocument()); @@ -53,14 +54,14 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { new MakeAliases(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field - field=(SDField) document.getField("title"); - assertEquals(DataType.STRING,field.getDataType()); + field = (SDField) document.getField("title"); + assertEquals(DataType.STRING, field.getDataType()); assertEquals("{ input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", field.getIndexingScript().toString()); assertFalse(schema.getIndex("default").isPrefix()); assertTrue(schema.getIndex("title").isPrefix()); - Iterator<String> titleAliases= schema.getIndex("title").aliasIterator(); - assertEquals("aliaz",titleAliases.next()); - assertEquals("analias.totitle",titleAliases.next()); + Iterator<String> titleAliases = schema.getIndex("title").aliasIterator(); + assertEquals("aliaz", titleAliases.next()); + assertEquals("analias.totitle", titleAliases.next()); assertEquals("analias.todefault", schema.getIndex("default").aliasIterator().next()); assertEquals(RankType.IDENTITY, field.getRankType()); @@ -70,7 +71,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertTrue(field.isHeader()); // Second field - field=(SDField) document.getField("description"); + field = (SDField) document.getField("description"); assertEquals(RankType.ABOUT, field.getRankType()); assertEquals(SummaryTransform.NONE, field.getSummaryField("description").getTransform()); @@ -81,30 +82,30 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals("hallo", schema.getIndex("description").aliasIterator().next()); // Third field - field=(SDField) document.getField("chatter"); + field = (SDField) document.getField("chatter"); assertEquals(RankType.ABOUT, field.getRankType()); assertNull(field.getStemming()); assertTrue(field.getNormalizing().doRemoveAccents()); // Fourth field - field=(SDField) document.getField("category"); + field = (SDField) document.getField("category"); assertEquals(0, field.getAttributes().size()); assertEquals(Stemming.NONE, field.getStemming()); assertFalse(field.getNormalizing().doRemoveAccents()); // Fifth field - field=(SDField) document.getField("popularity"); + field = (SDField) document.getField("popularity"); assertEquals("{ input popularity | attribute popularity; }", field.getIndexingScript().toString()); // Sixth field - field=(SDField) document.getField("measurement"); - assertEquals(DataType.INT,field.getDataType()); + field = (SDField) document.getField("measurement"); + assertEquals(DataType.INT, field.getDataType()); assertEquals(RankType.EMPTY, field.getRankType()); assertEquals(1, field.getAttributes().size()); // Seventh field - field= schema.getConcreteField("categories"); + field = schema.getConcreteField("categories"); assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", field.getIndexingScript().toString()); assertTrue(field.isHeader()); @@ -139,27 +140,27 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); // Rank Profiles - RankProfile profile=rankProfileRegistry.get(schema, "default"); + RankProfile profile = rankProfileRegistry.get(schema, "default"); assertNotNull(profile); - assertNull(profile.getInheritedName()); + assertEmpty(profile.inheritedNames()); assertNull(profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE)); assertEquals(RankType.EMPTY, profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - profile=rankProfileRegistry.get(schema, "experimental"); + profile = rankProfileRegistry.get(schema, "experimental"); assertNotNull(profile); - assertEquals("default",profile.getInheritedName()); + assertEquals("default", profile.inheritedNames().get(0)); assertEquals(RankType.IDENTITY, profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - profile=rankProfileRegistry.get(schema, "other"); + profile = rankProfileRegistry.get(schema, "other"); assertNotNull(profile); - assertEquals("experimental",profile.getInheritedName()); + assertEquals("experimental", profile.inheritedNames().get(0)); // The extra-document field - SDField exact= schema.getConcreteField("exact"); - assertNotNull("Extra field was parsed",exact); - assertEquals("exact",exact.getName()); - assertEquals(Stemming.NONE,exact.getStemming()); + SDField exact = schema.getConcreteField("exact"); + assertNotNull("Extra field was parsed", exact); + assertEquals("exact", exact.getName()); + assertEquals(Stemming.NONE, exact.getStemming()); assertFalse(exact.getNormalizing().doRemoveAccents()); assertEquals("{ input title . \" \" . input category | tokenize | summary exact | index exact; }", exact.getIndexingScript().toString()); @@ -170,7 +171,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { public void testDocumentImporting() throws IOException, ParseException { try { // Having two documents in one sd-file is illegal. - SchemaBuilder.buildFromFile("src/test/examples/documents.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/documents.sd"); fail(); } catch (IllegalArgumentException e) { } @@ -178,11 +179,11 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { @Test public void testIdImporting() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/strange.sd"); - SDField idecidemyide=(SDField) schema.getDocument().getField("idecidemyide"); - assertEquals(5,idecidemyide.getId()); - SDField sodoi=(SDField) schema.getDocument().getField("sodoi"); - assertEquals(7,sodoi.getId()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/strange.sd"); + SDField idecidemyide = (SDField)schema.getDocument().getField("idecidemyide"); + assertEquals(5, idecidemyide.getId()); + SDField sodoi = (SDField) schema.getDocument().getField("sodoi"); + assertEquals(7, sodoi.getId()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java index dfe60c5871e..6d619b76c18 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java @@ -19,13 +19,13 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/simple.sd")); + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); } @Test public void requireThatParseExceptionPositionIsCorrect() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); } catch (ParseException e) { if ( ! e.getMessage().contains("at line 5, column 36.")) { throw e; @@ -36,7 +36,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatParserHandlesLexicalError() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); } catch (ParseException e) { if (!e.getMessage().contains("at line 7, column 27.")) { throw e; @@ -47,7 +47,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireErrorWhenJunkAfterSearchBlock() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); fail("Illegal junk at end of SD passed"); } catch (ParseException e) { if (!e.getMessage().contains("at line 10, column 1")) { @@ -59,7 +59,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireErrorWhenMissingClosingSearchBracket() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); fail("SD without closing bracket passed"); } catch (ParseException e) { if (!e.getMessage().contains("Encountered \"<EOF>\" at line 8, column 1")) { @@ -71,7 +71,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void illegalSearchDefinitionName() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid-name.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid-name.sd"); fail("Name with dash passed"); } catch (ParseException e) { if ( ! e.getMessage().contains("invalid-name")) { @@ -80,10 +80,4 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { } } - // TODO: Remove in Vespa 8 - @Test - public void requireThatParserHandlesHeadAndBody() throws IOException, ParseException { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/header_body.sd")); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index d906685d502..112dbd2587e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -31,10 +31,10 @@ public class SchemaTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, schema); + ApplicationBuilder.createFromStrings(logger, schema); assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", logger.entries.get(0).message); - fail("Expected failure"); + fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", e.getMessage()); @@ -60,7 +60,8 @@ public class SchemaTestCase { " }" + " }" + "}"); - SchemaBuilder.createFromStrings(new DeployLoggerStub(), parent, child); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), parent, child); + fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("schema 'child' inherits 'parent', " + @@ -166,12 +167,12 @@ public class SchemaTestCase { " import field parentschema_ref.name as child2_imported {}" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.importString(parentLines); - builder.importString(child1Lines); - builder.importString(child2Lines); + builder.addSchema(parentLines); + builder.addSchema(child1Lines); + builder.addSchema(child2Lines); builder.build(true); var application = builder.application(); @@ -187,7 +188,7 @@ public class SchemaTestCase { assertNotNull(child1.getExtraField("child1_field")); assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").getInheritedName()); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").inheritedNames().get(0)); assertNotNull(child1.rankingConstants().get("parent_constant")); assertNotNull(child1.rankingConstants().get("child1_constant")); assertTrue(child1.rankingConstants().asMap().containsKey("parent_constant")); @@ -222,7 +223,7 @@ public class SchemaTestCase { assertNotNull(child2.getExtraField("child2_field")); assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").getInheritedName()); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").inheritedNames().get(0)); assertNotNull(child2.rankingConstants().get("parent_constant")); assertNotNull(child2.rankingConstants().get("child2_constant")); assertTrue(child2.rankingConstants().asMap().containsKey("parent_constant")); @@ -307,12 +308,12 @@ public class SchemaTestCase { " }" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.importString(parentLines); - builder.importString(childLines); - builder.importString(grandchildLines); + builder.addSchema(parentLines); + builder.addSchema(childLines); + builder.addSchema(grandchildLines); builder.build(true); var application = builder.application(); @@ -320,6 +321,98 @@ public class SchemaTestCase { assertInheritedFromParent(application.schemas().get("grandchild"), application, builder.getRankProfileRegistry()); } + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed1() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Only one of the profiles inherited by rank profile 'r3' can contain first-phase expression, but it is present in all of [rank profile 'r1', rank profile 'r2']", + e.getMessage()); + } + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsAllowedWhenDefinedInChild() throws ParseException { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " first-phase {" + // Redefined here so this does not cause failure + " expression: nativeRank" + + " }" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed2() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("rank profile 'r3' inherits rank profile 'r2' which contains function 'f1', but this function is already defined in another profile this inherits", + e.getMessage()); + } + } + private void assertInheritedFromParent(Schema schema, Application application, RankProfileRegistry rankProfileRegistry) { assertEquals("pf1", schema.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); assertEquals(Stemming.NONE, schema.getStemming()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java index 75dcd6bc209..aec258f2df0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -20,7 +20,7 @@ public class StemmingSettingTestCase extends AbstractSchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); SDField artist = (SDField) schema.getDocument().getField("artist"); assertEquals(Stemming.SHORTEST, artist.getStemming(schema)); @@ -43,7 +43,7 @@ public class StemmingSettingTestCase extends AbstractSchemaTestCase { @Test public void requireThatStemmingIsDefaultBest() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); assertNull(schema.getConcreteField("my_str").getStemming()); assertEquals(Stemming.BEST, schema.getConcreteField("my_str").getStemming(schema)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index 34093bf72ef..c27dd9dfdfb 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -25,7 +25,7 @@ public class StructTestCase extends AbstractSchemaTestCase { @Test public void testBadStruct() throws IOException { try { - SchemaBuilder.buildFromFile("src/test/examples/badstruct.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/badstruct.sd"); fail("Should throw exception."); } catch (ParseException expected) { // success @@ -46,7 +46,7 @@ public class StructTestCase extends AbstractSchemaTestCase { */ @Test(expected = IllegalArgumentException.class) public void testStructOutsideDocumentIllegal() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 7c8e806d768..086e3485962 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -39,7 +39,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -61,7 +61,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertEquals(1, logger.entries.size()); assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("summary field 'foo2' in document summary 'foobar' references source field 'ondisk', " + @@ -89,7 +89,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -120,7 +120,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -156,7 +156,7 @@ public class SummaryTestCase { " }", "}"); var logger = new DeployLoggerStub(); - var search = SchemaBuilder.createFromString(sd, logger).getSchema(); + var search = ApplicationBuilder.createFromString(sd, logger).getSchema(); assertEquals(List.of(), logger.entries); var titleField = "title"; @@ -208,7 +208,7 @@ public class SummaryTestCase { "}"); var logger = new DeployLoggerStub(); try { - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("For schema 'music', summary class 'title2', summary field 'title': Can not use " + @@ -228,7 +228,7 @@ public class SummaryTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, schema); + ApplicationBuilder.createFromStrings(logger, schema); assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", logger.entries.get(0).message); // fail("Expected failure"); @@ -264,7 +264,7 @@ public class SummaryTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, parent, child); + ApplicationBuilder.createFromStrings(logger, parent, child); logger.entries.forEach(e -> System.out.println(e)); //assertTrue(logger.entries.isEmpty()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java index 9fdeb9ece1d..f2d78796553 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java @@ -15,14 +15,14 @@ public class UrlFieldValidationTestCase { @Test public void requireThatInheritedRiseFieldsStillCanBeInConflictButDontThrowException() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString("search test {" + - " document test { " + - " field a type uri { indexing: attribute | summary }" + - " }" + - "}"); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search test {" + + " document test { " + + " field a type uri { indexing: attribute | summary }" + + " }" + + "}"); try { - builder.build(); + builder.build(true); fail("Should have caused an exception"); // success } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 8b54455d176..456efdb08ae 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; @@ -38,14 +38,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { toDir.mkdirs(); deleteContent(toDir); - SchemaBuilder builder = SchemaBuilder.createFromDirectory(searchDefRoot + dirName + "/", new MockFileRegistry(), logger, properties); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory(searchDefRoot + dirName + "/", new MockFileRegistry(), logger, properties); return derive(dirName, searchDefinitionName, properties, builder, logger); } private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties, - SchemaBuilder builder, + ApplicationBuilder builder, DeployLogger logger) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(searchDefinitionName), logger, @@ -57,14 +57,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { return export(dirName, builder, config); } - DerivedConfiguration derive(String dirName, SchemaBuilder builder, Schema schema) throws IOException { + DerivedConfiguration derive(String dirName, ApplicationBuilder builder, Schema schema) throws IOException { DerivedConfiguration config = new DerivedConfiguration(schema, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry()); return export(dirName, builder, config); } - private DerivedConfiguration export(String name, SchemaBuilder builder, DerivedConfiguration config) throws IOException { + private DerivedConfiguration export(String name, ApplicationBuilder builder, DerivedConfiguration config) throws IOException { String path = exportConfig(name, config); DerivedConfiguration.exportDocuments(new DocumentManager().useV8DocManagerCfg(useV8DocManagerCfg()) .produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); @@ -100,6 +100,11 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { DeployLogger logger) throws IOException, ParseException { return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); } + protected DerivedConfiguration assertCorrectDeriving(String dirName, + TestProperties properties) throws IOException, ParseException + { + return assertCorrectDeriving(dirName, null, properties, new TestableDeployLogger()); + } protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, @@ -114,14 +119,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { * Asserts config is correctly derived given a builder. * This will fail if the builder contains multiple search definitions. */ - protected DerivedConfiguration assertCorrectDeriving(SchemaBuilder builder, String dirName, DeployLogger logger) throws IOException { - builder.build(); + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, String dirName, DeployLogger logger) throws IOException { + builder.build(true); DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); assertCorrectConfigFiles(dirName); return derived; } - protected DerivedConfiguration assertCorrectDeriving(SchemaBuilder builder, Schema schema, String name) throws IOException { + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, Schema schema, String name) throws IOException { DerivedConfiguration derived = derive(name, builder, schema); assertCorrectConfigFiles(name); return derived; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 58a4350b73b..a272ab14dad 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.parser.ParseException; @@ -25,7 +25,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { // Test attribute importing - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); // Test attribute deriving AttributeFields attributeFields = new AttributeFields(schema); @@ -71,7 +71,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_array_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); @@ -81,7 +81,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_map_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); @@ -101,7 +101,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { - Schema schema = SchemaBuilder.createFromString( + Schema schema = ApplicationBuilder.createFromString( joinLines("search test {", " document test {", " field pos_array type array<position> {", @@ -117,7 +117,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_map_of_primitive_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java index 06d72bb6972..ee9a9eac02c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ public class CasingTestCase extends AbstractSchemaTestCase { @Test public void testCasing() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/casing.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/casing.sd"); assertEquals(schema.getIndex("color").getName(), "color"); assertEquals(schema.getIndex("Foo").getName(), "Foo"); assertEquals(schema.getIndex("Price").getName(), "Price"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 604082fb52e..237e6c8c992 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -20,7 +21,7 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType doc = new SDDocumentType("test"); schema.addDocument(doc); @@ -30,7 +31,7 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { doc.addField(field); doc.addField(new SDField("c", DataType.STRING)); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 8cc8967269a..05f8f1a6b93 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -24,27 +24,32 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testPositionArray() throws IOException, ParseException { - assertCorrectDeriving("position_array"); + assertCorrectDeriving("position_array", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionAttribute() throws IOException, ParseException { - assertCorrectDeriving("position_attribute"); + assertCorrectDeriving("position_attribute", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionExtra() throws IOException, ParseException { - assertCorrectDeriving("position_extra"); + assertCorrectDeriving("position_extra", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionNoSummary() throws IOException, ParseException { - assertCorrectDeriving("position_nosummary"); + assertCorrectDeriving("position_nosummary", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionSummary() throws IOException, ParseException { - assertCorrectDeriving("position_summary"); + assertCorrectDeriving("position_summary", + new TestProperties().setUseV8GeoPositions(true)); } @Test @@ -152,10 +157,10 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testTensor2() throws IOException, ParseException { String dir = "src/test/derived/tensor2/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "first.sd"); - builder.importFile(dir + "second.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "first.sd"); + builder.addSchemaFile(dir + "second.sd"); + builder.build(true); derive("tensor2", builder, builder.getSchema("second")); assertCorrectConfigFiles("tensor2"); } @@ -177,4 +182,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertEquals(0, logger.warnings.size()); } + @Test + public void testRankProfileModularity() throws IOException, ParseException { + assertCorrectDeriving("rankprofilemodularity"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index b262ef92e1a..440f067dd00 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; @@ -25,7 +26,7 @@ public class IdTestCase extends AbstractExportingTestCase { @Test public void testExplicitUpperCaseIdField() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField uri = new SDField("URI", DataType.URI); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index f00072a5a19..bcf68387294 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -40,10 +41,10 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatIndexedStructFieldCanBeInherited() throws IOException, ParseException { String dir = "src/test/derived/inheritstruct/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritstruct", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritstruct"); } @@ -52,7 +53,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { public void requireThatInheritFromNullIsCaught() throws IOException, ParseException { try { assertCorrectDeriving("inheritfromnull"); - } catch (IllegalStateException e) { + } catch (IllegalArgumentException e) { assertEquals("Document type 'foo' not found", e.getMessage()); } } @@ -63,12 +64,12 @@ public class InheritanceTestCase extends AbstractExportingTestCase { List<String> files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); File outDir = tmpDir.newFolder("out"); for (int startIdx = 0; startIdx < files.size(); ++startIdx) { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { String fileName = files.get(fileIdx % files.size()); - builder.importFile(dir + fileName); + builder.addSchemaFile(dir + fileName); } - builder.build(); + builder.build(true); DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); @@ -110,10 +111,10 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructTypesAreInheritedFromParent() throws IOException, ParseException { String dir = "src/test/derived/inheritfromparent/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritfromparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromparent"); } @@ -121,11 +122,11 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructTypesAreInheritedFromGrandParent() throws IOException, ParseException { String dir = "src/test/derived/inheritfromgrandparent/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "grandparent.sd"); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritfromgrandparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromgrandparent"); } @@ -133,12 +134,12 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void testInheritance() throws IOException, ParseException { String dir = "src/test/derived/inheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "grandparent.sd"); - builder.importFile(dir + "father.sd"); - builder.importFile(dir + "mother.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "father.sd"); + builder.addSchemaFile(dir + "mother.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritance", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritance"); } @@ -146,7 +147,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void testIndexSettingInheritance() { SDDocumentType parent = new SDDocumentType("parent"); - Schema parentSchema = new Schema("parent"); + Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); parentSchema.addDocument(parent); SDField prefixed = parent.addField("prefixed", DataType.STRING); prefixed.parseIndexingScript("{ index }"); @@ -154,7 +155,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { SDDocumentType child = new SDDocumentType("child"); child.inherit(parent); - Schema childSchema = new Schema("child"); + Schema childSchema = new Schema("child", MockApplicationPackage.createEmpty()); childSchema.addDocument(child); prefixed = (SDField)child.getField("prefixed"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index f8f1bf9e4f1..1e7dd3a2405 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -2,12 +2,13 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; @@ -30,7 +31,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { */ @Test public void testLiteralBoost() { - Schema schema = new Schema("literalboost"); + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); @@ -63,7 +64,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { */ @Test public void testNonDefaultRankLiteralBoost() { - Schema schema = new Schema("literalboost"); + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); @@ -73,7 +74,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); // Check il script addition @@ -89,7 +90,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { /** Tests literal boosts in two fields going to the same index */ @Test public void testTwoLiteralBoostFields() { - Schema schema = new Schema("msb"); + Schema schema = new Schema("msb", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("msb"); schema.addDocument(document); @@ -100,7 +101,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.parseIndexingScript("{ summary | index }"); field2.setLiteralBoost(20); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java index 9ad6dfbb972..53bf55fc73f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; import java.io.IOException; @@ -16,8 +16,8 @@ public class MailTestCase extends AbstractExportingTestCase { @Test public void testMail() throws IOException, ParseException { String dir = "src/test/derived/mail/"; - SchemaBuilder sb = new SchemaBuilder(); - sb.importFile(dir + "mail.sd"); + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile(dir + "mail.sd"); assertCorrectDeriving(sb, dir, new TestableDeployLogger()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java index 51135ae1cb2..0e6d7b8442f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -14,7 +13,6 @@ import java.io.IOException; */ public class MultipleSummariesTestCase extends AbstractExportingTestCase { @Test - @Ignore public void testMultipleSummaries() throws IOException, ParseException { assertCorrectDeriving("multiplesummaries"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java index c7297c41c62..a458036a03f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,10 +19,10 @@ public class SchemaInheritanceTestCase extends AbstractExportingTestCase { @Test public void testIt() throws IOException, ParseException { try { - SchemaBuilder builder = SchemaBuilder.createFromDirectory("src/test/derived/schemainheritance/", - new MockFileRegistry(), - new TestableDeployLogger(), - new TestProperties()); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory("src/test/derived/schemainheritance/", + new MockFileRegistry(), + new TestableDeployLogger(), + new TestProperties()); derive("schemainheritance", builder, builder.getSchema("child")); assertCorrectConfigFiles("schemainheritance"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index b3f2fb62ac2..34d33a00d9e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.DocumentReference; @@ -62,7 +63,7 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { } private static Schema createSchema(String name, Map<String, Schema> schemas) { - Schema schema = new Schema(name); + Schema schema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); document.setDocumentReferences(new DocumentReferences(emptyMap())); schema.addDocument(document); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index c0f2b6887d2..d89e5f2c957 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,10 +19,10 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { String name = "emptychild"; final String expectedResultsDirName = "src/test/derived/" + name + "/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(expectedResultsDirName + "parent.sd"); - builder.importFile(expectedResultsDirName + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(expectedResultsDirName + "parent.sd"); + builder.addSchemaFile(expectedResultsDirName + "child.sd"); + builder.build(true); Schema schema = builder.getSchema("child"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java new file mode 100644 index 00000000000..e6c7efd7052 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SliceTestCase extends AbstractExportingTestCase { + + @Test + public void testSlice() throws IOException, ParseException { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("slice"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java index b2c2a54ce5e..1f6c70c9383 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; @@ -31,8 +31,8 @@ public class StructInheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructCanInherit() throws IOException, ParseException { String dir = "src/test/derived/structinheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "simple.sd"); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "simple.sd"); builder.build(false); derive("structinheritance", builder, builder.getSchema("simple")); assertCorrectConfigFiles("structinheritance"); @@ -43,9 +43,9 @@ public class StructInheritanceTestCase extends AbstractExportingTestCase { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("cannot inherit from base and redeclare field name"); String dir = "src/test/derived/structinheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "bad.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "bad.sd"); + builder.build(true); derive("structinheritance", builder, builder.getSchema("bad")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 4994cffb92a..c0fa3e8311f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.searchdefinition.*; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -29,7 +30,7 @@ import static org.junit.Assert.assertTrue; public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryMap summaryMap=new SummaryMap(schema); Iterator transforms=summaryMap.resultTransformIterator(); @@ -73,7 +74,7 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { } @Test public void testPositionDeriving() { - Schema schema = new Schema("store"); + Schema schema = new Schema("store", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("store"); schema.addDocument(document); String fieldName = "location"; @@ -147,7 +148,7 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testFailOnSummaryFieldSourceCollision() { try { - SchemaBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { assertTrue(e.getMessage().matches(".*equally named field.*")); } @@ -189,13 +190,13 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { } private Schema buildSearch(String field) throws ParseException { - var builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines("search test {", - " document test {", - field, - " }", - "}")); - builder.build(); + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + field, + " }", + "}")); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index c996fb0c1b9..9a36ef90cd7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.SummaryConfig; @@ -35,7 +35,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " }", "}"); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); } @@ -50,14 +50,14 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " }", "}"); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); } @Test public void testDeriving() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals("default", summary.getName()); @@ -134,22 +134,22 @@ public class SummaryTestCase extends AbstractSchemaTestCase { } private static Schema buildCampaignAdModel() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString("search campaign { document campaign {} }"); - builder.importString(joinLines("search ad {", - " document ad {", - " field campaign_ref type reference<campaign> {", - " indexing: summary | attribute", - " }", - " field other_campaign_ref type reference<campaign> {", - " indexing: summary | attribute", - " }", - " }", - " document-summary my_summary {", - " summary other_campaign_ref type reference<campaign> {}", - " }", - "}")); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search campaign { document campaign {} }"); + builder.addSchema(joinLines("search ad {", + " document ad {", + " field campaign_ref type reference<campaign> {", + " indexing: summary | attribute", + " }", + " field other_campaign_ref type reference<campaign> {", + " indexing: summary | attribute", + " }", + " }", + " document-summary my_summary {", + " summary other_campaign_ref type reference<campaign> {}", + " }", + "}")); + builder.build(true); return builder.getSchema("ad"); } @@ -168,7 +168,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " summary foo type string {}", " }", "}"); - var search = SchemaBuilder.createFromString(sd).getSchema(); + var search = ApplicationBuilder.createFromString(sd).getSchema(); assertOmitSummaryFeatures(true, search, "bar"); assertOmitSummaryFeatures(false, search, "baz"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java index c3a78bab441..4e629fcb4d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -13,20 +13,22 @@ import java.io.IOException; * @author arnej27959 */ public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { + @Test public void testTwoStreamingStructsExporting() throws ParseException, IOException { String root = "src/test/derived/twostreamingstructs"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(root + "/streamingstruct.sd"); - builder.importFile(root + "/whatever.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); - builder = new SchemaBuilder(); - builder.importFile(root + "/streamingstruct.sd"); - builder.importFile(root + "/whatever.sd"); - builder.build(); + builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 13289d72884..dbb32e61144 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; @@ -25,7 +26,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { /** Tests that exact-string stuff is not spilled over to the default index */ @Test public void testExactStringToStringTypeConversion() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 138992477c0..5ab5a8057e8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -24,7 +24,7 @@ public class VsmFieldsTestCase { @SuppressWarnings("deprecation") @Test public void reference_type_field_is_unsearchable() { - Schema schema = new Schema("test", new Application(MockApplicationPackage.createEmpty()), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); schema.addDocument(new SDDocumentType("test")); SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); refField.parseIndexingScript("{ summary }"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 481a4db11ec..99692e70041 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.document; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ public class ComplexAttributeFieldUtilsTestCase { private final ImmutableSDField field; FixtureBase(String fieldName, String sdContent) throws ParseException { - Schema schema = SchemaBuilder.createFromString(sdContent).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sdContent).getSchema(); field = schema.getConcreteField(fieldName); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 715b7b803e2..9ca97f4dbc7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -48,7 +48,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { } private static Schema createSearch(String documentType) { - return new Schema(documentType, new Application(MockApplicationPackage.createEmpty()), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + return new Schema(documentType, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); } private static Schema createSearchWithDocument(String documentName) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java index 6a736a5331e..534c82157e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import java.io.IOException; @@ -14,13 +14,13 @@ import static org.junit.Assert.*; public abstract class AssertSearchBuilder { public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { - assertNotNull(SchemaBuilder.buildFromFile(searchDefinitionFileName)); + assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); } public static void assertBuildFails(String searchDefinitionFileName, String expectedException) throws IOException, ParseException { try { - SchemaBuilder.buildFromFile(searchDefinitionFileName); + ApplicationBuilder.buildFromFile(searchDefinitionFileName); fail(searchDefinitionFileName); } catch (IllegalArgumentException e) { assertEquals(expectedException, e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java index f1aa64c1a60..67b369fd951 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { @Test public void testAttributesExactMatch() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); assertEquals(schema.getConcreteField("color").getMatching().getType(), Matching.Type.EXACT); assertEquals(schema.getConcreteField("artist").getMatching().getType(), Matching.Type.WORD); assertEquals(schema.getConcreteField("drummer").getMatching().getType(), Matching.Type.WORD); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index f74e46d92dc..efec9206ed9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -31,7 +31,7 @@ public class BoldingTestCase extends AbstractSchemaTestCase { @Test public void testBoldOnNonString() throws ParseException { try { - SchemaBuilder.createFromString(boldonnonstring); + ApplicationBuilder.createFromString(boldonnonstring); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("'bolding: on' for non-text field 'year4' (datatype int (code: 0)) is not allowed", @@ -52,7 +52,7 @@ public class BoldingTestCase extends AbstractSchemaTestCase { @Test public void testBoldOnArray() throws ParseException { try { - SchemaBuilder.createFromString(boldonarray); + ApplicationBuilder.createFromString(boldonarray); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array<string> (code: -1486737430)) is not allowed", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java index 342ff4ba2d4..db27bbbb84d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java index 6f8ab24de8e..a17992fad18 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; @@ -30,7 +30,7 @@ public class DictionaryTestCase { return builder.build(); } private Schema createSearch(String def) throws ParseException { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); return sb.getSchema(); } @Test @@ -196,7 +196,7 @@ public class DictionaryTestCase { " }", "}"); try { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); fail("Controlling dictionary for non-numeric fields are not yet supported."); } catch (IllegalArgumentException e) { assertEquals("For schema 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); @@ -214,7 +214,7 @@ public class DictionaryTestCase { " }", "}"); try { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); fail("Controlling dictionary for non-fast-search fields are not allowed."); } catch (IllegalArgumentException e) { assertEquals("For schema 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java index 03125c48d1d..b03aff455c5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -43,15 +43,15 @@ public class DisallowComplexMapAndWsetKeyTypesTestCase { private void testFieldType(String fieldType) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " struct mystruct {}\n" + " field a type " + fieldType + " {}\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java index dd958fb34cd..0c25cef49a1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; import org.junit.Test; @@ -20,15 +20,15 @@ public class FastAccessValidatorTest { @Test public void throws_exception_on_incompatible_use_of_fastaccess() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema( TestUtil.joinLines( "schema parent {", " document parent {", " field int_field type int { indexing: attribute }", " }", "}")); - builder.importString( + builder.addSchema( TestUtil.joinLines( "schema test {", " document test { ", @@ -55,7 +55,7 @@ public class FastAccessValidatorTest { "For schema 'test': The following attributes have a type that is incompatible " + "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + "Predicate, tensor and reference attributes are incompatible with fast-access."); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java index 43077fadfcd..833a6effe4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -18,7 +18,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -30,7 +30,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -38,12 +38,14 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { assertNotNull(docType.getField("foo")); assertNotNull(docType.getField("bar")); assertNotNull(docType.getField("cox")); - assertEquals(3, docType.getFieldCount()); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); } @Test public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -57,7 +59,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -70,7 +72,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -82,9 +84,9 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { - SchemaBuilder sb = new SchemaBuilder(); - sb.importFile("src/test/examples/nextgen/simple.sd"); - sb.build(); + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); + sb.build(true); assertNotNull(sb.getSchema()); new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java index bd645b68728..443eff07f41 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.document.*; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -16,7 +16,7 @@ import static org.junit.Assert.*; public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -25,7 +25,7 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { } @Test public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java index 9c1e5d0d8d7..cc2bc7d7bf6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; @@ -29,7 +29,7 @@ public class ImplicitSummariesTestCase { LogHandler log = new LogHandler(); Logger.getLogger("").addHandler(log); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); assertNotNull(schema); assertTrue(log.records.isEmpty()); } @@ -60,19 +60,19 @@ public class ImplicitSummariesTestCase { @Test public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array").getTransform()); } @Test public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("str_elem_map").getTransform()); } @Test public void attribute_combiner_transform_is_not_set_when_map_of_struct_has_some_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.NONE, schema.getSummaryField("int_elem_map").getTransform()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java index 44b1d9387f1..175b8d6fe1e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; @@ -17,7 +17,7 @@ public class ImplicitSummaryFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); assertNotNull(schema); DocumentSummary docsum = schema.getSummary("default"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java index 6eab1dddc79..b15b81b717d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; @@ -68,21 +68,21 @@ public class ImportedFieldsTestCase { } private static Schema buildAdSearch(String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "schema campaign {", " document campaign {", " field budget type int { indexing: attribute }", " }", "}")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema person {", " document person {", " field name type string { indexing: attribute }", " }", "}")); - builder.importString(sdContent); - builder.build(); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("ad"); } @@ -312,19 +312,19 @@ public class ImportedFieldsTestCase { } private static Schema buildChildSearch(String parentSdContent, String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(parentSdContent); - builder.importString(sdContent); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("child"); } private static Schema buildChildSearch(String grandParentSdContent, String parentSdContent, String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(grandParentSdContent); - builder.importString(parentSdContent); - builder.importString(sdContent); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(grandParentSdContent); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("child"); } @@ -488,41 +488,41 @@ public class ImportedFieldsTestCase { assertTrue(attrs.containsAttribute("entries.value")); } - private SchemaBuilder buildParentsUsingInheritance() throws ParseException { - var builder = new SchemaBuilder(); - builder.importString(joinLines("schema parent_a {", - "document parent_a {", - " struct Entry {", - " field key type string {}", - " field value type string {}", - " }", - " field entries type array<Entry> {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - " }", - "}", - "}")); - - builder.importString(joinLines("schema parent_b {", - "document parent_b inherits parent_a {", - "}", - "}")); - - builder.importString(joinLines("schema child {", - "document child {", - " field ref_parent_a type reference<parent_a> {", - " indexing: attribute", - " }", - " field ref_parent_b type reference<parent_b> {", - " indexing: attribute", - " }", - "}", - "import field ref_parent_a.entries as entries_from_a {}", - "import field ref_parent_b.entries as entries_from_b {}", - "}")); - - builder.build(); + private ApplicationBuilder buildParentsUsingInheritance() throws ParseException { + var builder = new ApplicationBuilder(); + builder.addSchema(joinLines("schema parent_a {", + "document parent_a {", + " struct Entry {", + " field key type string {}", + " field value type string {}", + " }", + " field entries type array<Entry> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + " }", + "}", + "}")); + + builder.addSchema(joinLines("schema parent_b {", + "document parent_b inherits parent_a {", + "}", + "}")); + + builder.addSchema(joinLines("schema child {", + "document child {", + " field ref_parent_a type reference<parent_a> {", + " indexing: attribute", + " }", + " field ref_parent_b type reference<parent_b> {", + " indexing: attribute", + " }", + "}", + "import field ref_parent_a.entries as entries_from_a {}", + "import field ref_parent_b.entries as entries_from_b {}", + "}")); + + builder.build(true); return builder; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 3105e3c7efd..9e4ba1c6728 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -2,11 +2,12 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -120,7 +121,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { "clear_state | guard { input smallattribute | attribute smallattribute; }", "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), - SchemaBuilder.buildFromFile("src/test/examples/simple.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); } @Test @@ -129,7 +130,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + " tokenize | index title; }", "clear_state | guard { input title_src | summary title_s; }"), - SchemaBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); } @Test @@ -154,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { private static ScriptExpression processField(SDField unprocessedField) { SDDocumentType sdoc = new SDDocumentType("test"); sdoc.addField(unprocessedField); - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); schema.addDocument(sdoc); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false, Set.of()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java index 5b5c5cedc0d..74a8cdfdb6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -64,7 +64,7 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { assertIndexing( Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), - SchemaBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 4b13590c777..2465839b72b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -24,7 +24,7 @@ public class IntegerIndex2AttributeTestCase extends AbstractSchemaTestCase { @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); new IntegerIndex2Attribute(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java index 7eea2fca0e6..b48927d58a3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -175,18 +175,18 @@ public class MatchedElementsOnlyResolverTestCase { } private Schema buildSearch(String field, String summary) throws ParseException { - var builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - field, - " }", - summary, - "}")); - builder.build(); + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + field, + " }", + summary, + "}")); + builder.build(true); return builder.getSchema(); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java index 1adb909ff21..7169a97f3fc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; @@ -25,7 +25,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testNGram() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/ngram.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/ngram.sd"); assertNotNull(schema); SDField gram1 = schema.getConcreteField("gram_1"); @@ -55,7 +55,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting1() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { @@ -66,7 +66,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting2() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { @@ -77,7 +77,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting3() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java index 5fce16f381a..4eeab15fdd2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java @@ -1,19 +1,70 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; +import java.util.Optional; + import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromStrings; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class PagedAttributeValidatorTestCase { @Test - public void dense_tensor_attribute_does_support_paged_setting() throws ParseException { - createFromString(getSd("tensor(x[2],y[2])")); + public void dense_tensor_attribute_supports_paged_setting() throws ParseException { + assertPagedSupported("tensor(x[2],y[2])"); + } + + @Test + public void primitive_attribute_types_support_paged_setting() throws ParseException { + assertPagedSupported("int"); + assertPagedSupported("array<int>"); + assertPagedSupported("weightedset<int>"); + + assertPagedSupported("string"); + assertPagedSupported("array<string>"); + assertPagedSupported("weightedset<string>"); + } + + @Test + public void struct_field_attributes_support_paged_setting() throws ParseException { + var sd = joinLines("schema test {", + " document test {", + " struct elem {", + " field first type int {}", + " field second type string {}", + " }", + " field foo type array<elem> {", + " indexing: summary", + " struct-field first {", + " indexing: attribute", + " attribute: paged", + " }", + " struct-field second {", + " indexing: attribute", + " attribute: paged", + " }", + " }", + " }", + "}"); + + var appBuilder = createFromString(sd); + var field = appBuilder.getSchema().getField("foo"); + assertTrue(field.getStructField("first").getAttribute().isPaged()); + assertTrue(field.getStructField("second").getAttribute().isPaged()); + } + + private void assertPagedSupported(String fieldType) throws ParseException { + var appBuilder = createFromString(getSd(fieldType)); + var attribute = appBuilder.getSchema().getAttribute("foo"); + assertTrue(attribute.isPaged()); } @Test @@ -22,25 +73,42 @@ public class PagedAttributeValidatorTestCase { } @Test - public void non_tensor_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("string"); + public void predicate_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("predicate"); + } + + @Test + public void reference_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("reference<parent>", Optional.of(getSd("parent", "int"))); } private void assertPagedSettingNotSupported(String fieldType) throws ParseException { + assertPagedSettingNotSupported(fieldType, Optional.empty()); + } + + private void assertPagedSettingNotSupported(String fieldType, Optional<String> parentSd) throws ParseException { try { - createFromString(getSd(fieldType)); + if (parentSd.isPresent()) { + createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); + } else { + createFromString(getSd(fieldType)); + } fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is only supported for dense tensor types", + assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", e.getMessage()); } } - private String getSd(String type) { + private String getSd(String fieldType) { + return getSd("test", fieldType); + } + + private String getSd(String docType, String fieldType) { return joinLines( - "schema test {", - " document test {", - " field foo type " + type + "{", + "schema " + docType + " {", + " document " + docType + " {", + " field foo type " + fieldType + "{", " indexing: attribute", " attribute: paged", " }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index 74fa7c72554..b14c7287537 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -23,7 +23,6 @@ import com.yahoo.searchdefinition.document.TemporarySDField; */ public class ParentChildSearchModel { - private final Application application = new Application(MockApplicationPackage.createEmpty()); public Schema parentSchema; public Schema childSchema; @@ -33,7 +32,7 @@ public class ParentChildSearchModel { } protected Schema createSearch(String name) { - Schema result = new Schema(name, application, new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); result.addDocument(new SDDocumentType(name)); return result; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java index c3beeeaa17e..eaf38396efc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.SummaryField; @@ -27,7 +27,7 @@ public class PositionTestCase { @Test public void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { - SchemaBuilder sb = SchemaBuilder.createFromFiles(Arrays.asList( + ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( "src/test/examples/position_base.sd", "src/test/examples/position_inherited.sd")); @@ -38,7 +38,7 @@ public class PositionTestCase { @Test public void requireThatPositionCanBeAttribute() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_attribute.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); @@ -50,7 +50,7 @@ public class PositionTestCase { @Test public void requireThatPositionCanNotBeIndex() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/position_index.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/position_index.sd"); fail(); } catch (IllegalArgumentException e) { assertEquals("For schema 'position_index', field 'pos': Indexing of data type 'position' is not " + @@ -60,7 +60,7 @@ public class PositionTestCase { @Test public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_summary.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_summary.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); @@ -79,7 +79,7 @@ public class PositionTestCase { @Test public void requireThatExtraFieldCanBePositionAttribute() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_extra.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_extra.sd"); assertNull(schema.getAttribute("pos_ext")); assertNull(schema.getAttribute("pos_ext.x")); assertNull(schema.getAttribute("pos_ext.y")); @@ -90,7 +90,7 @@ public class PositionTestCase { @Test public void requireThatPositionArrayIsSupported() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_array.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_array.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java index 87dd92f41d9..3a0ceebcb0a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,6 +17,6 @@ import java.io.IOException; public class RankModifierTestCase extends AbstractSchemaTestCase { @Test public void testLiteral() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 2707f60f828..2d0bdb58122 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -13,7 +13,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; @@ -68,7 +68,7 @@ class RankProfileSearchFixture { String rankProfiles, String constant, String field) throws ParseException { this.queryProfileRegistry = queryProfileRegistry; - SchemaBuilder builder = new SchemaBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -77,8 +77,8 @@ class RankProfileSearchFixture { rankProfiles + "\n" + "}"; - builder.importString(sdContent); - builder.build(); + builder.addSchema(sdContent); + builder.build(true); schema = builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index b1624b7fd7e..0ab7406b5b1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -21,10 +21,10 @@ public class RankPropertyVariablesTestCase extends AbstractSchemaTestCase { @Test public void testRankPropVariables() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/rankpropvars.sd", - new BaseDeployLogger(), - rankProfileRegistry, - new QueryProfileRegistry()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar1", "foo"); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar_2", "bar"); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvarOne23", "baz"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 4f18c9b68fd..7f5e8f8753a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; @@ -33,8 +33,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorFirstPhaseMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -48,7 +48,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -61,8 +61,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(d0[3]) {", @@ -96,7 +96,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -110,8 +110,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorSecondPhaseMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -128,7 +128,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -140,8 +140,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { try { - SchemaBuilder schemaBuilder = new SchemaBuilder(); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(); + schemaBuilder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[5]) {", @@ -158,7 +158,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -172,8 +172,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -194,7 +194,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), @@ -205,8 +205,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testTensorFunctionInvocationTypes_Nested() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[1]) {", @@ -236,7 +236,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -247,8 +247,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testAttributeInvocationViaBoundIdentifier() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search newsarticle {", " document newsarticle {", " field title type string {", @@ -280,14 +280,14 @@ public class RankingExpressionTypeResolverTestCase { " first-phase { expression: commonfirstphase(eustaticrank) }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); } @Test public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[1]) {", @@ -320,7 +320,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -331,8 +331,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test {", " field t1 type tensor<float>(y{}) { indexing: attribute | summary }", @@ -345,7 +345,7 @@ public class RankingExpressionTypeResolverTestCase { " summary-features { test_func_via_func_with_expr }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); assertEquals(TensorType.fromSpec("tensor<float>(y{})"), summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); @@ -353,8 +353,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void importedFieldsAreAvailable() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search parent {", " document parent {", " field a type tensor(x[5],y[1000]) {", @@ -363,7 +363,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.importString(joinLines( + builder.addSchema(joinLines( "search child {", " document child { ", " field ref type reference<parent> {", @@ -378,14 +378,14 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); } @Test public void undeclaredQueryFeaturesAreAccepted() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type double {" + @@ -410,8 +410,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type tensor(d[2]) {", @@ -439,7 +439,7 @@ public class RankingExpressionTypeResolverTestCase { @Test public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); + ApplicationBuilder builder = new ApplicationBuilder(logger); QueryProfileType myType = new QueryProfileType("mytype"); myType.addField(new FieldDescription("rank.feature.query(foo)", new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), @@ -451,7 +451,7 @@ public class RankingExpressionTypeResolverTestCase { new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), builder.getQueryProfileRegistry().getTypeRegistry()); builder.getQueryProfileRegistry().getTypeRegistry().register(myType); - builder.importString(joinLines( + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type tensor(d[2]) {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index 09aa09a626b..d44c102a5c3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -259,7 +259,7 @@ public class RankingExpressionWithOnnxTestCase { public void testFunctionGeneration() { final String name = "small_constants_and_functions"; final String rankProfiles = - " rank-profile my_profile {\n" + + " rank-profile my_profile {\n" + " function input() {\n" + " expression: tensor<float>(d0[3])(0.0)\n" + " }\n" + @@ -399,8 +399,10 @@ public class RankingExpressionWithOnnxTestCase { @Override public List<NamedReader> getFiles(Path path, String suffix) { + File[] files = getFileReference(path).listFiles(); + if (files == null) return List.of(); List<NamedReader> readers = new ArrayList<>(); - for (File file : getFileReference(path).listFiles()) { + for (File file : files) { if ( ! file.getName().endsWith(suffix)) continue; try { readers.add(new NamedReader(file.getName(), new FileReader(file))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java index 93de116883a..ffc073ed434 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -9,7 +9,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; import com.yahoo.searchdefinition.parser.ParseException; @@ -87,9 +87,9 @@ public class RankingExpressionWithTransformerTokensTestCase { " document test {}\n" + " rank-profile my_profile inherits default {}\n" + "}"; - SchemaBuilder schemaBuilder = new SchemaBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); - schemaBuilder.importString(sdContent); - schemaBuilder.build(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + schemaBuilder.addSchema(sdContent); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index f4742be6b30..b42f932ef98 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -11,7 +11,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -31,7 +31,7 @@ import static org.junit.Assert.assertTrue; public class RankingExpressionsTestCase extends AbstractSchemaTestCase { private static Schema createSearch(String dir, ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return SchemaBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); + return ApplicationBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java index c9437761e0d..d60cb9040d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.Field; import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.DocumentGraphValidator; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; @@ -28,7 +28,7 @@ public class ReferenceFieldTestCase { @Test public void reference_fields_are_parsed_from_search_definition() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); String campaignSdContent = "search campaign {\n" + " document campaign {\n" + @@ -46,10 +46,10 @@ public class ReferenceFieldTestCase { " field salesperson_ref type reference<salesperson> { indexing: attribute }\n" + " }\n" + "}"; - builder.importString(campaignSdContent); - builder.importString(salespersonSdContent); - builder.importString(adSdContent); - builder.build(); + builder.addSchema(campaignSdContent); + builder.addSchema(salespersonSdContent); + builder.addSchema(adSdContent); + builder.build(true); Schema schema = builder.getSchema("ad"); assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); @@ -57,7 +57,7 @@ public class ReferenceFieldTestCase { @Test public void cyclic_document_dependencies_are_detected() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); String campaignSdContent = "search campaign {\n" + " document campaign {\n" + @@ -70,11 +70,11 @@ public class ReferenceFieldTestCase { " field campaign_ref type reference<campaign> { indexing: attribute }\n" + " }\n" + "}"; - builder.importString(campaignSdContent); - builder.importString(adSdContent); + builder.addSchema(campaignSdContent); + builder.addSchema(adSdContent); exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); exceptionRule.expectMessage("Document dependency cycle detected: campaign->ad->campaign."); - builder.build(); + builder.build(true); } private static void assertSearchContainsReferenceField(String expectedFieldname, @@ -87,4 +87,5 @@ public class ReferenceFieldTestCase { ReferenceDataType refField = (ReferenceDataType) dataType; assertEquals(referencedDocType, refField.getTargetType().getName()); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 922680e8f1a..d3d79fd7798 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -21,8 +21,8 @@ public class ReservedRankingExpressionFunctionNamesTestCase { public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(deployLogger, rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(deployLogger, rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java index 24dc98e4d93..4cfe4c09052 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ public class SchemaMustHaveDocumentTest { @Test public void requireErrorWhenMissingDocument() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); fail("SD without document"); } catch (IllegalArgumentException e) { if (!e.getMessage() diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java index 067a132cc4a..f224dc0688e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; @@ -39,7 +39,7 @@ public class SummaryConsistencyTestCase { "", "}" ); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index eb3e61b9f7a..1eb5e66df93 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -21,7 +21,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); @@ -31,7 +31,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); @@ -41,7 +41,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); @@ -50,7 +50,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); new SummaryFieldsMustHaveValidSource(schema, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index 82b288dc66d..9e53bd57d77 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index eec2ef4c3a0..c71fd3dd489 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -14,7 +14,7 @@ import com.yahoo.searchdefinition.LargeRankExpressions; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -140,8 +140,8 @@ public class TensorTransformTestCase extends AbstractSchemaTestCase { private List<Pair<String, String>> buildSearch(String expression) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfiles); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( "search test {\n" + " document test { \n" + " field double_field type double { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index fe9d19310a9..22fd4e45c4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -51,7 +51,7 @@ public class ValidateFieldTypesTest { private static Schema createSearch(String documentType) { return new Schema(documentType, - new Application(MockApplicationPackage.createEmpty()), + MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java index 2a737f4eb69..e56d6ccf343 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ public class WeightedSetSummaryToTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); assertNotNull(schema); } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java index 754979fb391..33d84cf4313 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -27,27 +27,27 @@ public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenc } private static class TestDocumentModelBuilder { - private final SchemaBuilder builder = new SchemaBuilder(); + private final ApplicationBuilder builder = new ApplicationBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { - builder.importString(joinLines("search campaign {", - " document campaign {", - " field cool_field type string { indexing: attribute }", - " field swag_field type long { indexing: attribute }", - " }", - "}")); + builder.addSchema(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); return this; } public TestDocumentModelBuilder addPerson() throws ParseException { - builder.importString(joinLines("search person {", - " document person {", - " field name type string { indexing: attribute }", - " }", - "}")); + builder.addSchema(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); return this; } public DocumentModel build(String adSdContent) throws ParseException { - builder.importString(adSdContent); - builder.build(); + builder.addSchema(adSdContent); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index f9b9bf2610e..00f39a23a8f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.ReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -56,22 +56,22 @@ public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReference } private static class TestDocumentModelBuilder { - private final SchemaBuilder builder = new SchemaBuilder(); + private final ApplicationBuilder builder = new ApplicationBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { - builder.importString(joinLines("search campaign {", - " document campaign {}", - "}")); + builder.addSchema(joinLines("search campaign {", + " document campaign {}", + "}")); return this; } public TestDocumentModelBuilder addPerson() throws ParseException { - builder.importString(joinLines("search person {", - " document person {}", - "}")); + builder.addSchema(joinLines("search person {", + " document person {}", + "}")); return this; } public DocumentModel build(String adSdContent) throws ParseException { - builder.importString(adSdContent); - builder.build(); + builder.addSchema(adSdContent); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index 9dfd05b6fdc..dc88590a198 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; @@ -35,10 +35,10 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { @Test public void testDocumentTypesWithDocumentField() throws IOException, ParseException { - SchemaBuilder search = new SchemaBuilder(); - search.importFile("src/test/configmodel/types/other_doc.sd"); - search.importFile("src/test/configmodel/types/type_with_doc_field.sd"); - search.build(); + ApplicationBuilder search = new ApplicationBuilder(); + search.addSchemaFile("src/test/configmodel/types/other_doc.sd"); + search.addSchemaFile("src/test/configmodel/types/type_with_doc_field.sd"); + search.build(true); DocumentModel model = search.getModel(); DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); @@ -48,15 +48,15 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { @Test public void testMultipleInheritanceArray() throws IOException, ParseException { - SchemaBuilder search = new SchemaBuilder(); - search.importFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); - search.importFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); - search.importFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); - search.build(); + ApplicationBuilder search = new ApplicationBuilder(); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); + search.build(true); } private DocumentModel createAndTestModel(String sd) throws IOException, ParseException { - SchemaBuilder search = SchemaBuilder.createFromFile(sd); + ApplicationBuilder search = ApplicationBuilder.createFromFile(sd); DocumentModel model = search.getModel(); assertEquals(2, model.getDocumentManager().getTypes().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 80a388a822e..4d8dd3dacfd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -26,7 +26,7 @@ public class HostResourceTest { TestService service = new TestService(root, 1); try { - service.initService(root.deployLogger()); + service.initService(root.getDeployState()); } catch (RuntimeException e) { assertTrue(e.getMessage().endsWith("No host found for service 'hostresourcetest$testservice0'. " + "The hostalias is probably missing from hosts.xml.")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 474013c17fc..8012a00076b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -291,7 +291,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { Service cc = model.getService("admin/cluster-controllers/0").get(); assertTrue(cc instanceof ClusterControllerContainer); - assertEquals("-Dio.netty.allocator.pageSize=4096 -Dio.netty.allocator.maxOrder=8", cc.getJvmOptions()); + assertEquals("-Dio.netty.allocator.pageSize=4096 -Dio.netty.allocator.maxOrder=5 -Dio.netty.allocator.numHeapArenas=1 -Dio.netty.allocator.numDirectArenas=1", cc.getJvmOptions()); } private boolean existsHostsWithClusterControllerConfigId(VespaModel model) { @@ -397,7 +397,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); - assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); + assertEquals(16, qrStartConfig.jvm().baseMaxDirectMemorySize()); assertReindexingConfigPresent(model); assertReindexingConfiguredOnAdminCluster(model); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java deleted file mode 100644 index e2386e145ca..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.provision.Cloud; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import org.junit.Before; - -/** - * @author gjoranv - */ -public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBase { - - @Before - public void setup() { - validator = new AwsAccessControlValidator(); - zone = new Zone(Cloud.builder().requireAccessControl(true).build(), - SystemName.main, Environment.prod, RegionName.from("foo")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index e2eae30d78d..b5b93be6cd7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -2,39 +2,44 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.Manifest; +import java.util.jar.JarOutputStream; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class BundleValidatorTest { - private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); @Test public void basicBundleValidation() throws Exception { // Valid jar file - JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); + JarFile ok = createTemporaryJarFile("ok"); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), ok); + bundleValidator.validateJarFile(DeployState.createTestState(), ok); // No manifest - validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); + validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); } private void validateWithException(String jarName, String exceptionMessage) throws IOException { try { - JarFile jarFile = new JarFile(JARS_DIR + jarName); + JarFile jarFile = createTemporaryJarFile(jarName); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile); + bundleValidator.validateJarFile(DeployState.createTestState(), jarFile); assert (false); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), exceptionMessage); @@ -45,26 +50,48 @@ public class BundleValidatorTest { public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); - - new BundleValidator().validateJarFile(logger, new JarFile(JARS_DIR + "snapshot_bundle.jar")); + DeployState state = createDeployState(buffer); + JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); + new BundleValidator().validateJarFile(state, jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @Test public void outputs_deploy_warning_on_import_of_packages_from_deprecated_artifact() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); + DeployState state = createDeployState(buffer); BundleValidator validator = new BundleValidator(); - Manifest manifest = new Manifest(Files.newInputStream(Paths.get(JARS_DIR + "/manifest-producing-import-warnings.MF"))); - validator.validateManifest(logger, "my-app-bundle.jar", manifest); + JarFile jarFile = createTemporaryJarFile("import-warnings"); + validator.validateJarFile(state, jarFile); assertThat(buffer.toString()) - .contains("For JAR file 'my-app-bundle.jar': \n" + - "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + - "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); + .contains("JAR file 'import-warnings.jar' imports the packages [org.json] from 'org.json:json'. \n" + + "This bundle is no longer provided on Vespa 8 - see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } - private DeployLogger createDeployLogger(StringBuffer buffer) { - return (__, message) -> buffer.append(message).append('\n'); + private DeployState createDeployState(StringBuffer buffer) { + DeployLogger logger = (__, message) -> buffer.append(message).append('\n'); + return DeployState.createTestState(logger); } + + private JarFile createTemporaryJarFile(String testArtifact) throws IOException { + Path jarFile = tempDir.newFile(testArtifact + ".jar").toPath(); + Path artifactDirectory = Paths.get("src/test/cfg/application/validation/testjars/" + testArtifact); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jarFile))) { + Files.walk(artifactDirectory).forEach(path -> { + Path relativePath = artifactDirectory.relativize(path); + String zipName = relativePath.toString(); + uncheck(() -> { + if (Files.isDirectory(path)) { + out.putNextEntry(new JarEntry(zipName + "/")); + } else { + out.putNextEntry(new JarEntry(zipName)); + out.write(Files.readAllBytes(path)); + } + out.closeEntry(); + }); + }); + } + return new JarFile(jarFile.toFile()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 689f1c70b6e..85050aa0cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -105,7 +105,7 @@ public class ComplexAttributeFieldsValidatorTestCase { DeployState deployState = createDeployState(servicesXml(), schema); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); - Validation.validate(model, validationParameters, deployState); + new Validation().validate(model, validationParameters, deployState); } private static DeployState createDeployState(String servicesXml, String schema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 90d5c0b27a4..3b5c621a581 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -10,7 +10,7 @@ import java.io.Reader; import java.io.StringReader; import static com.yahoo.test.json.JsonTestHelper.inputJson; -import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; +import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; public class ConstantTensorJsonValidatorTest { @@ -88,7 +88,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_bound_tensor_outside_limits_is_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 5 not within limits of bound dimension 'x'"); validateTensorJson( @@ -121,7 +121,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_integer_strings_in_address_points_are_disallowed_unbound() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( @@ -139,7 +139,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_tensor_coordinates_are_strings() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor label is not a string (VALUE_NUMBER_INT)"); validateTensorJson( @@ -157,7 +157,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_integer_strings_in_address_points_are_disallowed_bounded() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( @@ -175,7 +175,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_missing_coordinates_fail() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor address missing dimension(s) y, z"); validateTensorJson( @@ -193,7 +193,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_number_values_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor value is not a number (VALUE_STRING)"); validateTensorJson( @@ -211,7 +211,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_extra_dimensions_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor dimension 'z' does not exist"); validateTensorJson( @@ -229,7 +229,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_duplicate_dimensions_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Duplicate tensor dimension 'y'"); validateTensorJson( @@ -247,7 +247,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_invalid_json_fails() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Failed to parse JSON stream"); validateTensorJson( @@ -265,7 +265,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_invalid_json_not_in_tensor_format_fails() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Expected field name 'cells', got 'stats'"); validateTensorJson(TensorType.fromSpec("tensor(x[], y[])"), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java index b1847e5d134..f6fa73dce74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; -import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationFailed; +import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationException; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -20,7 +20,7 @@ public class RankingConstantsValidatorTest { try { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); fail(); - } catch (TensorValidationFailed e) { + } catch (TensorValidationException e) { assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_2' (tensors/constant_tensor_2.json): Tensor label is not a string (VALUE_NUMBER_INT)")); assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_3' (tensors/constant_tensor_3.json): Tensor dimension 'cd' does not exist")); assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_4' (tensors/constant_tensor_4.json): Tensor dimension 'z' does not exist")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java index 16bff341c0b..93d42e07dd8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java @@ -37,7 +37,7 @@ public class ValidationOverridesValidatorTest { var deployState = createDeployState(validationOverridesXml); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); } @Test @@ -58,7 +58,7 @@ public class ValidationOverridesValidatorTest { try { var deployState = createDeployState(validationOverridesXml); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { assertEquals(message, e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java deleted file mode 100644 index b9e92e1b866..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.first; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.application.validation.AccessControlValidatorTestBase; -import org.junit.Before; - -/** - * @author gjoranv - */ -public class AccessControlOnFirstDeploymentValidatorTest extends AccessControlValidatorTestBase { - - @Before - public void setup() { - validator = new AccessControlOnFirstDeploymentValidator(); - zone = new Zone(Environment.prod, RegionName.from("foo")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index c746db168ee..77fa0b685fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.config.GenericConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; @@ -35,7 +36,6 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -328,10 +328,64 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(200000, b.getRootGroup().getMmapNoCoreLimit().get().longValue()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(200000, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + } + + @Test + public void canAddEnvironmentVariable() { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + + " </group>" + + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + node.addEnvironmentVariable("MY_ENV_1", 7); + node.addEnvironmentVariable("MY_ENV_2", "7 8"); + assertTrue(node.getEnv().contains("MY_ENV_1=7")); + assertTrue(node.getEnv().contains("MY_ENV_2=\"7 8\"")); + node.addEnvironmentVariable("MY_PARSED_ENV_1=7"); + node.addEnvironmentVariable("MY_PARSED_ENV_2=7 8"); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_1=\"7\"")); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_2=\"7 8\"")); + } + + @Test + public void addsEnvironmentVariablesfromFeatureFlag() { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + + " </group>" + + "</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + assertTrue(node.getEnv().contains("MY_1_ENV=\"xyz abc\"")); + assertTrue(node.getEnv().contains("MY_2_ENV=\"2\"")); } @Test @@ -357,10 +411,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertTrue(b.getRootGroup().getCoreOnOOM().get()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertTrue(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -383,10 +435,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertFalse(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -409,10 +459,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getMmapNoCoreLimit().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(-1, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=")); } @Test @@ -435,10 +483,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -471,15 +517,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(4, s.getSearchNodes().size()); for (SearchNode n : s.getSearchNodes()) { - assertEquals("proton", n.getNoVespaMalloc()); - assertEquals("VESPA_USE_NO_VESPAMALLOC=\"proton\" ", n.getNoVespaMallocEnvVariable()); - assertEquals("distributord", n.getVespaMallocDebug()); - assertEquals("VESPA_USE_VESPAMALLOC=\"storaged\" ", n.getVespaMallocEnvVariable()); - assertEquals("all", n.getVespaMallocDebugStackTrace()); - assertEquals("VESPA_USE_VESPAMALLOC_D=\"distributord\" ", n.getVespaMallocDebugEnvVariable()); - assertEquals("storaged", n.getVespaMalloc()); - assertEquals("VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getVespaMallocDebugStackTraceEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\"", n.getEnv()); } } @@ -508,10 +546,10 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getVespaMallocDebugStackTrace().isPresent()); assertEquals(4, s.getSearchNodes().size()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" ", s.getSearchNodes().get(0).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_D=\"distributord\" ", s.getSearchNodes().get(1).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_DST=\"all\" ", s.getSearchNodes().get(2).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC=\"storaged\" ", s.getSearchNodes().get(3).getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\"", s.getSearchNodes().get(0).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_D=\"distributord\"", s.getSearchNodes().get(1).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_DST=\"all\"", s.getSearchNodes().get(2).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC=\"storaged\"", s.getSearchNodes().get(3).getEnv()); } @Test @@ -908,6 +946,9 @@ public class ContentBuilderTest extends DomBuilderTest { } private ContentCluster createContent(String xml) { + return createContent(xml, new TestProperties()); + } + private ContentCluster createContent(String xml, TestProperties props) { String combined = "" + "<services>"+ " <admin version='2.0'>" + @@ -917,12 +958,13 @@ public class ContentBuilderTest extends DomBuilderTest { "</services>"; + var deployStateBuilder = new DeployState.Builder().properties(props); VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withHosts(getHosts()) - .withServices(combined) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(); + .withHosts(getHosts()) + .withServices(combined) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); return m.getContentClusters().isEmpty() ? null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index eb1cf668cc9..198b5713876 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -6,7 +6,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; @@ -178,7 +177,7 @@ public class ContainerClusterTest { public void testClusterControllerResourceUsage() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertEquals(1, cluster.getContainers().size()); QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); cluster.getConfig(qrBuilder); @@ -190,7 +189,7 @@ public class ContainerClusterTest { root.freezeModelTopology(); ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); - assertEquals(10, threadpoolConfig.maxthreads()); + assertEquals(2, threadpoolConfig.maxthreads()); assertEquals(50, threadpoolConfig.queueSize()); } @@ -198,7 +197,7 @@ public class ContainerClusterTest { public void testThatLinguisticsIsExcludedForClusterControllerCluster() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertFalse(contains("com.yahoo.language.provider.DefaultLinguisticsProvider", cluster.getAllComponents())); } @@ -365,71 +364,61 @@ public class ContainerClusterTest { assertNames(main, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix"), - List.of("search-cluster--default--t1.endpoint.suffix")); + List.of("search-cluster.default.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), - List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); // cd system: assertNames(cd, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix"), - List.of("search-cluster--cd--default--t1.endpoint.suffix")); + List.of("search-cluster.cd.default.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a--b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), + new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix", "a--b.x.y.z", "rotation-2.x.y.z")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); } - private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names, List<String> expectedSharedNames) { + private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names) { Zone zone = new Zone(systemName, Environment.defaultEnvironment(), RegionName.defaultName()); DeployState state = new DeployState.Builder() .zone(zone) @@ -445,7 +434,7 @@ public class ContainerClusterTest { cluster.doPrepare(state); List<ApplicationClusterEndpoint> endpoints = cluster.endpoints(); - assertNames(expectedSharedNames, endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); + assertNames(List.of(), endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); assertNames(expectedSharedL4Names, endpoints.stream().filter(e -> e.routingMethod() == sharedLayer4).collect(Collectors.toList())); List<ContainerEndpoint> endpointsWithWeight = @@ -493,17 +482,16 @@ public class ContainerClusterTest { HostResource hostResource) { ApplicationContainer container = new ApplicationContainer(cluster, name, 0, root.getDeployState()); container.setHostResource(hostResource); - container.initService(root.deployLogger()); + container.initService(root.getDeployState()); cluster.addContainer(container); } - private static void addClusterController(DeployLogger deployLogger, - ClusterControllerContainerCluster cluster, + private static void addClusterController(ClusterControllerContainerCluster cluster, String hostName, DeployState deployState) { ClusterControllerContainer container = new ClusterControllerContainer(cluster, 1, false, deployState, false); container.setHostResource(new HostResource(new Host(null, hostName))); - container.initService(deployLogger); + container.initService(deployState); cluster.addContainer(container); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 4ca85a19c35..29cff23d3bd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -73,7 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); - assertEquals("", config.snapshotMethod()); + assertEquals("gz", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -152,8 +152,7 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar") - .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); + .region("bar"); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index fc7f8674149..95c3262f2f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; -import java.util.Objects; import java.util.Optional; /** @@ -18,7 +17,6 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); - private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -108,9 +106,6 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } - @Override - public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } - public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -136,10 +131,4 @@ public class TestOptions implements CloudConfigOptions { return this; } - public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { - Objects.requireNonNull(snapshotMethod); - this.zooKeeperSnapshotMethod = snapshotMethod; - return this; - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 5630d3cc186..f246b87d9bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.tensor.Tensor; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * Tests the ModelsEvaluatorTester. @@ -17,6 +19,7 @@ public class ModelsEvaluatorTest { @Test public void testModelsEvaluatorTester() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java index a9bab88a065..2cb0e8f0ab1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.List; import java.util.Set; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -80,8 +81,8 @@ public class SourceGroupTest { searchChains.validate(); fail("Expected exception"); } catch (Exception e) { - assertTrue(e.getMessage().contains("Same id used for a source")); - assertTrue(e.getMessage().contains("'sameId'")); + assertEquals("Id 'sameId' is used both for a source and another search chain/provider", + e.getMessage()); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java index b50261b7568..3490568770c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java @@ -31,7 +31,7 @@ import static org.junit.Assert.fail; /** * @author gjoranv */ -public class SchemaBuilderTest extends ContainerModelBuilderTestBase { +public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { private ChainsConfig chainsConfig() { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index f7dbd65ab7d..16b4a8bed9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -778,7 +778,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .build(); createModel(root, state, null, clusterElem); } catch (RuntimeException e) { - assertEquals("cloud secret store is not supported in non-public system, please see documentation", + assertEquals("Cloud secret store is not supported in non-public system, see the documentation", e.getMessage()); return; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index a17459ea800..814b2f94e44 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -181,7 +181,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-XX:+ParallelGCThreads=8 foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM GC options in services.xml: bar,foo")); + assertTrue(e.getMessage().startsWith("Invalid or misplaced JVM GC options in services.xml: bar,foo")); } } @@ -205,8 +205,10 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals(Level.WARNING, firstOption.getFirst()); Collections.sort(strings); - assertEquals("Invalid JVM " + (optionName.equals("gc-options") ? "GC " : "") + - "options in services.xml: " + String.join(",", strings), firstOption.getSecond()); + assertEquals("Invalid or misplaced JVM" + (optionName.equals("gc-options") ? " GC" : "") + + " options in services.xml: " + String.join(",", strings) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm" + , firstOption.getSecond()); } private void buildModelWithJvmOptions(boolean isHosted, TestLogger logger, String optionName, String override) throws IOException, SAXException { @@ -267,7 +269,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-Xms2G foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM options in services.xml: bar,foo")); + assertTrue(e.getMessage().contains("Invalid or misplaced JVM options in services.xml: bar,foo")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 8cc7805fe3e..c9863cf2144 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.fail; public class ClusterResourceLimitsTest { private static class Fixture { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private final ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); @@ -30,12 +29,7 @@ public class ClusterResourceLimitsTest { this(false); } - public Fixture(boolean enableFeedBlockInDistributor) { - this(enableFeedBlockInDistributor, false); - } - - public Fixture(boolean enableFeedBlockInDistributor, boolean hostedVespa) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + public Fixture(boolean hostedVespa) { this.hostedVespa = hostedVespa; } @@ -57,8 +51,7 @@ public class ClusterResourceLimitsTest { } public ClusterResourceLimits build() { ModelContext.FeatureFlags featureFlags = new TestProperties(); - var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - hostedVespa, + var builder = new ClusterResourceLimits.Builder(hostedVespa, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); builder.setClusterControllerBuilder(ctrlBuilder); @@ -71,50 +64,32 @@ public class ClusterResourceLimitsTest { public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { assertLimits(0.4, 0.7, 0.7, 0.85, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7)); - assertLimits(0.4, null, 0.7, null, + assertLimits(0.4, 0.8, 0.7, 0.9, new Fixture().ctrlDisk(0.4)); - assertLimits(null, 0.7, null, 0.85, + assertLimits(0.75, 0.7, 0.875, 0.85, new Fixture().ctrlMemory(0.7)); - - - assertLimits(0.4, 0.7, 0.7, 0.85, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7)); - assertLimits(0.4, 0.8, 0.7, 0.9, - new Fixture(true).ctrlDisk(0.4)); - assertLimits(0.8, 0.7, 0.9, 0.85, - new Fixture(true).ctrlMemory(0.7)); } @Test public void content_node_limits_can_be_set_explicit() { assertLimits(0.4, 0.7, 0.9, 0.95, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.4, null, 0.95, null, + assertLimits(0.4, 0.8, 0.95, 0.9, new Fixture().ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(null, 0.7, null, 0.95, + assertLimits(0.75, 0.7, 0.875, 0.95, new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); - - assertLimits(0.4, 0.7, 0.9, 0.95, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.4, 0.8, 0.95, 0.9, - new Fixture(true).ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(0.8, 0.7, 0.9, 0.95, - new Fixture(true).ctrlMemory(0.7).nodeMemory(0.95)); } @Test public void cluster_controller_limits_are_equal_to_content_node_limits_minus_one_percent_if_not_set() { assertLimits(0.89, 0.94, 0.9, 0.95, new Fixture().nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.89, null, 0.9, null, + assertLimits(0.89, 0.8, 0.9, 0.9, new Fixture().nodeDisk(0.9)); - assertLimits(null, 0.94, null, 0.95, + assertLimits(0.75, 0.94, 0.875, 0.95, new Fixture().nodeMemory(0.95)); - assertLimits(null, 0.0, null, 0.005, + assertLimits(0.75, 0.0, 0.875, 0.005, new Fixture().nodeMemory(0.005)); - - assertLimits(0.89, 0.94, 0.9, 0.95, - new Fixture(true).nodeDisk(0.9).nodeMemory(0.95)); } @Test @@ -127,7 +102,7 @@ public class ClusterResourceLimitsTest { @Test public void default_resource_limits_when_feed_block_is_enabled_in_distributor() { - assertLimits(0.8, 0.8, 0.9, 0.9, + assertLimits(0.75, 0.8, 0.875, 0.9, new Fixture(true)); } @@ -193,7 +168,6 @@ public class ClusterResourceLimitsTest { "</cluster>"); ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, - true, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 87a962339e9..9f571167d8c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1045,6 +1045,28 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } + private ProtonConfig.Replay_throttling_policy.Type.Enum resolveReplayThrottlePolicyType(Optional<String> throttlerType) { + TestProperties testProperties = new TestProperties(); + if (throttlerType.isPresent()) { + testProperties.setPersistenceAsyncThrottling(throttlerType.get()); + } + VespaModel model = createEnd2EndOneNode(testProperties); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.replay_throttling_policy().type(); + } + + @Test + public void replay_throttling_policy_type_controlled_by_properties() { + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.empty())); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("UNLIMITED"))); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("INVALID"))); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.DYNAMIC, resolveReplayThrottlePolicyType(Optional.of("DYNAMIC"))); + } + private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); @@ -1120,54 +1142,38 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void distributor_merge_busy_wait_controlled_by_properties() throws Exception { - assertEquals(1, resolveDistributorMergeBusyWaitConfig(Optional.empty())); - assertEquals(5, resolveDistributorMergeBusyWaitConfig(Optional.of(5))); - } - - private int resolveDistributorMergeBusyWaitConfig(Optional<Integer> mergeBusyWait) throws Exception { - var props = new TestProperties(); - if (mergeBusyWait.isPresent()) { - props.setDistributorMergeBusyWait(mergeBusyWait.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).inhibit_merge_sending_on_busy_node_duration_sec(); - } - - @Test - public void distributor_enhanced_maintenance_scheduling_controlled_by_properties() throws Exception { - assertFalse(resolveDistributorEnhancedSchedulingConfig(Optional.of(false))); - assertTrue(resolveDistributorEnhancedSchedulingConfig(Optional.empty())); + public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { + assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); + assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); } - private boolean resolveDistributorEnhancedSchedulingConfig(Optional<Boolean> enhancedScheduling) throws Exception { + private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { var props = new TestProperties(); - if (enhancedScheduling.isPresent()) { - props.distributorEnhancedMaintenanceScheduling(enhancedScheduling.get()); + if (unorderedMergeChaining.isPresent()) { + props.setUnorderedMergeChaining(unorderedMergeChaining.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).implicitly_clear_bucket_priority_on_schedule(); + return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); } @Test - public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { - assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); - assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); + public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { + assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); + assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); + assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); } - private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { + private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { var props = new TestProperties(); - if (unorderedMergeChaining.isPresent()) { - props.setUnorderedMergeChaining(unorderedMergeChaining.get()); + if (inhibitDefaultMerges.isPresent()) { + props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); + return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 68e722f45d3..ff399fd2294 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -77,12 +77,6 @@ public class ContentSchemaClusterTest { return new ProtonConfig(builder); } - private static ContentCluster createClusterWithFeatureFlag(String clusterXml, boolean enableFeedBlockInDistributor) throws Exception { - var deployStateBuilder = new DeployState.Builder().properties( - new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)); - return createCluster(clusterXml, deployStateBuilder); - } - private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { assertProtonResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); } @@ -124,7 +118,7 @@ public class ContentSchemaClusterTest { @Test public void requireThatOnlyMemoryLimitCanBeSet() throws Exception { - assertProtonResourceLimits(0.9, 0.77, + assertProtonResourceLimits(0.875, 0.77, new ContentClusterBuilder().protonMemoryLimit(0.77).getXml()); } @@ -142,17 +136,10 @@ public class ContentSchemaClusterTest { } @Test - public void default_resource_limits_when_feed_block_is_disabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), false); - assertProtonResourceLimits(0.8, 0.8, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); - } - - @Test - public void default_resource_limits_when_feed_block_is_enabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), true); - assertProtonResourceLimits(0.9, 0.9, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); + public void default_resource_limits_with_feed_block_in_distributor() throws Exception { + var cluster = createCluster(new ContentClusterBuilder().getXml()); + assertProtonResourceLimits(0.875, 0.9, cluster); + assertClusterControllerResourceLimits(0.75, 0.8, cluster); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 90e3aff0c45..d23dca65dd1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -107,7 +107,7 @@ public class DistributorTest { assertEquals(1024, conf.splitcount()); assertEquals(512, conf.joincount()); - assertEquals(33544432, conf.splitsize()); + assertEquals(16772216, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(8, conf.minsplitcount()); assertTrue(conf.inlinebucketsplitting()); @@ -138,7 +138,7 @@ public class DistributorTest { assertEquals(1024, conf.splitcount()); assertEquals(512, conf.joincount()); - assertEquals(33544432, conf.splitsize()); + assertEquals(16772216, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(14, conf.minsplitcount()); assertTrue(conf.inlinebucketsplitting()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index b27a8a192bc..f9fcb777408 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -19,14 +19,12 @@ public class FleetControllerClusterTest { private ClusterControllerConfig parse(String xml, TestProperties props) { Document doc = XML.getDocument(xml); var deployState = new DeployState.Builder().properties(props).build(); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, - new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, + new ClusterResourceLimits.Builder(false, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) @@ -34,7 +32,7 @@ public class FleetControllerClusterTest { } private ClusterControllerConfig parse(String xml) { - return parse(xml, new TestProperties().enableFeedBlockInDistributor(true)); + return parse(xml, new TestProperties()); } @Test @@ -104,17 +102,16 @@ public class FleetControllerClusterTest { assertEquals(0.0, config.min_node_ratio_per_group(), 0.01); } - @Test public void default_cluster_feed_block_limits_are_set() { - assertLimits(0.8, 0.8, getConfigForBasicCluster()); + assertLimits(0.75, 0.8, getConfigForBasicCluster()); } @Test public void resource_limits_can_be_set_in_tuning() { assertLimits(0.6, 0.7, getConfigForResourceLimitsTuning(0.6, 0.7)); assertLimits(0.6, 0.8, getConfigForResourceLimitsTuning(0.6, null)); - assertLimits(0.8, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); + assertLimits(0.75, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); } private static final double DELTA = 0.00001; @@ -124,7 +121,7 @@ public class FleetControllerClusterTest { assertEquals(3, limits.size()); assertEquals(expDisk, limits.get("disk"), DELTA); assertEquals(expMemory, limits.get("memory"), DELTA); - assertEquals(0.89, limits.get("attribute-address-space"), DELTA); + assertEquals(0.9, limits.get("attribute-address-space"), DELTA); } private FleetcontrollerConfig getConfigForResourceLimitsTuning(Double diskLimit, Double memoryLimit) { @@ -143,17 +140,6 @@ public class FleetControllerClusterTest { } @Test - public void feature_flag_controls_enable_cluster_feed_block() { - verifyThatFeatureFlagControlsEnableClusterFeedBlock(true); - verifyThatFeatureFlagControlsEnableClusterFeedBlock(false); - } - - private void verifyThatFeatureFlagControlsEnableClusterFeedBlock(boolean flag) { - var config = getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(flag)); - assertEquals(flag, config.enable_cluster_feed_block()); - } - - @Test public void feature_flag_controls_min_node_ratio_per_group() { verifyFeatureFlagControlsMinNodeRatioPerGroup(0.0, new TestProperties()); verifyFeatureFlagControlsMinNodeRatioPerGroup(0.3, @@ -175,6 +161,6 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForBasicCluster() { - return getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(true)); + return getConfigForBasicCluster(new TestProperties()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index cea8d724c30..bcb197ed540 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -444,7 +444,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { for (SearchClusterSpec cluster : searchClusterSpecs) { for (SearchDefSpec def : cluster.searchDefs) { - sds.add(ApplicationPackageUtils.generateSearchDefinition(def.typeName, def.field1Name, def.field2Name)); + sds.add(ApplicationPackageUtils.generateSchema(def.typeName, def.field1Name, def.field2Name)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index cf877d3bf88..509055df7f4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -160,21 +160,22 @@ public class StorageClusterTest { } @Test - public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() { - var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true)); - assertTrue(config.disable_queue_limits_for_chained_merges()); - - config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false)); - assertFalse(config.disable_queue_limits_for_chained_merges()); + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); } @Test - public void async_apply_bucket_diff_can_be_controlled_by_feature_flag() { - var config = filestorConfigFromProperties(new TestProperties()); - assertTrue(config.async_apply_bucket_diff()); + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); - config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(false)); - assertFalse(config.async_apply_bucket_diff()); + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); } @Test @@ -308,20 +309,40 @@ public class StorageClusterTest { @Test public void persistence_async_throttle_config_defaults_to_unlimited() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); } @Test public void persistence_async_throttle_config_is_derived_from_flag() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); // Invalid enum values fall back to the default config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); + } + + @Test + public void persistence_dynamic_throttling_parameters_have_sane_defaults() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); + } + + @Test + public void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties() + .setPersistenceThrottlingWsDecrementFactor(1.5) + .setPersistenceThrottlingWsBackoff(0.8))); + assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001); } @Test @@ -432,8 +453,8 @@ public class StorageClusterTest { ContentClusterUtils.createCluster(xml, root); fail("Did not fail when having both group and nodes"); } catch (RuntimeException e) { - e.printStackTrace(); - assertEquals("Both group and nodes exists, only one of these tags is legal", e.getMessage()); + assertEquals("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSchemaBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentApplicationBuilderTest.java index b8788ece7b6..11aeea5f90d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSchemaBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentApplicationBuilderTest.java @@ -15,7 +15,7 @@ import static org.junit.Assert.assertNull; /** * @author Simon Thoresen Hult */ -public class DomContentSchemaBuilderTest { +public class DomContentApplicationBuilderTest { @Test public void requireThatDefaultsAreNull() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 7627ba6319b..c60817704cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.handler.ModelsEvaluationHandler; @@ -30,6 +31,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * Tests stateless model evaluation (turned on by the "model-evaluation" tag in "container") @@ -61,6 +63,7 @@ public class ModelEvaluationTest { @Test public void testMl_serving() throws IOException { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/ml_serving"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 6e096dd68e4..bad6fcc68c2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * Tests stateless model evaluation (turned on by the "model-evaluation" tag in "container") @@ -39,7 +41,8 @@ import static org.junit.Assert.assertTrue; public class StatelessOnnxEvaluationTest { @Test - public void testStatelessOnnxModelNameCollision() throws IOException { + public void testStatelessOnnxModelNameCollision() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx_name_collision"); try { ImportedModelTester tester = new ImportedModelTester("onnx", appDir); @@ -60,6 +63,7 @@ public class StatelessOnnxEvaluationTest { @Test public void testStatelessOnnxModelEvaluation() throws IOException { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 3d4991eebed..30678205e3a 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.junit.Ignore; import org.junit.Test; import java.io.BufferedReader; @@ -36,7 +35,6 @@ public class RoutingTestCase { private static final boolean WRITE_FILES = false; @Test - @Ignore // TODO: Why? public void testRoutingContent() throws IOException { assertApplication(new File("src/test/cfg/routing/contentsimpleconfig")); assertApplication(new File("src/test/cfg/routing/content_two_clusters")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index 6830ce79f9f..e58c29cc4fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.component.ComponentId; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.QrSearchersConfig; import com.yahoo.document.DataType; import com.yahoo.search.config.ClusterConfig; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -40,7 +41,7 @@ public class SchemaClusterTest { public void testSdConfigLogical() { // sd1 SDDocumentType sdt1 = new SDDocumentType("s1"); - Schema schema1 = new Schema("s1"); + Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); SDField f1 = new SDField("f1", DataType.STRING); f1.addAttribute(new Attribute("f1", DataType.STRING)); f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); @@ -49,17 +50,17 @@ public class SchemaClusterTest { // sd2 SDDocumentType sdt2 = new SDDocumentType("s2"); - Schema schema2 = new Schema("s2"); + Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); SDField f2=new SDField("f2", DataType.STRING); f2.addAttribute(new Attribute("f2", DataType.STRING)); f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); schema2.addDocument(sdt2); - SchemaBuilder builder = new SchemaBuilder(); - builder.importRawSchema(schema1); - builder.importRawSchema(schema2); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.add(schema1); + builder.add(schema2); + builder.build(true); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index 226045f4d8a..a347c0a9c43 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -40,11 +40,11 @@ public class SchemaNodeTest { TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); - tls.initService(root.deployLogger()); + tls.initService(root.getDeployState()); node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); root.freezeModelTopology(); } @@ -78,7 +78,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); } @@ -87,7 +87,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); assertTrue(node.getPreShutdownCommand().get().contains("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 3388f9d2670..3de8cfe540f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.test; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRegistry; @@ -10,6 +9,7 @@ import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.model.admin.AdminModel; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -152,13 +152,13 @@ public class ModelAmendingTestCase { @Override public void doBuild(AdminModelAmender model, Element spec, ConfigModelContext modelContext) { for (AdminModel adminModel : model.adminModels) - amend(modelContext.getDeployLogger(), adminModel); + amend(modelContext.getDeployState(), adminModel); } - private void amend(DeployLogger deployLogger, AdminModel adminModel) { + private void amend(DeployState deployState, AdminModel adminModel) { for (HostResource host : adminModel.getAdmin().hostSystem().getHosts()) { if ( ! host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()) continue; // already amended - adminModel.getAdmin().addAndInitializeService(deployLogger, host, new AmendedService(host.getHost())); + adminModel.getAdmin().addAndInitializeService(deployState, host, new AmendedService(host.getHost())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index bf4d066e454..7f8bca825d2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -242,7 +242,7 @@ public class VespaModelTestCase { .build(); DeployState deployState = builder.deployLogger(logger).applicationPackage(app).build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE), deployState); + new Validation().validate(model, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE), deployState); assertFalse(logger.msgs.isEmpty()); } @@ -312,7 +312,7 @@ public class VespaModelTestCase { .deployLogger(logger) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); assertContainsWarning(logger.msgs, "Directory searchdefinitions/ should not be used for schemas, use schemas/ instead"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 1992b01788f..3a5bb3dc1b3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -11,10 +11,9 @@ import java.util.List; */ public class ApplicationPackageUtils { - public static String generateSearchDefinition(String name, String field1, String field2) { - return "" + - "search " + name + "{" + - " document " + name + "{" + + public static String generateSchema(String name, String field1, String field2) { + return "schema " + name + " {" + + " document " + name + " {" + " field " + field1 + " type string {\n" + " indexing: index | summary\n" + " summary: dynamic\n" + @@ -51,7 +50,7 @@ public class ApplicationPackageUtils { List<String> sds = new ArrayList<>(); int i = 0; for (String sdName : sdNames) { - sds.add(generateSearchDefinition(sdName, "f" + (i + 1), "f" + (i + 2))); + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2))); i = i + 2; } return sds; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 351b080eb5b..565e4c7c076 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -66,7 +66,7 @@ public class VespaModelCreatorWithFilePkg { // is constructed in a special way and cannot always be validated in // this step for unit tests) ValidationParameters validationParameters = new ValidationParameters(IgnoreValidationErrors.TRUE, FailOnIncompatibleChange.TRUE, CheckRouting.FALSE); - Validation.validate(model, validationParameters, deployState); + new Validation().validate(model, validationParameters, deployState); return model; } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 66dc63bbb02..d777fefbbe5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -75,7 +75,7 @@ public class VespaModelCreatorWithMockPkg { // is constructed in a special way and cannot always be validated in // this step for unit tests) ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); - configChangeActions = Validation.validate(model, validationParameters, deployState); + configChangeActions = new Validation().validate(model, validationParameters, deployState); } return model; } catch (Exception e) { diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index a3c2e7671e8..fdfbbfc9049 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -21,6 +21,7 @@ Provisioning APIs. <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index a617c7bcea3..25ec7485c24 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -32,10 +32,7 @@ public enum NodeType { controller("Controller node"), /** Host of a controller node */ - controllerhost("Controller host", controller), - - /** Host capable of running multiple node types, only used in {@link SystemName#dev} */ - devhost("Dev host", config, controller, tenant); + controllerhost("Controller host", controller); private final String description; private final List<NodeType> childNodeTypes; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java index 3a5f136642b..0cb7acf8199 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -17,7 +17,7 @@ public enum RoutingMethod { /** Routing happens through a shared layer 4 load balancer */ sharedLayer4; - /** Returns whether this method routes requests directly to the Vespa container cluster */ + /** Returns whether this method uses layer 4 routing */ public boolean isDirect() { return this == exclusive || this == sharedLayer4; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index 36d364fef94..b50d8ce2e7c 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -13,11 +13,22 @@ public interface ZoneApi { SystemName getSystemName(); + /** + * Returns the ID of the zone. + * + * WARNING: The ID of a controller zone is equal to the ID of a prod zone in the same region. + * @see #getVirtualId() + */ ZoneId getId(); + /** Returns the SYSTEM.ENVIRONMENT.REGION string. */ + default String getFullName() { + return getSystemName().value() + "." + getEnvironment().value() + "." + getRegionName().value(); + } + /** - * Returns the virtual ID of this zone. For ordinary zones this is the same as {@link ZoneApi#getId()}, for a - * system represented as a zone this is a fixed ID that is independent of the actual zone ID. + * Returns {@link #getId()} for all zones except the controller zone. Unlike {@link #getId()}, + * the virtual ID of a controller is distinct from all other zones. */ default ZoneId getVirtualId() { return getId(); diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index a855d84de71..a1dd174d4e1 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -91,17 +91,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 0ae02e4c17b..9340363def8 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -33,6 +33,7 @@ import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; * * @author hmusum */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class ProxyServer implements Runnable { private static final DaemonThreadFactory threadFactory = new DaemonThreadFactory("ProxyServer"); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 56fdae477b2..c09a868a786 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -45,6 +45,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final Supervisor supervisor = new Supervisor(new Transport("config-source-client")); private final ResponseHandler responseHandler; + @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSourceSet configSourceSet; private final Object subscribersLock = new Object(); private final Map<ConfigCacheKey, Subscriber> subscribers = new ConcurrentHashMap<>(); @@ -59,6 +60,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); private final ScheduledFuture<?> delayedResponsesFuture; + @SuppressWarnings("removal") // TODO Vespa 8: remove RpcConfigSourceClient(ResponseHandler responseHandler, ConfigSourceSet configSourceSet) { this.responseHandler = responseHandler; this.configSourceSet = configSourceSet; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 68570722117..8e471aa1f5d 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -26,6 +26,7 @@ public class FileDistributionAndUrlDownload { private final ScheduledExecutorService cleanupExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); + @SuppressWarnings("removal") // TODO Vespa 8: remove public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source)); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); @@ -44,6 +45,7 @@ public class FileDistributionAndUrlDownload { } } + @SuppressWarnings("removal") // TODO Vespa 8: remove private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), supervisor, diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index d8459b175a7..a7f6a2a97a7 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -106,7 +106,7 @@ export LD_LIBRARY_PATH="$VESPA_HOME/lib64" case $1 in start) - nohup sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & + nohup nice sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS jvmopts="-Xms32M -Xmx128M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index f743c0ed231..2105bd72975 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -256,6 +256,7 @@ public class ConfigProxyRpcServerTest { assertEquals("success", req.returnValues().get(0).asString()); } + @SuppressWarnings("removal") // TODO Vespa 8: remove private static ProxyServer createTestServer(ConfigSourceSet source) { return new ProxyServer(null, source, new RpcConfigSourceClient(new ResponseHandler(), source)); } @@ -264,6 +265,7 @@ public class ConfigProxyRpcServerTest { private static final Spec SPEC = new Spec(0); + @SuppressWarnings("removal") // TODO Vespa 8: remove private final ProxyServer proxyServer = createTestServer(new ConfigSourceSet(configSourceAddress)); private final Supervisor supervisor = new Supervisor(new Transport()); private final ConfigProxyRpcServer rpcServer = new ConfigProxyRpcServer(proxyServer, supervisor, SPEC); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java index 0377e20330a..2b3624fe2b8 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java @@ -15,6 +15,7 @@ import java.util.Set; * * @author hmusum */ +@SuppressWarnings("removal") // TODO Vespa 8: remove class MockConfigSource extends ConfigSourceSet { private final HashMap<ConfigKey<?>, RawConfig> backing = new HashMap<>(); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index ee058ba903e..09eae6a297d 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -222,6 +222,7 @@ public class ProxyServerTest { assertEquals(ProxyServer.DEFAULT_PROXY_CONFIG_SOURCES, properties.configSources[0]); } + @SuppressWarnings("removal") // TODO Vespa 8: remove private static ProxyServer createTestServer(ConfigSourceSet source, ConfigSourceClient configSourceClient) { return new ProxyServer(null, source, configSourceClient); } diff --git a/config/abi-spec.json b/config/abi-spec.json index 844835ae1c5..e94749cfba8 100644 --- a/config/abi-spec.json +++ b/config/abi-spec.json @@ -297,5 +297,16 @@ "fields": [ "public final java.lang.String payload" ] + }, + "com.yahoo.config.subscription.SubscriberClosedException": { + "superClass": "java.lang.RuntimeException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()" + ], + "fields": [] } }
\ No newline at end of file diff --git a/config/pom.xml b/config/pom.xml index 05b0bd758f8..546e65c048c 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -25,13 +25,14 @@ <artifactId>guava</artifactId> <scope>provided</scope> </dependency> - - <!-- compile scope --> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> + + <!-- compile scope --> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-lib</artifactId> @@ -185,6 +186,14 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> + <!-- TODO Vespa 8: remove configuration. + Included to allow 'removal' warnings for classes in its own module --> + <configuration> + <compilerArgs> + <arg>-Xlint:all,-serial,-try,-processing,-removal</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index e6375ac9c47..9bbc14552b0 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -2,10 +2,10 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> -#include <vespa/config/config.h> #include <vespa/config/frt/frtconfigrequestfactory.h> #include <vespa/config/frt/frtconnection.h> #include <vespa/config/common/payload_converter.h> +#include <vespa/config/common/configvalue.h> #include <vespa/fastos/app.h> #include <string> @@ -92,8 +92,8 @@ GetConfig::Main() bool debugging = false; int c = -1; - std::vector<vespalib::string> defSchema; - const char *schema = nullptr; + StringVector defSchema; + const char *schemaString = nullptr; const char *defName = nullptr; const char *defMD5 = ""; std::string defNamespace("config"); @@ -119,7 +119,7 @@ GetConfig::Main() int retval = 1; switch (c) { case 'a': - schema = optArg; + schemaString = optArg; break; case 'n': defName = optArg; @@ -184,17 +184,32 @@ GetConfig::Main() defNamespace = std::string(tmp, defName - tmp - 1); } - if (schema != nullptr) { - std::ifstream is; - is.open(schema); - std::string item; - while (std::getline(is, item)) { - if (item.find("namespace=") == std::string::npos) { - defSchema.push_back(item); - } - } - is.close(); + std::string schema; + if (schemaString == nullptr) { + std::ostringstream tmp; + tmp << getenv("VESPA_HOME"); + tmp << "/share/vespa/configdefinitions/"; + tmp << defNamespace; + tmp << "."; + tmp << defName; + tmp << ".def"; + schema = tmp.str(); + } else { + schema = schemaString; } + if (debugging) { + printf("Using schema in %s\n", schema.c_str()); + } + std::ifstream is; + is.open(schema); + std::string item; + while (std::getline(is, item)) { + if (item.find("namespace=") == std::string::npos) { + defSchema.push_back(item); + } + } + is.close(); + std::ostringstream tmp; tmp << "tcp/"; tmp << serverHost; @@ -228,8 +243,8 @@ GetConfig::Main() } else { response->fill(); ConfigKey rKey(response->getKey()); - ConfigState rState(response->getConfigState()); - ConfigValue rValue(response->getValue()); + const ConfigState & rState = response->getConfigState(); + const ConfigValue & rValue = response->getValue(); if (debugging) { printf("defName %s\n", rKey.getDefName().c_str()); printf("defMD5 %s\n", rKey.getDefMd5().c_str()); @@ -247,7 +262,7 @@ GetConfig::Main() if (printAsJson) { printf("%s\n", rValue.asJson().c_str()); } else { - std::vector<vespalib::string> lines = rValue.getLegacyFormat(); + StringVector lines = rValue.getLegacyFormat(); for (uint32_t j = 0; j < lines.size(); j++) { printf("%s\n", lines[j].c_str()); } diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 1a110fe8e6c..7d3fdad43c7 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -17,7 +17,9 @@ import static java.util.logging.Level.FINEST; * Deserializes config payload (cfg format) to a ConfigPayload. * * @author hmusum + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class CfgConfigPayloadBuilder { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(CfgConfigPayloadBuilder.class.getName()); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index 85aa7a05e17..2fcbdce7030 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -9,6 +9,7 @@ import java.util.logging.Logger; import static java.util.logging.Level.INFO; // Debug class that provides useful helper routines +@Deprecated(forRemoval = true, since = "7") public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey<?> key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java index f349c8fe47d..305ecbd6483 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -34,7 +34,10 @@ public class ConfigGetter<T extends ConfigInstance> { * * @param source a {@link ConfigSource} * @param clazz a config class + * + * @deprecated Config should always be injected via the component class constructor. For unit tests, use config builders. */ + @Deprecated(forRemoval = true, since = "7") public ConfigGetter(ConfigSource source, Class<T> clazz) { this.clazz = clazz; this.source = source; @@ -74,7 +77,10 @@ public class ConfigGetter<T extends ConfigInstance> { * @param configId a config id to use when getting the config * @param source a {@link ConfigSource} * @return an instance of a config class + * + * @deprecated Config should always be injected via the component class constructor. For unit tests, use config builders. */ + @Deprecated(forRemoval = true, since = "7") public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId, ConfigSource source) { ConfigGetter<T> getter = new ConfigGetter<>(source, c); return getter.getConfig(configId); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java index c0eff773bac..6b832205211 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java @@ -10,7 +10,9 @@ import com.yahoo.config.subscription.impl.ConfigSubscription; * * @param <T> the type of the config * @author vegardh + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigHandle<T extends ConfigInstance> { private final ConfigSubscription<T> sub; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java index e035991abf4..d8062a9f95b 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -9,7 +9,9 @@ import com.yahoo.slime.Slime; * Implements a config instance serializer, serializing a config instance to a slime object. * * @author Ulf Lilleengen + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigInstanceSerializer implements Serializer { private final Slime slime; private final Cursor root; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java index f1b7952f3b1..5831753eacb 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -14,7 +14,9 @@ import com.yahoo.vespa.config.*; /** * @author gjoranv + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigInstanceUtil { /** diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java index a7c2c2d9127..050b7a81fa2 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -4,8 +4,10 @@ package com.yahoo.config.subscription; /** * This exception is thrown when any blocking call within the Config API is interrupted. * @author Ulf Lilleengen + * @deprecated Will be removed in Vespa 8. Only for internal use. */ @SuppressWarnings("serial") +@Deprecated(forRemoval = true, since = "7") public class ConfigInterruptedException extends RuntimeException { public ConfigInterruptedException(Throwable cause) { super(cause); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java index 02cc87f65d7..36d9047aa87 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -11,7 +11,9 @@ import com.yahoo.vespa.config.ConfigKey; * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s * * @author Vegard Havdal + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigSet implements ConfigSource { private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java index 5814a004216..b8abeb48c3d 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -5,7 +5,9 @@ package com.yahoo.config.subscription; * A type of source of config * * @author Vegard Havdal + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public interface ConfigSource { } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 81aeda52b54..37f91605a54 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -18,7 +18,9 @@ import static java.util.logging.Level.INFO; * upper/lower-casing and whitespaces. * * @author gjoranv + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigSourceSet implements ConfigSource { private static final Logger log = Logger.getLogger(ConfigSourceSet.class.getName()); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 4bf442a0890..ee6ce37036c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -28,7 +28,9 @@ import static java.util.stream.Collectors.toList; * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. * * @author Vegard Havdal + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigSubscriber implements AutoCloseable { private static final Logger log = Logger.getLogger(ConfigSubscriber.class.getName()); @@ -154,6 +156,7 @@ public class ConfigSubscriber implements AutoCloseable { * false if this is for reconfiguration * @return true if a config/reconfig of your system should happen * @throws ConfigInterruptedException if thread performing this call interrupted. + * @throws SubscriberClosedException if subscriber is closed */ public boolean nextConfig(boolean isInitializing) { return nextConfig(TimingValues.defaultNextConfigTimeout, isInitializing); @@ -185,6 +188,7 @@ public class ConfigSubscriber implements AutoCloseable { * false if this is for reconfiguration * @return true if a config/reconfig of your system should happen * @throws ConfigInterruptedException if thread performing this call interrupted. + * @throws SubscriberClosedException if subscriber is closed */ public boolean nextConfig(long timeoutMillis, boolean isInitializing) { return acquireSnapshot(timeoutMillis, true, isInitializing); @@ -216,6 +220,7 @@ public class ConfigSubscriber implements AutoCloseable { * false if this is for reconfiguration * @return true if generations for all configs have been updated. * @throws ConfigInterruptedException if thread performing this call interrupted. + * @throws SubscriberClosedException if subscriber is closed */ public boolean nextGeneration(boolean isInitializing) { return nextGeneration(TimingValues.defaultNextConfigTimeout, isInitializing); @@ -247,6 +252,7 @@ public class ConfigSubscriber implements AutoCloseable { * false if this is for reconfiguration * @return true if generations for all configs have been updated. * @throws ConfigInterruptedException if thread performing this call interrupted. + * @throws SubscriberClosedException if subscriber is closed */ public boolean nextGeneration(long timeoutMillis, boolean isInitializing) { return acquireSnapshot(timeoutMillis, false, isInitializing); @@ -269,11 +275,12 @@ public class ConfigSubscriber implements AutoCloseable { private boolean acquireSnapshot(long timeoutInMillis, boolean requireChange, boolean isInitializing) { boolean applyOnRestartOnly; synchronized (monitor) { - if (state == State.CLOSED) return false; + if (state == State.CLOSED) throw new SubscriberClosedException(); state = State.FROZEN; applyOnRestartOnly = applyOnRestart; } - long started = System.currentTimeMillis(); + boolean expiredOnEntry = (timeoutInMillis <= 0); + long started = now(expiredOnEntry); long timeLeftMillis = timeoutInMillis; boolean anyConfigChanged = false; @@ -302,7 +309,7 @@ public class ConfigSubscriber implements AutoCloseable { allGenerationsChanged &= config.isGenerationChanged(); anyConfigChanged |= config.isConfigChanged(); applyOnRestartOnly |= config.applyOnRestart(); - timeLeftMillis = timeoutInMillis + started - System.currentTimeMillis(); + timeLeftMillis = timeoutInMillis + started - now(expiredOnEntry); } reconfigDue = (isInitializing || !applyOnRestartOnly) && (anyConfigChanged || !requireChange) && allGenerationsChanged && allGenerationsTheSame; @@ -329,6 +336,10 @@ public class ConfigSubscriber implements AutoCloseable { return reconfigDue; } + private long now(boolean alreadyExpired) { + return alreadyExpired ? 0 : System.currentTimeMillis(); + } + private void sleep(long timeLeftMillis) { try { Thread.sleep(Math.min(10, timeLeftMillis)); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java index 6053ba5303a..46db3c6c83a 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java @@ -10,7 +10,9 @@ import com.yahoo.config.subscription.impl.JRTConfigRequester; * object to simplify parameter passing. * * @author Ulf Lilleengen + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class ConfigURI { private String configId; diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java index 240d1b8e81d..48d52bd49f3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -6,8 +6,9 @@ import java.io.File; /** * Source specifying config from a local directory * @author Vegard Havdal - * + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class DirSource implements ConfigSource { private final File dir; diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java index 2178c64cca3..bbad7286b4c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -7,7 +7,9 @@ import java.io.File; * Source specifying config from one local file * * @author Vegard Havdal + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class FileSource implements ConfigSource { private final File file; diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java index e367f78c31f..f39ee5e9262 100644 --- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -6,8 +6,9 @@ import java.util.jar.JarFile; /** * Source specifying config as a jar file entry * @author Vegard Havdal - * + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class JarSource implements ConfigSource { private final String path; private final JarFile jarFile; diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java index 8b73d2058f5..28b249c6ece 100644 --- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -5,7 +5,9 @@ package com.yahoo.config.subscription; * Source specifying raw config, where payload is given programmatically * * @author Vegard Havdal + * @deprecated Will be removed in Vespa 8. Only for internal use. */ +@Deprecated(forRemoval = true, since = "7") public class RawSource implements ConfigSource { public final String payload; diff --git a/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java b/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java new file mode 100644 index 00000000000..f7051ab1b38 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * Thrown when {@link ConfigSubscriber} is closed + * + * @author bjorncs + * @deprecated Will be removed in Vespa 8. Only for internal use. + */ +@Deprecated(forRemoval = true, since = "7") +public class SubscriberClosedException extends RuntimeException {} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java index 85bb1cd4ca7..68f1dc2df17 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -29,23 +29,31 @@ public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubsc setGeneration(0L); } + private boolean hasConfigChanged() { + T myInstance = getNewInstance(); + ConfigState<T> configState = getConfigState(); + // User forced reload + if (checkReloaded()) { + setConfigIfChanged(myInstance); + return true; + } + if (!myInstance.equals(configState.getConfig())) { + setConfigIncGen(myInstance); + return true; + } + return false; + } + @Override public boolean nextConfig(long timeout) { - long end = System.currentTimeMillis() + timeout; + if (hasConfigChanged()) return true; + if (timeout <= 0) return false; + + long end = System.nanoTime() + timeout * 1_000_000; do { - T myInstance = getNewInstance(); - ConfigState<T> configState = getConfigState(); - // User forced reload - if (checkReloaded()) { - setConfigIfChanged(myInstance); - return true; - } - if (!myInstance.equals(configState.getConfig())) { - setConfigIncGen(myInstance); - return true; - } sleep(); - } while (System.currentTimeMillis() < end); + if (hasConfigChanged()) return true; + } while (System.nanoTime() < end); return false; } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index f8a45a11b70..caa11b211ac 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -10,6 +10,7 @@ import com.yahoo.config.subscription.DirSource; import com.yahoo.config.subscription.FileSource; import com.yahoo.config.subscription.JarSource; import com.yahoo.config.subscription.RawSource; +import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.TimingValues; @@ -198,13 +199,14 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { } void setConfigAndGeneration(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) { - ConfigState<T> prev = this.config.get(); - boolean configChanged = !Objects.equals(prev.getConfig(), config); - String message = "Config has changed unexpectedly for " + key + ", generation " + generation; - if (configChanged) { - if (log.isLoggable(Level.FINE)) - message = message + ", config in state :" + prev.getConfig() + ", new config: " + config; - log.log(Level.WARNING, message); + T previousConfig = this.config.get().getConfig(); + boolean configChanged = ! Objects.equals(previousConfig, config); + if (previousConfig != null && configChanged) { + SnippetGenerator generator = new SnippetGenerator(); + int sizeHint = 500; + log.log(Level.WARNING, "Config has changed unexpectedly for " + key + ", generation " + generation + + ", config in state :" + generator.makeSnippet(previousConfig.toString(), sizeHint) + ", new config: " + + generator.makeSnippet(config.toString(), sizeHint)); } this.config.set(new ConfigState<>(true, generation, applyOnRestart, configChanged, config, payloadChecksums)); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index a18f1b4b260..90c538ac19b 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -72,12 +72,12 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc JRTClientConfigRequest response = pollQueue(timeoutMillis); // There might be more than one response on the queue, so empty queue by polling with // 0 timeout until queue is empty (returned value is null) - JRTClientConfigRequest temp; - do { + JRTClientConfigRequest temp = response; + while (temp != null) { temp = pollQueue(0); if (temp != null) response = temp; - } while (temp != null); + } return response; } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java index 9a7ac58e339..2fccabe111b 100755 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -108,16 +108,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable return configClass; } - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("name="); - sb.append(name); - sb.append(",namespace="); - sb.append(namespace); - sb.append(",configId="); - sb.append(configId); - return sb.toString(); - } + public String toString() { return "name=" + namespace + "." + name + ",configId=" + configId; } public static ConfigKey<?> createFull(String name, String configId, String namespace) { return new ConfigKey<>(name, configId, namespace, null); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java index c9e7bb23402..c141ed8403c 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java @@ -18,7 +18,7 @@ public class ConfigSetTest { ConfigSet set = new ConfigSet(); SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); set.addBuilder("foo", builder); - assertTrue(Pattern.matches("name=simpletypes,namespace=foo,configId=foo=>com.yahoo.foo.SimpletypesConfig.*", + assertTrue(Pattern.matches("name=foo.simpletypes,configId=foo=>com.yahoo.foo.SimpletypesConfig.*", set.toString())); } } diff --git a/config/src/tests/api/api.cpp b/config/src/tests/api/api.cpp index 9cf6ea610bf..3377d256b97 100644 --- a/config/src/tests/api/api.cpp +++ b/config/src/tests/api/api.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/configcontext.h> #include <config-my.h> +#include <vespa/config/subscription/configsubscriber.hpp> using namespace config; diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index 3fbbee8c601..10f321c181d 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> -#include <vespa/config/raw/rawsource.h> #include <vespa/config/common/misc.h> #include <vespa/config/common/configrequest.h> #include <vespa/config/common/timingvalues.h> #include <vespa/config/common/trace.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configholder.h> #include <vespa/config/frt/frtconfigagent.h> #include <config-my.h> @@ -30,10 +30,10 @@ public: class MyConfigResponse : public ConfigResponse { public: - MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool valid, int64_t timestamp, + MyConfigResponse(const ConfigKey & key, ConfigValue value, bool valid, int64_t timestamp, const vespalib::string & xxhash64, const std::string & errorMsg, int errorC0de, bool iserror) : _key(key), - _value(value), + _value(std::move(value)), _fillCalled(false), _valid(valid), _state(xxhash64, timestamp, false), @@ -83,10 +83,8 @@ public: class MyHolder : public IConfigHolder { public: - MyHolder() - : _update() - { - } + MyHolder() noexcept = default; + ~MyHolder() = default; std::unique_ptr<ConfigUpdate> provide() override { @@ -116,7 +114,7 @@ private: ConfigValue createValue(const std::string & myField, const std::string & xxhash64) { - std::vector< vespalib::string > lines; + StringVector lines; lines.push_back("myField \"" + myField + "\""); return ConfigValue(lines, xxhash64); } @@ -135,7 +133,7 @@ static TimingValues testTimingValues( 2000); // maxDelayMultiplier TEST("require that agent returns correct values") { - FRTConfigAgent handler(IConfigHolder::SP(new MyHolder()), testTimingValues); + FRTConfigAgent handler(std::make_shared<MyHolder>(), testTimingValues); ASSERT_EQUAL(500u, handler.getTimeout()); ASSERT_EQUAL(0u, handler.getWaitTime()); ConfigState cs; @@ -147,12 +145,12 @@ TEST("require that agent returns correct values") { TEST("require that successful request is delivered to holder") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue(createValue("l33t", "a")); - IConfigHolder::SP latch(new MyHolder()); + auto latch = std::make_shared<MyHolder>(); FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); ASSERT_TRUE(latch->poll()); - ConfigUpdate::UP update(latch->provide()); + std::unique_ptr<ConfigUpdate> update(latch->provide()); ASSERT_TRUE(update); ASSERT_TRUE(update->hasChanged()); MyConfig cfg(update->getValue()); @@ -163,13 +161,13 @@ TEST("require that important(the change) request is delivered to holder even if const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue1(createValue("l33t", "a")); const ConfigValue testValue2(createValue("l34t", "b")); - IConfigHolder::SP latch(new MyHolder()); + auto latch = std::make_shared<MyHolder>(); FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getXxhash64())); ASSERT_TRUE(latch->poll()); - ConfigUpdate::UP update(latch->provide()); + std::unique_ptr<ConfigUpdate> update(latch->provide()); ASSERT_TRUE(update); ASSERT_TRUE(update->hasChanged()); MyConfig cfg(update->getValue()); @@ -190,7 +188,7 @@ TEST("require that important(the change) request is delivered to holder even if TEST("require that successful request sets correct wait time") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue(createValue("l33t", "a")); - IConfigHolder::SP latch(new MyHolder()); + auto latch = std::make_shared<MyHolder>(); FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); @@ -203,7 +201,7 @@ TEST("require that successful request sets correct wait time") { TEST("require that bad config response returns false") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue(createValue("myval", "a")); - IConfigHolder::SP latch(new MyHolder()); + auto latch = std::make_shared<MyHolder>(); FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); @@ -241,10 +239,9 @@ TEST("require that bad config response returns false") { TEST("require that bad response returns false") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); - std::vector<vespalib::string> lines; - const ConfigValue testValue(lines, "a"); + const ConfigValue testValue(StringVector(), "a"); - IConfigHolder::SP latch(new MyHolder()); + auto latch = std::make_shared<MyHolder>(); FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); diff --git a/config/src/tests/configfetcher/configfetcher.cpp b/config/src/tests/configfetcher/configfetcher.cpp index 4782557338d..6142f9469fc 100644 --- a/config/src/tests/configfetcher/configfetcher.cpp +++ b/config/src/tests/configfetcher/configfetcher.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/configfetcher.hpp> #include <vespa/config/common/configcontext.h> #include <vespa/vespalib/util/exception.h> #include "config-my.h" @@ -125,7 +125,7 @@ namespace { struct ConfigFixture { MyConfigBuilder builder; ConfigSet set; - ConfigContext::SP context; + std::shared_ptr<ConfigContext> context; ConfigFixture() : builder(), set(), context() { set.addBuilder("cfgid", &builder); context = std::make_shared<ConfigContext>(set); diff --git a/config/src/tests/configformat/configformat.cpp b/config/src/tests/configformat/configformat.cpp index 6623e2ee254..65c40eaea8d 100644 --- a/config/src/tests/configformat/configformat.cpp +++ b/config/src/tests/configformat/configformat.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/print/fileconfigformatter.h> #include <vespa/vespalib/data/slime/slime.h> diff --git a/config/src/tests/configgen/configgen.cpp b/config/src/tests/configgen/configgen.cpp index b7113dec972..2d08b526e8d 100644 --- a/config/src/tests/configgen/configgen.cpp +++ b/config/src/tests/configgen/configgen.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> #include "config-motd.h" diff --git a/config/src/tests/configgen/map_inserter.cpp b/config/src/tests/configgen/map_inserter.cpp index 2a37d401792..bf1b856972e 100644 --- a/config/src/tests/configgen/map_inserter.cpp +++ b/config/src/tests/configgen/map_inserter.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/configgen/map_inserter.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/config/configgen/map_inserter.hpp> using namespace config; using namespace config::internal; diff --git a/config/src/tests/configgen/vector_inserter.cpp b/config/src/tests/configgen/vector_inserter.cpp index b8611317650..80ebf0fc95f 100644 --- a/config/src/tests/configgen/vector_inserter.cpp +++ b/config/src/tests/configgen/vector_inserter.cpp @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/configgen/vector_inserter.h> +#include <vespa/config/common/types.h> +#include <vespa/config/configgen/vector_inserter.hpp> #include <vespa/vespalib/data/slime/slime.h> using namespace config; @@ -27,7 +28,7 @@ TEST("require that vector of ints can be inserted") { root.addLong(3); root.addLong(2); root.addLong(6); - VectorInserter<int32_t> inserter(vector); + VectorInserter inserter(vector); root.traverse(inserter); ASSERT_EQUAL(3u, vector.size()); ASSERT_EQUAL(3, vector[0]); @@ -45,7 +46,7 @@ TEST("require that vector of struct can be inserted") { Cursor & two = root.addObject(); two.setLong("foo", 1); two.setLong("bar", 6); - VectorInserter<MyType> inserter(typeVector); + VectorInserter inserter(typeVector); root.traverse(inserter); ASSERT_EQUAL(2u, typeVector.size()); ASSERT_EQUAL(3, typeVector[0].foo); @@ -61,7 +62,7 @@ TEST("require that vector of long can be inserted") { root.addLong(3); root.addLong(2); root.addLong(6); - VectorInserter<int64_t> inserter(vector); + VectorInserter inserter(vector); root.traverse(inserter); ASSERT_EQUAL(3u, vector.size()); ASSERT_EQUAL(3, vector[0]); @@ -76,7 +77,7 @@ TEST("require that vector of double can be inserted") { root.addDouble(3.1); root.addDouble(2.4); root.addDouble(6.6); - VectorInserter<double> inserter(vector); + VectorInserter inserter(vector); root.traverse(inserter); ASSERT_EQUAL(3u, vector.size()); ASSERT_EQUAL(3.1, vector[0]); @@ -91,7 +92,7 @@ TEST("require that vector of bool can be inserted") { root.addBool(true); root.addBool(false); root.addBool(true); - VectorInserter<bool> inserter(vector); + VectorInserter inserter(vector); root.traverse(inserter); ASSERT_EQUAL(3u, vector.size()); ASSERT_TRUE(vector[0]); @@ -99,14 +100,15 @@ TEST("require that vector of bool can be inserted") { ASSERT_TRUE(vector[2]); } -TEST("require that vector of string can be inserted") { - std::vector<vespalib::string> vector; +template<typename V> +void +verify_vector_strings_can_be_inserted(V vector) { Slime slime; Cursor & root = slime.setArray(); root.addString("foo"); root.addString("bar"); root.addString("baz"); - VectorInserter<vespalib::string> inserter(vector); + VectorInserter inserter(vector); root.traverse(inserter); ASSERT_EQUAL(3u, vector.size()); ASSERT_EQUAL("foo", vector[0]); @@ -114,4 +116,9 @@ TEST("require that vector of string can be inserted") { ASSERT_EQUAL("baz", vector[2]); } +TEST("require that different vectors of strings can be inserted") { + verify_vector_strings_can_be_inserted(std::vector<vespalib::string>()); + verify_vector_strings_can_be_inserted(StringVector()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp index db32cc2e309..b2f6cd83693 100644 --- a/config/src/tests/configholder/configholder.cpp +++ b/config/src/tests/configholder/configholder.cpp @@ -14,8 +14,8 @@ constexpr vespalib::duration ONE_MINUTE = 60s; TEST("Require that element order is correct") { - ConfigValue value(std::vector<vespalib::string>(), "foo"); - ConfigValue value2(std::vector<vespalib::string>(), "bar"); + ConfigValue value(StringVector(), "foo"); + ConfigValue value2(StringVector(), "bar"); ConfigHolder holder; holder.handle(std::make_unique<ConfigUpdate>(value, true, 0)); diff --git a/config/src/tests/configmanager/configmanager.cpp b/config/src/tests/configmanager/configmanager.cpp index 2cc5d4fad4f..a7171b11137 100644 --- a/config/src/tests/configmanager/configmanager.cpp +++ b/config/src/tests/configmanager/configmanager.cpp @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/noncopyable.hpp> #include <vespa/config/common/configmanager.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/common/timingvalues.h> #include <vespa/config/subscription/sourcespec.h> +#include <vespa/config/common/iconfigholder.h> + #include <vespa/config/raw/rawsource.h> #include "config-my.h" @@ -16,7 +17,7 @@ namespace { ConfigValue createValue(const std::string & myField, const std::string & md5) { - std::vector< vespalib::string > lines; + StringVector lines; lines.push_back("myField \"" + myField + "\""); return ConfigValue(lines, md5); } @@ -36,12 +37,12 @@ namespace { class MySource : public Source { public: - MySource(TestContext * data, const IConfigHolder::SP & holder) : _holder(holder), _data(data) { } + MySource(TestContext * data, std::shared_ptr<IConfigHolder> holder) : _holder(std::move(holder)), _data(data) { } void getConfig() override { _data->numGetConfig++; if (_data->respond) { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), true, _data->generation))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), true, _data->generation)); } } void reload(int64_t generation) override @@ -53,7 +54,7 @@ namespace { { _data->numClose++; } - IConfigHolder::SP _holder; + std::shared_ptr<IConfigHolder> _holder; TestContext * _data; }; @@ -61,10 +62,10 @@ namespace { { public: MySourceFactory(TestContext * d) : data(d) { } - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override { (void) key; - return Source::UP(new MySource(data, holder)); + return std::make_unique<MySource>(data, std::move(holder)); } TestContext * data; }; @@ -78,9 +79,9 @@ namespace { { } SourceSpecKey createKey() const { return SourceSpecKey(_key); } - SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const override { + std::unique_ptr<SourceFactory> createSourceFactory(const TimingValues & timingValues) const override { (void) timingValues; - return SourceFactory::UP(new MySourceFactory(_data)); + return std::make_unique<MySourceFactory>(_data); } SourceSpec * clone() const { return new MySpec(*this); } private: diff --git a/config/src/tests/configparser/configparser.cpp b/config/src/tests/configparser/configparser.cpp index 965c2a5a312..32043ae79dc 100644 --- a/config/src/tests/configparser/configparser.cpp +++ b/config/src/tests/configparser/configparser.cpp @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/configparser.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/misc.h> #include "config-foo.h" #include <fstream> #include <vespa/vespalib/stllike/asciistream.h> @@ -23,7 +24,7 @@ namespace { ConfigValue readConfig(const vespalib::string & fileName) { asciistream is(asciistream::createFromFile(fileName)); - return ConfigValue(is.getlines(), ""); + return ConfigValue(getlines(is), ""); } } @@ -102,14 +103,14 @@ TEST("require that array lengths may be specified") } TEST("require that escaped values are properly unescaped") { - std::vector<vespalib::string> payload; + StringVector payload; payload.push_back("foo \"a\\nb\\rc\\\\d\\\"e\x42g\""); vespalib::string value(ConfigParser::parse<vespalib::string>("foo", payload)); ASSERT_EQUAL("a\nb\rc\\d\"eBg", value); } TEST("verify that locale does not affect double parsing") { - std::vector<vespalib::string> payload; + StringVector payload; setlocale(LC_NUMERIC, "nb_NO.UTF-8"); payload.push_back("foo 3,14"); ASSERT_EXCEPTION(ConfigParser::parse<double>("foo", payload), InvalidConfigException, "Value 3,14 is not a legal double"); @@ -127,7 +128,7 @@ TEST("require that maps can be parsed") } TEST("handles quotes for bool values") { - std::vector<vespalib::string> payload; + StringVector payload; payload.push_back("foo \"true\""); payload.push_back("bar \"123\""); payload.push_back("baz \"1234\""); diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp index 1c39b20a1ff..fdd106e44f2 100644 --- a/config/src/tests/configretriever/configretriever.cpp +++ b/config/src/tests/configretriever/configretriever.cpp @@ -14,6 +14,8 @@ #include <vespa/config/subscription/configsubscription.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/frt/protocol.h> +#include <vespa/config/retriever/configsnapshot.hpp> #include <thread> #include <atomic> @@ -34,7 +36,7 @@ struct ConfigTestFixture { BootstrapConfigBuilder bootstrapBuilder; map<std::string, ComponentFixture::SP> componentConfig; ConfigSet set; - IConfigContext::SP context; + std::shared_ptr<IConfigContext> context; int idcounter; ConfigTestFixture(const std::string & id) @@ -115,11 +117,11 @@ struct MySource : public Source struct SubscriptionFixture { - IConfigHolder::SP holder; - ConfigSubscription::SP sub; + std::shared_ptr<IConfigHolder> holder; + std::shared_ptr<ConfigSubscription> sub; SubscriptionFixture(const ConfigKey & key, const ConfigValue value) - : holder(new ConfigHolder()), - sub(new ConfigSubscription(0, key, holder, Source::UP(new MySource()))) + : holder(std::make_shared<ConfigHolder>()), + sub(std::make_shared<ConfigSubscription>(0, key, holder, std::make_unique<MySource>())) { holder->handle(std::make_unique<ConfigUpdate>(value, 3, 3)); ASSERT_TRUE(sub->nextUpdate(0, 0ms)); @@ -215,7 +217,7 @@ TEST("require that SimpleConfigRetriever usage works") { barBuilder.barValue = "fooz"; set.addBuilder("id", &fooBuilder); set.addBuilder("id", &barBuilder); - IConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); ConfigKeySet sub; sub.add<FooConfig>("id"); sub.add<BarConfig>("id"); @@ -281,7 +283,7 @@ TEST_F("require that SimpleConfigurer usage works", ConfigurableFixture()) { barBuilder.barValue = "fooz"; set.addBuilder("id", &fooBuilder); set.addBuilder("id", &barBuilder); - IConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); ConfigKeySet sub; sub.add<FooConfig>("id"); sub.add<BarConfig>("id"); @@ -351,8 +353,8 @@ TEST_FF("require that snapshots throws exception if invalid key", ConfigTestFixt f1.addComponent("c3", "foo3", "bar3"); ConfigSnapshot snap1 = f2.retriever->getBootstrapConfigs(); ASSERT_FALSE(snap1.hasConfig<BarConfig>("doesnotexist")); - ASSERT_EXCEPTION(snap1.getConfig<BarConfig>("doesnotexist"), IllegalConfigKeyException, "Unable to find config for key name=bar,namespace=config,configId=doesnotexist"); - ASSERT_EXCEPTION(snap1.isChanged<BarConfig>("doesnotexist", 0), IllegalConfigKeyException, "Unable to find config for key name=bar,namespace=config,configId=doesnotexist"); + ASSERT_EXCEPTION(snap1.getConfig<BarConfig>("doesnotexist"), IllegalConfigKeyException, "Unable to find config for key name=config.bar,configId=doesnotexist"); + ASSERT_EXCEPTION(snap1.isChanged<BarConfig>("doesnotexist", 0), IllegalConfigKeyException, "Unable to find config for key name=config.bar,configId=doesnotexist"); ASSERT_TRUE(snap1.hasConfig<BootstrapConfig>("myid")); } diff --git a/config/src/tests/configuri/configuri_test.cpp b/config/src/tests/configuri/configuri_test.cpp index 1089886cb1d..e4b3ea06195 100644 --- a/config/src/tests/configuri/configuri_test.cpp +++ b/config/src/tests/configuri/configuri_test.cpp @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/configcontext.h> +#include <vespa/config/subscription/configsubscriber.hpp> +#include <vespa/config/subscription/configuri.h> #include "config-my.h" using namespace config; @@ -47,7 +48,7 @@ TEST("Require that URI can be created from instance") { } -TEST_F("Require that URI can be \"forked\"", IConfigContext::SP(new ConfigContext())) { +TEST_F("Require that URI can be \"forked\"", std::shared_ptr<IConfigContext>(std::make_shared<ConfigContext>())) { assertConfigId("baz", ConfigUri("foo/bar").createWithNewId("baz")); ConfigUri parent("foo", f1); ConfigUri child = parent.createWithNewId("baz"); diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 4bac1fb9062..e342a477b13 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/common/misc.h> #include <vespa/config/frt/protocol.h> -#include <vespa/config/config.h> #include <vespa/config/common/configcontext.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -189,7 +188,7 @@ TimingValues testTimingValues( 1200); // fatalDelay struct ConfigCheckFixture { - IConfigContext::SP ctx; + std::shared_ptr<IConfigContext> ctx; NetworkFixture & nf; ConfigCheckFixture(NetworkFixture & f2) @@ -221,7 +220,7 @@ struct ConfigCheckFixture { }; struct ConfigReloadFixture { - IConfigContext::SP ctx; + std::shared_ptr<IConfigContext> ctx; NetworkFixture & nf; ConfigSubscriber s; ConfigHandle<MyConfig>::UP handle; diff --git a/config/src/tests/file_subscription/file_subscription.cpp b/config/src/tests/file_subscription/file_subscription.cpp index 1201f72f90e..2d1a8498ba5 100644 --- a/config/src/tests/file_subscription/file_subscription.cpp +++ b/config/src/tests/file_subscription/file_subscription.cpp @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/config/common/configholder.h> -#include <vespa/config/file/filesource.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/common/sourcefactory.h> #include <vespa/config/common/configcontext.h> @@ -63,12 +62,12 @@ TEST("requireThatFileSpecGivesCorrectSource") { SourceFactory::UP factory(spec.createSourceFactory(TimingValues())); ASSERT_TRUE(factory); auto holder = std::make_shared<ConfigHolder>(); - Source::UP src = factory->createSource(holder, ConfigKey("my", "my", "bar", "foo")); + std::unique_ptr<Source> src = factory->createSource(holder, ConfigKey("my", "my", "bar", "foo")); ASSERT_TRUE(src); src->getConfig(); ASSERT_TRUE(holder->poll()); - ConfigUpdate::UP update(holder->provide()); + std::unique_ptr<ConfigUpdate> update(holder->provide()); ASSERT_TRUE(update); const ConfigValue & value(update->getValue()); ASSERT_EQUAL(1u, value.numLines()); diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 54adbf4d787..0a0c3262b84 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -16,6 +16,8 @@ #include <vespa/fnet/frt/error.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/config/frt/protocol.h> +#include <vespa/config/common/configvalue.hpp> + #include <lz4.h> #include <thread> @@ -29,15 +31,15 @@ using namespace config::protocol::v3; namespace { struct UpdateFixture : public IConfigHolder { - ConfigUpdate::UP update; + std::unique_ptr<ConfigUpdate> update; bool notified; UpdateFixture() : update(), notified(false) { } - ConfigUpdate::UP provide() override { return ConfigUpdate::UP(); } - void handle(ConfigUpdate::UP u) override { update = std::move(u); } + std::unique_ptr<ConfigUpdate> provide() override { return std::unique_ptr<ConfigUpdate>(); } + void handle(std::unique_ptr<ConfigUpdate> u) override { update = std::move(u); } bool wait(milliseconds timeoutInMillis) override { (void) timeoutInMillis; return notified; } bool poll() override { return notified; } void interrupt() override { } @@ -75,7 +77,7 @@ namespace { const vespalib::string & configXxhash64="", int changed=0, long generation=0, - const std::vector<vespalib::string> & payload = std::vector<vespalib::string>(), + const StringVector & payload = StringVector(), const vespalib::string & ns = "") { FRT_RPCRequest * req = new FRT_RPCRequest(); diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp index 80433b0382c..333645176a0 100644 --- a/config/src/tests/functiontest/functiontest.cpp +++ b/config/src/tests/functiontest/functiontest.cpp @@ -1,13 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/config.h> -#include <vespa/config/common/exceptions.h> #include "config-function-test.h" - -#include <fstream> -#include <vespa/log/log.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/configgen/configpayload.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <fstream> + +#include <vespa/log/log.h> LOG_SETUP("functiontest_test"); diff --git a/config/src/tests/getconfig/getconfig.cpp b/config/src/tests/getconfig/getconfig.cpp index 4081ce2f1d6..a9598df9be9 100644 --- a/config/src/tests/getconfig/getconfig.cpp +++ b/config/src/tests/getconfig/getconfig.cpp @@ -12,7 +12,7 @@ namespace { struct ConfigFixture { MyConfigBuilder builder; ConfigSet set; - ConfigContext::SP context; + std::shared_ptr<IConfigContext> context; ConfigFixture() : builder(), set(), context() { set.addBuilder("cfgid", &builder); context = std::make_shared<ConfigContext>(set); diff --git a/config/src/tests/legacysubscriber/legacysubscriber.cpp b/config/src/tests/legacysubscriber/legacysubscriber.cpp index 51a32731b0e..7b5f2e2fc0e 100644 --- a/config/src/tests/legacysubscriber/legacysubscriber.cpp +++ b/config/src/tests/legacysubscriber/legacysubscriber.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/helper/legacysubscriber.h> +#include <vespa/config/helper/legacysubscriber.hpp> #include <fstream> #include <config-my.h> #include <config-foo.h> diff --git a/config/src/tests/misc/configsystem.cpp b/config/src/tests/misc/configsystem.cpp index c02e27ca989..db3d0a80706 100644 --- a/config/src/tests/misc/configsystem.cpp +++ b/config/src/tests/misc/configsystem.cpp @@ -44,7 +44,7 @@ TEST("require that correct pid file succeeds") { FastOS_File::MakeDirIfNotPresentOrExit("var/run"); FastOS_File pid_file("var/run/configproxy.pid"); pid_file.OpenWriteOnlyTruncate(); - pid_file.Close(); + ASSERT_TRUE(pid_file.Close()); ASSERT_EQUAL(0, setenv(VESPA_HOME, cwd, 1)); vespa::Defaults::bootstrap(cwd); diff --git a/config/src/tests/misc/misc.cpp b/config/src/tests/misc/misc.cpp index 25b6cf36326..25c6762326d 100644 --- a/config/src/tests/misc/misc.cpp +++ b/config/src/tests/misc/misc.cpp @@ -3,6 +3,7 @@ #include <vespa/config/common/configupdate.h> #include <vespa/config/common/misc.h> #include <vespa/config/common/configvalue.h> +#include <vespa/config/common/configkey.h> #include <vespa/config/common/errorcode.h> #include <vespa/config/common/vespa_version.h> #include <vespa/config/subscription/sourcespec.h> @@ -12,7 +13,7 @@ using namespace config; TEST("requireThatConfigUpdateWorks") { - std::vector<vespalib::string> lines; + StringVector lines; lines.push_back("foo"); ConfigUpdate up(ConfigValue(lines, "myxxhash"), true, 1337); @@ -25,13 +26,13 @@ TEST("requireThatConfigUpdateWorks") { } TEST("requireThatConfigValueWorks") { - std::vector<vespalib::string> lines; + StringVector lines; lines.push_back("myFooField \"bar\""); - ConfigValue v1(lines, calculateContentXxhash64(lines)); - ConfigValue v2(lines, calculateContentXxhash64(lines)); - ConfigValue v3(lines, calculateContentXxhash64(lines)); + ConfigValue v1(lines); + ConfigValue v2(lines); + ConfigValue v3(lines); lines.push_back("myFooField \"bar2\""); - ConfigValue v4(lines, calculateContentXxhash64(lines)); + ConfigValue v4(lines); ASSERT_TRUE(v1 == v2); ASSERT_TRUE(v1 == v3); } @@ -108,11 +109,11 @@ TEST("requireThatConfigKeyWorks") { TEST("require that config key initializes schema") { - std::vector<vespalib::string> schema; + StringVector schema; schema.push_back("foo"); schema.push_back("bar"); ConfigKey key("id1", "def1", "namespace1", "xxhash1", schema); - const std::vector<vespalib::string> &vref(key.getDefSchema()); + const StringVector &vref(key.getDefSchema()); for (size_t i = 0; i < schema.size(); i++) { ASSERT_EQUAL(schema[i], vref[i]); } diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp index 2ecb2062944..d5212048b2e 100644 --- a/config/src/tests/payload_converter/payload_converter.cpp +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -19,7 +19,7 @@ TEST("require that v2 payload leaf values can be converted to cfg format") { root.setDouble("baz", 3.1); root.setBool("quux", true); PayloadConverter converter(root); - std::vector<vespalib::string> lines(converter.convert()); + StringVector lines(converter.convert()); std::sort(lines.begin(), lines.end()); ASSERT_EQUAL(4u, lines.size()); @@ -36,7 +36,7 @@ TEST("require that v2 payload struct values can be converted to cfg format") { inner.setString("foo", "bar"); inner.setLong("bar", 8); PayloadConverter converter(root); - std::vector<vespalib::string> lines(converter.convert()); + StringVector lines(converter.convert()); std::sort(lines.begin(), lines.end()); ASSERT_EQUAL(2u, lines.size()); @@ -51,7 +51,7 @@ TEST("require that v2 payload array values can be converted to cfg format") { inner.addString("foo"); inner.addLong(8); PayloadConverter converter(root); - std::vector<vespalib::string> lines(converter.convert()); + StringVector lines(converter.convert()); ASSERT_EQUAL(2u, lines.size()); EXPECT_EQUAL("arr[0] \"foo\"", lines[0]); EXPECT_EQUAL("arr[1] 8", lines[1]); @@ -72,7 +72,7 @@ TEST("require that v2 payload nested structures can be converted to cfg format") Cursor & innerArr2(innerobj.setArray("arr2")); innerArr2.addString("muhaha"); PayloadConverter converter(root); - std::vector<vespalib::string> lines(converter.convert()); + StringVector lines(converter.convert()); std::sort(lines.begin(), lines.end()); ASSERT_EQUAL(3u, lines.size()); EXPECT_EQUAL("arr[0].foo \"bar\"", lines[0]); diff --git a/config/src/tests/print/print.cpp b/config/src/tests/print/print.cpp index 325b8dc28fa..fa20482cf0a 100644 --- a/config/src/tests/print/print.cpp +++ b/config/src/tests/print/print.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/print.h> -#include <vespa/config/print/fileconfigreader.h> -#include <vespa/config/print/istreamconfigreader.h> +#include <vespa/config/print/fileconfigreader.hpp> +#include <vespa/config/print/istreamconfigreader.hpp> #include <vespa/config/helper/configgetter.hpp> +#include <vespa/vespalib/util/exceptions.h> #include "config-my.h" #include "config-motd.h" #include <sys/stat.h> diff --git a/config/src/tests/raw_subscription/raw_subscription.cpp b/config/src/tests/raw_subscription/raw_subscription.cpp index 7141a21d35d..da35d10da52 100644 --- a/config/src/tests/raw_subscription/raw_subscription.cpp +++ b/config/src/tests/raw_subscription/raw_subscription.cpp @@ -1,9 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/configholder.h> #include <vespa/config/common/sourcefactory.h> -#include <vespa/config/raw/rawsource.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include "config-my.h" using namespace config; @@ -13,13 +12,13 @@ TEST("require that raw spec can create source factory") RawSpec spec("myField \"foo\"\n"); auto raw = spec.createSourceFactory(TimingValues()); ASSERT_TRUE(raw); - IConfigHolder::SP holder(new ConfigHolder()); - Source::UP src = raw->createSource(holder, ConfigKey("myid", "my", "bar", "foo")); + std::shared_ptr<IConfigHolder> holder(new ConfigHolder()); + std::unique_ptr<Source> src = raw->createSource(holder, ConfigKey("myid", "my", "bar", "foo")); ASSERT_TRUE(src); src->getConfig(); ASSERT_TRUE(holder->poll()); - ConfigUpdate::UP update(holder->provide()); + std::unique_ptr<ConfigUpdate> update(holder->provide()); ASSERT_TRUE(update); const ConfigValue & value(update->getValue()); ASSERT_EQUAL(1u, value.numLines()); diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index 5871add2618..68e211ec3f8 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -3,11 +3,12 @@ #include "config-bar.h" #include "config-baz.h" #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/misc.h> #include <vespa/config/common/configholder.h> -#include <vespa/config/subscription/configsubscription.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/common/iconfigmanager.h> +#include <vespa/config/common/iconfigcontext.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <thread> using namespace config; @@ -18,9 +19,9 @@ namespace { ConfigValue createValue(const std::string & value) { - std::vector< vespalib::string > lines; + StringVector lines; lines.push_back(value); - return ConfigValue(lines, calculateContentXxhash64(lines)); + return ConfigValue(std::move(lines)); } ConfigValue createFooValue(const std::string & value) @@ -40,13 +41,13 @@ namespace { void verifyConfig(const std::string & expected, std::unique_ptr<FooConfig> cfg) { - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL(expected, cfg->fooValue); } void verifyConfig(const std::string & expected, std::unique_ptr<BarConfig> cfg) { - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL(expected, cfg->barValue); } @@ -72,7 +73,7 @@ namespace { SubscriptionId idCounter; - std::vector<IConfigHolder::SP> _holders; + std::vector<std::shared_ptr<IConfigHolder>> _holders; int numCancel; @@ -80,7 +81,7 @@ namespace { ConfigSubscription::SP subscribe(const ConfigKey & key, milliseconds timeoutInMillis) override { (void) timeoutInMillis; - IConfigHolder::SP holder(new ConfigHolder()); + auto holder = std::make_shared<ConfigHolder>(); _holders.push_back(holder); return std::make_shared<ConfigSubscription>(0, key, holder, std::make_unique<MySource>()); @@ -116,16 +117,18 @@ namespace { { public: MyManager & _m; - APIFixture(MyManager & m) + APIFixture(MyManager & m) noexcept : _m(m) { } - APIFixture(const APIFixture & rhs) + APIFixture(const APIFixture & rhs) noexcept : IConfigContext(rhs), _m(rhs._m) { } + ~APIFixture() override = default; + IConfigManager & getManagerInstance() override { return _m; } @@ -144,7 +147,7 @@ namespace { ConfigHandle<FooConfig>::UP h1; ConfigHandle<BarConfig>::UP h2; - StandardFixture(MyManager & F1, APIFixture & F2) : f1(F1), s(IConfigContext::SP(new APIFixture(F2))) + StandardFixture(MyManager & F1, APIFixture & F2) : f1(F1), s(std::make_shared<APIFixture>(F2)) { h1 = s.subscribe<FooConfig>("myid"); h2 = s.subscribe<BarConfig>("myid"); @@ -206,7 +209,7 @@ TEST_F("requireThatSubscriptionsCannotBeAddedWhenFrozen", SimpleFixture()) { } TEST_FF("requireThatNextConfigReturnsFalseUntilSubscriptionHasSucceeded", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); ASSERT_FALSE(s.nextConfigNow()); @@ -335,7 +338,7 @@ TEST_MT_FFF("requireThatNextConfigIsInterruptedOnClose", 2, MyManager, APIFixtur } TEST_FF("requireThatHandlesAreMarkedAsChanged", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); EXPECT_FALSE(s.nextConfigNow()); @@ -357,7 +360,7 @@ TEST_FF("requireThatHandlesAreMarkedAsChanged", MyManager, APIFixture(f1)) { } TEST_FF("requireThatNextGenerationMarksChanged", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); f1.updateValue(0, createFooValue("foo"), 1); @@ -380,7 +383,7 @@ TEST_FF("requireThatNextGenerationMarksChanged", MyManager, APIFixture(f1)) { } TEST_FF("requireThatgetGenerationIsSet", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); f1.updateValue(0, createFooValue("foo"), 1); @@ -407,7 +410,7 @@ TEST_FFF("requireThatConfigHandleStillHasConfigOnTimestampUpdate", MyManager, AP } TEST_FF("requireThatTimeStamp0Works", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); ConfigHandle<BazConfig>::UP h3 = s.subscribe<BazConfig>("myid"); @@ -421,7 +424,7 @@ TEST_FF("requireThatTimeStamp0Works", MyManager, APIFixture(f1)) { } TEST_FF("requireThatNextGenerationWorksWithManyConfigs", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); ConfigHandle<BazConfig>::UP h3 = s.subscribe<BazConfig>("myid"); @@ -483,7 +486,7 @@ TEST_FF("requireThatNextGenerationWorksWithManyConfigs", MyManager, APIFixture(f } TEST_FF("requireThatConfigSubscriberHandlesProxyCache", MyManager, APIFixture(f1)) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); f1.updateValue(0, createFooValue("foo"), 1); f1.updateGeneration(0, 2); @@ -501,7 +504,7 @@ TEST_FF("requireThatConfigSubscriberHandlesProxyCache", MyManager, APIFixture(f1 TEST_MT_FF("requireThatConfigSubscriberWaitsUntilNextConfigSucceeds", 2, MyManager, APIFixture(f1)) { if (thread_id == 0) { - ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigSubscriber s(std::make_shared<APIFixture>(f2)); ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); f1.updateValue(0, createFooValue("foo"), 1); ASSERT_TRUE(s.nextConfigNow()); diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp index a65528d67ee..f35ea3c6cef 100644 --- a/config/src/tests/subscription/subscription.cpp +++ b/config/src/tests/subscription/subscription.cpp @@ -38,12 +38,12 @@ namespace { struct SubscriptionFixture { - IConfigHolder::SP holder; + std::shared_ptr<IConfigHolder> holder; ConfigSubscription sub; SourceFixture src; SubscriptionFixture(const ConfigKey & key) : holder(new ConfigHolder()), - sub(0, key, holder, Source::UP(new MySource(&src))) + sub(0, key, holder, std::make_unique<MySource>(&src)) { } }; @@ -100,11 +100,11 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf TEST_F("Require that isChanged takes generation into account", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) { - f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(std::vector<vespalib::string>(), "a"), true, 1)); + f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(StringVector(), "a"), true, 1)); ASSERT_TRUE(f1.sub.nextUpdate(0, 0ms)); f1.sub.flip(); ASSERT_EQUAL(1, f1.sub.getLastGenerationChanged()); - f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(std::vector<vespalib::string>(), "b"), true, 2)); + f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(StringVector(), "b"), true, 2)); ASSERT_TRUE(f1.sub.nextUpdate(1, 0ms)); f1.sub.flip(); ASSERT_EQUAL(2, f1.sub.getLastGenerationChanged()); diff --git a/config/src/tests/unittest/unittest.cpp b/config/src/tests/unittest/unittest.cpp index 1ba11db89b9..207a139f31d 100644 --- a/config/src/tests/unittest/unittest.cpp +++ b/config/src/tests/unittest/unittest.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/common/configcontext.h> #include "config-my.h" #include "config-foo.h" #include "config-bar.h" +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/log/log.h> LOG_SETUP("unittest"); diff --git a/config/src/vespa/config/common/cancelhandler.h b/config/src/vespa/config/common/cancelhandler.h index 7641b774fbf..8dde83ba7d9 100644 --- a/config/src/vespa/config/common/cancelhandler.h +++ b/config/src/vespa/config/common/cancelhandler.h @@ -13,9 +13,9 @@ struct CancelHandler * * @param subscription ConfigSubscription to cancel */ - virtual void unsubscribe(const ConfigSubscription::SP & subscription) = 0; + virtual void unsubscribe(const std::shared_ptr<ConfigSubscription> & subscription) = 0; - virtual ~CancelHandler() { } + virtual ~CancelHandler() = default; }; } diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp index 21bafcda8f6..a02460f8e64 100644 --- a/config/src/vespa/config/common/configdefinition.cpp +++ b/config/src/vespa/config/common/configdefinition.cpp @@ -12,8 +12,8 @@ ConfigDefinition::ConfigDefinition() : _schema() {} -ConfigDefinition::ConfigDefinition(const std::vector<vespalib::string> & schema) - : _schema(schema) +ConfigDefinition::ConfigDefinition(StringVector schema) + : _schema(std::move(schema)) {} void @@ -36,8 +36,8 @@ vespalib::string ConfigDefinition::asString() const { vespalib::asciistream as; - for (auto it(_schema.begin()), mt(_schema.end()); it != mt; it++) { - as << *it; + for (const auto & line : _schema) { + as << line; } return as.str(); } diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h index 78ab7877394..5ef87cbe7f7 100644 --- a/config/src/vespa/config/common/configdefinition.h +++ b/config/src/vespa/config/common/configdefinition.h @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vector> +#include "types.h" namespace vespalib::slime { struct Cursor; @@ -16,12 +15,12 @@ namespace config { class ConfigDefinition { public: ConfigDefinition(); - ConfigDefinition(const std::vector<vespalib::string> & schema); + ConfigDefinition(StringVector schema); void deserialize(const vespalib::slime::Inspector & inspector); void serialize(vespalib::slime::Cursor & cursor) const; vespalib::string asString() const; private: - std::vector<vespalib::string> _schema; + StringVector _schema; }; } //namespace config diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp index f16d70f11b1..41c88e13b06 100644 --- a/config/src/vespa/config/common/configholder.cpp +++ b/config/src/vespa/config/common/configholder.cpp @@ -13,7 +13,7 @@ ConfigHolder::ConfigHolder() ConfigHolder::~ConfigHolder() = default; -ConfigUpdate::UP +std::unique_ptr<ConfigUpdate> ConfigHolder::provide() { std::lock_guard guard(_lock); @@ -21,7 +21,7 @@ ConfigHolder::provide() } void -ConfigHolder::handle(ConfigUpdate::UP update) +ConfigHolder::handle(std::unique_ptr<ConfigUpdate> update) { std::lock_guard guard(_lock); if (_current) { diff --git a/config/src/vespa/config/common/configholder.h b/config/src/vespa/config/common/configholder.h index 75324bd9d2e..c9b5a37765a 100644 --- a/config/src/vespa/config/common/configholder.h +++ b/config/src/vespa/config/common/configholder.h @@ -16,15 +16,15 @@ public: ConfigHolder(); ~ConfigHolder() override; - ConfigUpdate::UP provide() override; - void handle(ConfigUpdate::UP update) override; + std::unique_ptr<ConfigUpdate> provide() override; + void handle(std::unique_ptr<ConfigUpdate> update) override; bool wait(milliseconds timeoutInMillis) override; bool poll() override; void interrupt() override; public: - std::mutex _lock; - std::condition_variable _cond; - ConfigUpdate::UP _current; + std::mutex _lock; + std::condition_variable _cond; + std::unique_ptr<ConfigUpdate> _current; }; } // namespace config diff --git a/config/src/vespa/config/common/configkey.cpp b/config/src/vespa/config/common/configkey.cpp index 248890b05f9..f81e5fbcb87 100644 --- a/config/src/vespa/config/common/configkey.cpp +++ b/config/src/vespa/config/common/configkey.cpp @@ -20,7 +20,7 @@ ConfigKey::ConfigKey(vespalib::stringref configId, vespalib::stringref defName, vespalib::stringref defNamespace, vespalib::stringref defMd5, - const std::vector<vespalib::string> & defSchema) + const StringVector & defSchema) : _configId(configId), _defName(defName), _defNamespace(defNamespace), @@ -30,19 +30,12 @@ ConfigKey::ConfigKey(vespalib::stringref configId, { } -ConfigKey::ConfigKey() - : _configId(), - _defName(), - _defNamespace(), - _defMd5(), - _defSchema(), - _key() -{} - +ConfigKey::ConfigKey() = default; ConfigKey::ConfigKey(const ConfigKey &) = default; ConfigKey & ConfigKey::operator = (const ConfigKey &) = default; - -ConfigKey::~ConfigKey() { } +ConfigKey::ConfigKey(ConfigKey &&) noexcept = default; +ConfigKey & ConfigKey::operator = (ConfigKey &&) noexcept = default; +ConfigKey::~ConfigKey() = default; bool ConfigKey::operator<(const ConfigKey & rhs) const @@ -66,16 +59,16 @@ const vespalib::string & ConfigKey::getDefName() const { return _defName; } const vespalib::string & ConfigKey::getConfigId() const { return _configId; } const vespalib::string & ConfigKey::getDefNamespace() const { return _defNamespace; } const vespalib::string & ConfigKey::getDefMd5() const { return _defMd5; } -const std::vector<vespalib::string> & ConfigKey::getDefSchema() const { return _defSchema; } +const StringVector & ConfigKey::getDefSchema() const { return _defSchema; } const vespalib::string ConfigKey::toString() const { vespalib::string s; s.append("name="); - s.append(_defName); - s.append(",namespace="); s.append(_defNamespace); + s.append("."); + s.append(_defName); s.append(",configId="); s.append(_configId); return s; diff --git a/config/src/vespa/config/common/configkey.h b/config/src/vespa/config/common/configkey.h index 96b7ada92e4..2b8d2fc70b3 100644 --- a/config/src/vespa/config/common/configkey.h +++ b/config/src/vespa/config/common/configkey.h @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vector> +#include "types.h" namespace config { @@ -17,12 +16,12 @@ public: vespalib::stringref defName, vespalib::stringref defNamespace, vespalib::stringref defMd5, - const std::vector<vespalib::string> & defSchema); + const StringVector & defSchema); ConfigKey(const ConfigKey &); ConfigKey & operator = (const ConfigKey &); - ConfigKey(ConfigKey &&) = default; - ConfigKey & operator = (ConfigKey &&) = default; + ConfigKey(ConfigKey &&) noexcept; + ConfigKey & operator = (ConfigKey &&) noexcept; ConfigKey(); ~ConfigKey(); @@ -34,7 +33,7 @@ public: const vespalib::string & getConfigId() const; const vespalib::string & getDefNamespace() const; const vespalib::string & getDefMd5() const; - const std::vector<vespalib::string> & getDefSchema() const; + const StringVector & getDefSchema() const; template <typename ConfigType> static const ConfigKey create(vespalib::stringref configId) @@ -51,7 +50,7 @@ private: vespalib::string _defName; vespalib::string _defNamespace; vespalib::string _defMd5; - std::vector<vespalib::string> _defSchema; + StringVector _defSchema; vespalib::string _key; }; diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index fb082b8479a..d982a77bd1c 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -31,7 +31,7 @@ ConfigManager::subscribe(const ConfigKey & key, milliseconds timeoutInMillis) SubscriptionId id(_idGenerator.fetch_add(1)); auto holder = std::make_shared<ConfigHolder>(); - Source::UP source = _sourceFactory->createSource(holder, key); + std::unique_ptr<Source> source = _sourceFactory->createSource(holder, key); source->reload(_generation); source->getConfig(); diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp index a8785642ca7..97373071ed5 100644 --- a/config/src/vespa/config/common/configparser.cpp +++ b/config/src/vespa/config/common/configparser.cpp @@ -123,11 +123,10 @@ getValueForKey(vespalib::stringref key, vespalib::stringref line, } -std::vector<vespalib::string> -ConfigParser::getLinesForKey(vespalib::stringref key, - const vsvector & lines) +StringVector +ConfigParser::getLinesForKey(vespalib::stringref key, Cfg lines) { - vsvector retval; + StringVector retval; for (uint32_t i = 0; i < lines.size(); i++) { vespalib::string value; @@ -140,6 +139,18 @@ ConfigParser::getLinesForKey(vespalib::stringref key, return retval; } +std::set<vespalib::string> +ConfigParser::getUniqueNonWhiteSpaceLines(Cfg config) { + std::set<vespalib::string> unique; + for (uint32_t i = 0; i < config.size(); i++) { + vespalib::string line = stripWhitespace(config[i]); + if (!line.empty()) { + unique.insert(line); + } + } + return unique; +} + void ConfigParser::stripLinesForKey(vespalib::stringref key, std::set<vespalib::string>& config) @@ -155,10 +166,10 @@ ConfigParser::stripLinesForKey(vespalib::stringref key, } } -std::map<vespalib::string, ConfigParser::vsvector> -ConfigParser::splitMap(const vsvector & config) +std::map<vespalib::string, StringVector> +ConfigParser::splitMap(Cfg config) { - std::map<vespalib::string, vsvector> items; + std::map<vespalib::string, StringVector> items; vespalib::string lastValue; @@ -178,7 +189,7 @@ ConfigParser::splitMap(const vsvector & config) vespalib::string value = config[i].substr(pos + 1); if (key != lastValue) { - items[key] = vsvector(); + items[key] = StringVector(); lastValue = key; } @@ -191,10 +202,10 @@ ConfigParser::splitMap(const vsvector & config) return items; } -std::vector<ConfigParser::vsvector> -ConfigParser::splitArray(const vsvector & config) +std::vector<StringVector> +ConfigParser::splitArray(Cfg config) { - std::vector<vsvector> items; + std::vector<StringVector> items; vespalib::string lastValue; @@ -214,7 +225,7 @@ ConfigParser::splitArray(const vsvector & config) vespalib::string value = config[i].substr(pos + 1); if (key != lastValue) { - items.push_back(vsvector()); + items.push_back(StringVector()); lastValue = key; } @@ -266,7 +277,7 @@ ConfigParser::stripWhitespace(vespalib::stringref source) } vespalib::string -ConfigParser::arrayToString(const vsvector & array) +ConfigParser::arrayToString(Cfg array) { vespalib::asciistream ost; if (array.size() == 0) { @@ -281,13 +292,13 @@ ConfigParser::arrayToString(const vsvector & array) template<> bool -ConfigParser::convert<bool>(const vsvector & config) +ConfigParser::convert<bool>(const StringVector & config) { if (config.size() != 1) { throw InvalidConfigException("Expected single line with bool value, " "got " + arrayToString(config), VESPA_STRLOC); } - std::string value = stripWhitespace(deQuote(config[0])); + vespalib::string value = stripWhitespace(deQuote(config[0])); if (value == "true") { return true; @@ -301,13 +312,13 @@ ConfigParser::convert<bool>(const vsvector & config) template<> int32_t -ConfigParser::convert<int32_t>(const vsvector & config) +ConfigParser::convert<int32_t>(const StringVector & config) { if (config.size() != 1) { throw InvalidConfigException("Expected single line with int32_t value, " "got " + arrayToString(config), VESPA_STRLOC); } - std::string value(deQuote(stripWhitespace(config[0]))); + vespalib::string value(deQuote(stripWhitespace(config[0]))); const char *startp = value.c_str(); char *endp; @@ -321,13 +332,13 @@ ConfigParser::convert<int32_t>(const vsvector & config) template<> int64_t -ConfigParser::convert<int64_t>(const vsvector & config) +ConfigParser::convert<int64_t>(const StringVector & config) { if (config.size() != 1) { throw InvalidConfigException("Expected single line with int64_t value, " "got " + arrayToString(config), VESPA_STRLOC); } - std::string value(deQuote(stripWhitespace(config[0]))); + vespalib::string value(deQuote(stripWhitespace(config[0]))); const char *startp = value.c_str(); char *endp; @@ -341,13 +352,13 @@ ConfigParser::convert<int64_t>(const vsvector & config) template<> double -ConfigParser::convert<double>(const vsvector & config) +ConfigParser::convert<double>(const StringVector & config) { if (config.size() != 1) { throw InvalidConfigException("Expected single line with double value, " "got " + arrayToString(config), VESPA_STRLOC); } - std::string value(deQuote(stripWhitespace(config[0]))); + vespalib::string value(deQuote(stripWhitespace(config[0]))); const char *startp = value.c_str(); char *endp; @@ -362,14 +373,14 @@ ConfigParser::convert<double>(const vsvector & config) template<> vespalib::string -ConfigParser::convert<vespalib::string>(const vsvector & config) +ConfigParser::convert<vespalib::string>(const StringVector & config) { if (config.size() != 1) { throw InvalidConfigException("Expected single line with string value, " "got " + arrayToString(config), VESPA_STRLOC); } - std::string value = stripWhitespace(config[0]); + vespalib::string value = stripWhitespace(config[0]); return deQuote(value); } diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h index b96bf77fd77..42ee684eb59 100644 --- a/config/src/vespa/config/common/configparser.h +++ b/config/src/vespa/config/common/configparser.h @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "types.h" #include <vespa/vespalib/util/stringfmt.h> -#include <map> #include <set> -#include <vector> #include <cerrno> -#include <cstdint> namespace config { @@ -16,69 +14,83 @@ namespace config { */ class ConfigParser { public: - typedef std::vector<vespalib::string> vsvector; + class Cfg { + public: + Cfg(const std::vector<vespalib::string> & v) + : _cfg(&v[0]), _sz(v.size()) + { } + Cfg(const std::vector<vespalib::string, vespalib::allocator_large<vespalib::string>> & v) : + _cfg(&v[0]), + _sz(v.size()) + { } + size_t size() const { return _sz; } + const vespalib::string & operator[] (size_t idx) const { return _cfg[idx]; } + private: + const vespalib::string * _cfg; + size_t _sz; + }; private: - static vsvector getLinesForKey(vespalib::stringref key, const vsvector & config); + static StringVector getLinesForKey(vespalib::stringref key, Cfg config); - static std::vector<vsvector> splitArray( const vsvector & config); - static std::map<vespalib::string, vsvector> splitMap( const vsvector & config); + static std::vector<StringVector> splitArray(Cfg config); + static std::map<vespalib::string, StringVector> splitMap(Cfg config); static vespalib::string deQuote(const vespalib::string & source); static void throwNoDefaultValue(vespalib::stringref key); template<typename T> - static T convert(const vsvector &); + static T convert(const StringVector & config); - static vespalib::string arrayToString(const vsvector &); + static vespalib::string arrayToString(Cfg config); - template<typename T, typename V> - static T parseInternal(vespalib::stringref key, const V & config); - template<typename T, typename V> - static T parseInternal(vespalib::stringref key, const V & config, T defaultValue); + template<typename T> + static T parseInternal(vespalib::stringref key, Cfg config); + template<typename T> + static T parseInternal(vespalib::stringref key, Cfg config, T defaultValue); - template<typename T, typename V> - static std::vector<T> parseArrayInternal(vespalib::stringref key, const V & config); - template<typename T, typename V> - static std::map<vespalib::string, T> parseMapInternal(vespalib::stringref key, const V & config); - template<typename T, typename V> - static T parseStructInternal(vespalib::stringref key, const V & config); + template<typename V> + static V parseArrayInternal(vespalib::stringref key, Cfg config); + template<typename T> + static std::map<vespalib::string, T> parseMapInternal(vespalib::stringref key, Cfg config); + template<typename T> + static T parseStructInternal(vespalib::stringref key, Cfg config); public: - static void stripLinesForKey(vespalib::stringref key, - std::set<vespalib::string>& config); + static void stripLinesForKey(vespalib::stringref key, std::set<vespalib::string>& config); + static std::set<vespalib::string> getUniqueNonWhiteSpaceLines(Cfg config); static vespalib::string stripWhitespace(vespalib::stringref source); template<typename T> - static T parse(vespalib::stringref key, const vsvector & config) { - return parseInternal<T, vsvector>(key, config); + static T parse(vespalib::stringref key, Cfg config) { + return parseInternal<T>(key, config); } template<typename T> - static T parse(vespalib::stringref key, const vsvector & config, T defaultValue) { + static T parse(vespalib::stringref key, Cfg config, T defaultValue) { return parseInternal(key, config, defaultValue); } - template<typename T> - static std::vector<T> parseArray(vespalib::stringref key, const vsvector & config) { - return parseArrayInternal<T, vsvector>(key, config); + template<typename V> + static V parseArray(vespalib::stringref key, Cfg config) { + return parseArrayInternal<V>(key, config); } template<typename T> - static std::map<vespalib::string, T> parseMap(vespalib::stringref key, const vsvector & config) { - return parseMapInternal<T, vsvector>(key, config); + static std::map<vespalib::string, T> parseMap(vespalib::stringref key, Cfg config) { + return parseMapInternal<T>(key, config); } template<typename T> - static T parseStruct(vespalib::stringref key, const vsvector & config) { - return parseStructInternal<T, vsvector>(key, config); + static T parseStruct(vespalib::stringref key, Cfg config) { + return parseStructInternal<T>(key, config); } }; -template<typename T, typename V> +template<typename T> T -ConfigParser::parseInternal(vespalib::stringref key, const V & config) +ConfigParser::parseInternal(vespalib::stringref key, Cfg config) { - V lines = getLinesForKey(key, config); + StringVector lines = getLinesForKey(key, config); if (lines.size() == 0) { throwNoDefaultValue(key); @@ -86,11 +98,11 @@ ConfigParser::parseInternal(vespalib::stringref key, const V & config) return convert<T>(lines); } -template<typename T, typename V> +template<typename T> T -ConfigParser::parseInternal(vespalib::stringref key, const V & config, T defaultValue) +ConfigParser::parseInternal(vespalib::stringref key, Cfg config, T defaultValue) { - V lines = getLinesForKey(key, config); + StringVector lines = getLinesForKey(key, config); if (lines.size() == 0) { return defaultValue; @@ -101,67 +113,68 @@ ConfigParser::parseInternal(vespalib::stringref key, const V & config, T default template<typename T> T -ConfigParser::convert(const vsvector & lines) { +ConfigParser::convert(const StringVector & lines) { return T(lines); } -template<typename T, typename V> +template<typename T> std::map<vespalib::string, T> -ConfigParser::parseMapInternal(vespalib::stringref key, const V & config) +ConfigParser::parseMapInternal(vespalib::stringref key, Cfg config) { - V lines = getLinesForKey(key, config); - typedef std::map<vespalib::string, V> SplittedMap; + StringVector lines = getLinesForKey(key, config); + using SplittedMap = std::map<vespalib::string, StringVector>; SplittedMap s = splitMap(lines); std::map<vespalib::string, T> retval; - for (typename SplittedMap::iterator it(s.begin()), mt(s.end()); it != mt; it++) { - retval[it->first] = convert<T>(it->second); + for (const auto & e : s) { + retval[e.first] = convert<T>(e.second); } return retval; } -template<typename T, typename V> -std::vector<T> -ConfigParser::parseArrayInternal(vespalib::stringref key, const V & config) +template<typename V> +V +ConfigParser::parseArrayInternal(vespalib::stringref key, Cfg config) { - V lines = getLinesForKey(key, config); - std::vector<V> split = splitArray(lines); + StringVector lines = getLinesForKey(key, config); + std::vector<StringVector> split = splitArray(lines); - std::vector<T> retval; + V retval; + retval.reserve(split.size()); for (uint32_t i = 0; i < split.size(); i++) { - retval.push_back(convert<T>(split[i])); + retval.push_back(convert<typename V::value_type>(split[i])); } return retval; } -template<typename T, typename V> +template<typename T> T -ConfigParser::parseStructInternal(vespalib::stringref key, const V & config) +ConfigParser::parseStructInternal(vespalib::stringref key, Cfg config) { - V lines = getLinesForKey(key, config); + StringVector lines = getLinesForKey(key, config); return convert<T>(lines); } template<> bool -ConfigParser::convert<bool>(const vsvector & config); +ConfigParser::convert<bool>(const StringVector & config); template<> int32_t -ConfigParser::convert<int32_t>(const vsvector & config); +ConfigParser::convert<int32_t>(const StringVector & config); template<> int64_t -ConfigParser::convert<int64_t>(const vsvector & config); +ConfigParser::convert<int64_t>(const StringVector & config); template<> double -ConfigParser::convert<double>(const vsvector & config); +ConfigParser::convert<double>(const StringVector & config); template<> vespalib::string -ConfigParser::convert<vespalib::string>(const vsvector & config); +ConfigParser::convert<vespalib::string>(const StringVector & config); } // config diff --git a/config/src/vespa/config/common/configstate.h b/config/src/vespa/config/common/configstate.h index a0229f85cd5..0895517343c 100644 --- a/config/src/vespa/config/common/configstate.h +++ b/config/src/vespa/config/common/configstate.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/string.h> #include "misc.h" namespace config { diff --git a/config/src/vespa/config/common/configupdate.cpp b/config/src/vespa/config/common/configupdate.cpp index e46dd781b2d..55f12775a46 100644 --- a/config/src/vespa/config/common/configupdate.cpp +++ b/config/src/vespa/config/common/configupdate.cpp @@ -3,13 +3,13 @@ namespace config { -ConfigUpdate::ConfigUpdate(const ConfigValue & value, bool changed, int64_t generation) - : _value(value), +ConfigUpdate::ConfigUpdate(ConfigValue value, bool changed, int64_t generation) + : _value(std::move(value)), _hasChanged(changed), _generation(generation) { } - +ConfigUpdate::~ConfigUpdate() = default; const ConfigValue & ConfigUpdate::getValue() const { return _value; } bool ConfigUpdate::hasChanged() const { return _hasChanged; } int64_t ConfigUpdate::getGeneration() const { return _generation; } diff --git a/config/src/vespa/config/common/configupdate.h b/config/src/vespa/config/common/configupdate.h index 571b32a97dc..35d641615a0 100644 --- a/config/src/vespa/config/common/configupdate.h +++ b/config/src/vespa/config/common/configupdate.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <memory> #include "configvalue.h" namespace config { @@ -13,17 +12,18 @@ namespace config { class ConfigUpdate { public: - typedef std::unique_ptr<ConfigUpdate> UP; - ConfigUpdate(const ConfigValue & value, bool changed, int64_t generation); - + ConfigUpdate(ConfigValue value, bool changed, int64_t generation); + ConfigUpdate(const ConfigUpdate &) = delete; + ConfigUpdate & operator = (const ConfigUpdate &) = delete; + ~ConfigUpdate(); const ConfigValue & getValue() const; bool hasChanged() const; int64_t getGeneration() const; void merge(const ConfigUpdate & b) { _hasChanged = _hasChanged || b.hasChanged(); } private: ConfigValue _value; - bool _hasChanged; - int64_t _generation; + bool _hasChanged; + int64_t _generation; }; } // namespace config diff --git a/config/src/vespa/config/common/configvalue.cpp b/config/src/vespa/config/common/configvalue.cpp index da1cbfc792c..586ada889bc 100644 --- a/config/src/vespa/config/common/configvalue.cpp +++ b/config/src/vespa/config/common/configvalue.cpp @@ -2,16 +2,23 @@ #include "configvalue.h" #include "payload_converter.h" #include "misc.h" +#include <vespa/config/frt/protocol.h> #include <vespa/vespalib/data/slime/slime.h> namespace config { -ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash) +ConfigValue::ConfigValue(StringVector lines, const vespalib::string & xxhash) : _payload(), - _lines(lines), + _lines(std::move(lines)), _xxhash64(xxhash) { } +ConfigValue::ConfigValue(StringVector lines) + : _payload(), + _lines(std::move(lines)), + _xxhash64(calculateContentXxhash64(_lines)) +{ } + ConfigValue::ConfigValue() : _payload(), _lines(), @@ -26,7 +33,6 @@ ConfigValue::ConfigValue(PayloadPtr payload, const vespalib::string & xxhash) ConfigValue::ConfigValue(const ConfigValue &) = default; ConfigValue & ConfigValue::operator = (const ConfigValue &) = default; - ConfigValue::~ConfigValue() = default; int @@ -41,10 +47,10 @@ ConfigValue::operator!=(const ConfigValue & rhs) const return (!(*this == rhs)); } -std::vector<vespalib::string> +StringVector ConfigValue::getLegacyFormat() const { - std::vector<vespalib::string> lines; + StringVector lines; if (_payload) { const vespalib::slime::Inspector & payload(_payload->getSlimePayload()); PayloadConverter converter(payload); @@ -69,7 +75,7 @@ void ConfigValue::serializeV1(vespalib::slime::Cursor & cursor) const { // TODO: Remove v1 when we can bump disk format. - std::vector<vespalib::string> lines(getLegacyFormat()); + StringVector lines(getLegacyFormat()); for (size_t i = 0; i < lines.size(); i++) { cursor.addString(vespalib::Memory(lines[i])); } diff --git a/config/src/vespa/config/common/configvalue.h b/config/src/vespa/config/common/configvalue.h index bf4c320c061..c071b3cfd7c 100644 --- a/config/src/vespa/config/common/configvalue.h +++ b/config/src/vespa/config/common/configvalue.h @@ -1,15 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/config/frt/protocol.h> -#include <vespa/config/configgen/configpayload.h> -#include <vespa/vespalib/stllike/string.h> -#include <vector> +#include <vespa/config/common/types.h> #include <memory> #include <climits> namespace vespalib::slime { struct Cursor; } - +namespace config::protocol { struct Payload; } namespace config { typedef std::shared_ptr<const protocol::Payload> PayloadPtr; @@ -20,10 +17,12 @@ typedef std::shared_ptr<const protocol::Payload> PayloadPtr; */ class ConfigValue { public: - typedef std::unique_ptr<ConfigValue> UP; - ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash); + explicit ConfigValue(StringVector lines); + ConfigValue(StringVector lines, const vespalib::string & xxhash); ConfigValue(PayloadPtr data, const vespalib::string & xxhash); ConfigValue(); + ConfigValue(ConfigValue &&) noexcept = default; + ConfigValue & operator = (ConfigValue &&) noexcept = default; ConfigValue(const ConfigValue &); ConfigValue & operator = (const ConfigValue &); ~ConfigValue(); @@ -33,8 +32,8 @@ public: size_t numLines() const { return _lines.size(); } const vespalib::string & getLine(int i) const { return _lines.at(i); } - const std::vector<vespalib::string> & getLines() const { return _lines; } - std::vector<vespalib::string> getLegacyFormat() const; + const StringVector & getLines() const { return _lines; } + StringVector getLegacyFormat() const; vespalib::string asJson() const; const vespalib::string& getXxhash64() const { return _xxhash64; } @@ -45,12 +44,9 @@ public: std::unique_ptr<ConfigType> newInstance() const; private: - PayloadPtr _payload; - std::vector<vespalib::string> _lines; + PayloadPtr _payload; + StringVector _lines; vespalib::string _xxhash64; }; } //namespace config - -#include "configvalue.hpp" - diff --git a/config/src/vespa/config/common/configvalue.hpp b/config/src/vespa/config/common/configvalue.hpp index 665ce69c7a5..89177454dc3 100644 --- a/config/src/vespa/config/common/configvalue.hpp +++ b/config/src/vespa/config/common/configvalue.hpp @@ -1,5 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configvalue.h" +#include <vespa/config/configgen/configpayload.h> +#include <vespa/config/frt/protocol.h> + namespace config { template <typename ConfigType> @@ -8,9 +12,9 @@ ConfigValue::newInstance() const { if (_payload) { const vespalib::slime::Inspector & payload(_payload->getSlimePayload()); - return std::unique_ptr<ConfigType>(new ConfigType(::config::ConfigPayload(payload))); + return std::make_unique<ConfigType>(::config::ConfigPayload(payload)); } else { - return std::unique_ptr<ConfigType>(new ConfigType(*this)); + return std::make_unique<ConfigType>(*this); } } diff --git a/config/src/vespa/config/common/iconfigcontext.h b/config/src/vespa/config/common/iconfigcontext.h index acc97a7cb8c..71be1dec715 100644 --- a/config/src/vespa/config/common/iconfigcontext.h +++ b/config/src/vespa/config/common/iconfigcontext.h @@ -15,8 +15,6 @@ class IConfigManager; class IConfigContext { public: - typedef std::shared_ptr<IConfigContext> SP; - /** * Get an instance of the config manager. * @@ -29,7 +27,7 @@ public: */ virtual void reload() = 0; - virtual ~IConfigContext() { } + virtual ~IConfigContext() = default; }; } // namespace diff --git a/config/src/vespa/config/common/iconfigholder.h b/config/src/vespa/config/common/iconfigholder.h index ff8e117cb37..9b474ccf70b 100644 --- a/config/src/vespa/config/common/iconfigholder.h +++ b/config/src/vespa/config/common/iconfigholder.h @@ -17,8 +17,7 @@ class IConfigHolder : public ConfigHandler, public Interruptable { public: - typedef std::shared_ptr<IConfigHolder> SP; - virtual ~IConfigHolder() { } + virtual ~IConfigHolder() = default; }; } // namespace config diff --git a/config/src/vespa/config/common/iconfigmanager.h b/config/src/vespa/config/common/iconfigmanager.h index ab0576428f1..3f5304b075f 100644 --- a/config/src/vespa/config/common/iconfigmanager.h +++ b/config/src/vespa/config/common/iconfigmanager.h @@ -12,7 +12,7 @@ class IConfigManager : public SubscribeHandler, public ReloadHandler { public: - virtual ~IConfigManager() { } + virtual ~IConfigManager() = default; }; } // namespace config diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp index 34985d19c68..31c66cd4eb8 100644 --- a/config/src/vespa/config/common/misc.cpp +++ b/config/src/vespa/config/common/misc.cpp @@ -14,7 +14,7 @@ using vespalib::Memory; namespace config { vespalib::string -calculateContentXxhash64(const std::vector<vespalib::string> & fileContents) +calculateContentXxhash64(const StringVector & fileContents) { vespalib::string normalizedLines; XXH64_hash_t xxhash64; @@ -39,6 +39,16 @@ calculateContentXxhash64(const std::vector<vespalib::string> & fileContents) return ss.str(); } +StringVector +getlines(vespalib::asciistream & is, char delim) +{ + StringVector lines; + while (!is.eof()) { + lines.push_back(is.getline(delim)); + } + return lines; +} + bool isGenerationNewer(int64_t newGen, int64_t oldGen) { diff --git a/config/src/vespa/config/common/misc.h b/config/src/vespa/config/common/misc.h index 06546a85292..584a7ca527b 100644 --- a/config/src/vespa/config/common/misc.h +++ b/config/src/vespa/config/common/misc.h @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "configkey.h" -#include <vespa/vespalib/stllike/string.h> -#include <vector> +#include "types.h" #include <memory> namespace vespalib { + class asciistream; class Slime; namespace slime { struct Inspector; @@ -19,7 +18,7 @@ namespace config { /** * Miscellaneous utility functions specific to config. */ -vespalib::string calculateContentXxhash64(const std::vector<vespalib::string> & fileContents); +vespalib::string calculateContentXxhash64(const StringVector & fileContents); bool isGenerationNewer(int64_t newGen, int64_t oldGen); @@ -34,4 +33,6 @@ typedef std::shared_ptr<const vespalib::Slime> SlimePtr; */ void copySlimeObject(const vespalib::slime::Inspector & src, vespalib::slime::Cursor & dest); +StringVector getlines(vespalib::asciistream & is, char delim='\n'); + } diff --git a/config/src/vespa/config/common/payload_converter.cpp b/config/src/vespa/config/common/payload_converter.cpp index 3cdc745c77a..2ab6f6607ea 100644 --- a/config/src/vespa/config/common/payload_converter.cpp +++ b/config/src/vespa/config/common/payload_converter.cpp @@ -12,9 +12,9 @@ PayloadConverter::PayloadConverter(const Inspector & inspector) _lines() {} -PayloadConverter::~PayloadConverter() { } +PayloadConverter::~PayloadConverter() = default; -const std::vector<vespalib::string> & +const StringVector & PayloadConverter::convert() { _lines.clear(); diff --git a/config/src/vespa/config/common/payload_converter.h b/config/src/vespa/config/common/payload_converter.h index 504fb06cf0a..1bf239b4a97 100644 --- a/config/src/vespa/config/common/payload_converter.h +++ b/config/src/vespa/config/common/payload_converter.h @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "types.h" #include <vespa/vespalib/data/slime/object_traverser.h> #include <vespa/vespalib/data/slime/array_traverser.h> -#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/asciistream.h> namespace config { @@ -16,7 +16,7 @@ class PayloadConverter : public vespalib::slime::ObjectTraverser, public vespali public: PayloadConverter(const vespalib::slime::Inspector & inspector); ~PayloadConverter(); - const std::vector<vespalib::string> & convert(); + const StringVector & convert(); void field(const vespalib::Memory & symbol, const vespalib::slime::Inspector & inspector) override; void entry(size_t idx, const vespalib::slime::Inspector & inspector) override; private: @@ -38,11 +38,11 @@ private: Node(int idx) : name(""), arrayIndex(idx) {} Node(const vespalib::string & nm) : name(nm), arrayIndex(-1) {} }; + using NodeStack = std::vector<Node>; const vespalib::slime::Inspector & _inspector; - std::vector<vespalib::string> _lines; - typedef std::vector<Node> NodeStack; - NodeStack _nodeStack; - vespalib::asciistream _buf; + StringVector _lines; + NodeStack _nodeStack; + vespalib::asciistream _buf; }; } // namespace config diff --git a/config/src/vespa/config/common/source.h b/config/src/vespa/config/common/source.h index f3014d0854c..1e1f39fd07e 100644 --- a/config/src/vespa/config/common/source.h +++ b/config/src/vespa/config/common/source.h @@ -11,13 +11,11 @@ namespace config { */ class Source { public: - typedef std::unique_ptr<Source> UP; - virtual void getConfig() = 0; virtual void reload(int64_t generation) = 0; virtual void close() = 0; - virtual ~Source() { } + virtual ~Source() = default; }; } // namespace common diff --git a/config/src/vespa/config/common/sourcefactory.h b/config/src/vespa/config/common/sourcefactory.h index de97977c924..0236bea2802 100644 --- a/config/src/vespa/config/common/sourcefactory.h +++ b/config/src/vespa/config/common/sourcefactory.h @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <memory> #include "source.h" #include "configkey.h" -#include "iconfigholder.h" namespace config { +class IConfigHolder; + /* * Source factory, creating possible config sources. */ class SourceFactory { public: typedef std::unique_ptr<SourceFactory> UP; - virtual Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const = 0; - virtual ~SourceFactory() { } + virtual std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const = 0; + virtual ~SourceFactory() = default; }; } // namespace common diff --git a/config/src/vespa/config/common/subscribehandler.h b/config/src/vespa/config/common/subscribehandler.h index 5ab70677996..da8bdfadcf2 100644 --- a/config/src/vespa/config/common/subscribehandler.h +++ b/config/src/vespa/config/common/subscribehandler.h @@ -2,10 +2,11 @@ #pragma once #include "configkey.h" -#include <vespa/config/subscription/configsubscription.h> namespace config { +class ConfigSubscription; + struct SubscribeHandler { using milliseconds = std::chrono::milliseconds; @@ -18,8 +19,8 @@ struct SubscribeHandler * @param timeoutInMillis the timeout of the subscribe call. * @return subscription object containing data relevant to client */ - virtual ConfigSubscription::SP subscribe(const ConfigKey & key, milliseconds timeoutInMillis) = 0; - virtual ~SubscribeHandler() { } + virtual std::shared_ptr<ConfigSubscription> subscribe(const ConfigKey & key, milliseconds timeoutInMillis) = 0; + virtual ~SubscribeHandler() = default; }; } diff --git a/config/src/vespa/config/common/types.h b/config/src/vespa/config/common/types.h new file mode 100644 index 00000000000..7bec2666db7 --- /dev/null +++ b/config/src/vespa/config/common/types.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/allocator.h> +#include <vector> +#include <map> + +namespace config { + +using StringVector = std::vector<vespalib::string, vespalib::allocator_large<vespalib::string>>; +using BoolVector = std::vector<bool>; +using DoubleVector = std::vector<double>; +using LongVector = std::vector<int64_t>; +using IntVector = std::vector<int32_t>; +using StringMap = std::map<vespalib::string, vespalib::string>; +using BoolMap = std::map<vespalib::string, bool>; +using DoubleMap = std::map<vespalib::string, double>; +using LongMap = std::map<vespalib::string, int64_t>; +using IntMap = std::map<vespalib::string, int32_t>; + +} diff --git a/config/src/vespa/config/config.h b/config/src/vespa/config/config.h deleted file mode 100644 index 6c383f4fef3..00000000000 --- a/config/src/vespa/config/config.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/config/subscription/configsubscriber.h> -#include <vespa/config/subscription/confighandle.h> -#include <vespa/config/subscription/sourcespec.h> -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/helper/configgetter.h> -#include <vespa/config/helper/configfetcher.h> -#include <vespa/config/common/misc.h> -#include <vespa/config/retriever/configretriever.h> - -/*! \mainpage Cloud Config API for C++ - * - * /section Introduction - * - * This document is provided as an API reference to use when developing with the - * C++ config API. - */ - -/** - * @section DESCRIPTION - * - * This file contains all necessary includes as well as functions used to - * subscribe to and retrieve config. - */ - diff --git a/config/src/vespa/config/configgen/configinstance.h b/config/src/vespa/config/configgen/configinstance.h index f25c0536ac6..21acf15859b 100644 --- a/config/src/vespa/config/configgen/configinstance.h +++ b/config/src/vespa/config/configgen/configinstance.h @@ -22,7 +22,7 @@ public: virtual void serialize(ConfigDataBuffer & buffer) const = 0; - virtual ~ConfigInstance() { } + virtual ~ConfigInstance() = default; }; } // namespace config diff --git a/config/src/vespa/config/configgen/map_inserter.h b/config/src/vespa/config/configgen/map_inserter.h index 9f596912c5f..e35ee8cb0ac 100644 --- a/config/src/vespa/config/configgen/map_inserter.h +++ b/config/src/vespa/config/configgen/map_inserter.h @@ -4,12 +4,9 @@ #include "value_converter.h" #include <vespa/vespalib/data/slime/object_traverser.h> #include <vespa/vespalib/stllike/string.h> - #include <map> -namespace config { - -namespace internal { +namespace config::internal { template<typename T, typename Converter = config::internal::ValueConverter<T> > class MapInserter : public ::vespalib::slime::ObjectTraverser { @@ -20,9 +17,4 @@ private: std::map<vespalib::string, T> & _map; }; -} // namespace internal - -} // namespace config - -#include "map_inserter.hpp" - +} diff --git a/config/src/vespa/config/configgen/map_inserter.hpp b/config/src/vespa/config/configgen/map_inserter.hpp index 6ffdb0e893b..9bd04c95b11 100644 --- a/config/src/vespa/config/configgen/map_inserter.hpp +++ b/config/src/vespa/config/configgen/map_inserter.hpp @@ -1,7 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace config { -namespace internal { +#pragma once + +#include "map_inserter.h" + +namespace config::internal { template<typename T, typename Converter> MapInserter<T, Converter>::MapInserter(std::map<vespalib::string, T> & map) @@ -16,6 +19,4 @@ MapInserter<T, Converter>::field(const ::vespalib::Memory & symbol, const ::vesp _map[symbol.make_string()] = converter(inspector); } -} // namespace internal - } diff --git a/config/src/vespa/config/configgen/value_converter.cpp b/config/src/vespa/config/configgen/value_converter.cpp index 4b25c6040da..36843456b25 100644 --- a/config/src/vespa/config/configgen/value_converter.cpp +++ b/config/src/vespa/config/configgen/value_converter.cpp @@ -54,7 +54,7 @@ template<> vespalib::string convertValue(const ::vespalib::slime::Inspector & __inspector) { return __inspector.asString().make_string(); } void -requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) { +requireValid(vespalib::stringref __fieldName, const ::vespalib::slime::Inspector & __inspector) { if (!__inspector.valid()) { throw ::config::InvalidConfigException("Value for '" + __fieldName + "' required but not found"); } diff --git a/config/src/vespa/config/configgen/value_converter.h b/config/src/vespa/config/configgen/value_converter.h index 695e798b015..c583f1595dc 100644 --- a/config/src/vespa/config/configgen/value_converter.h +++ b/config/src/vespa/config/configgen/value_converter.h @@ -7,7 +7,7 @@ namespace config::internal { -void requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector); +void requireValid(vespalib::stringref __fieldName, const ::vespalib::slime::Inspector & __inspector); template<typename T> T convertValue(const ::vespalib::slime::Inspector & __inspector) { return T(::config::ConfigPayload(__inspector)); } @@ -29,7 +29,7 @@ vespalib::string convertValue(const ::vespalib::slime::Inspector & __inspector); template<typename T> struct ValueConverter { - T operator()(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) { + T operator()(vespalib::stringref __fieldName, const ::vespalib::slime::Inspector & __inspector) { requireValid(__fieldName, __inspector); return convertValue<T>(__inspector); } diff --git a/config/src/vespa/config/configgen/vector_inserter.h b/config/src/vespa/config/configgen/vector_inserter.h index 3c5a406ef67..4b8c7a5e0a1 100644 --- a/config/src/vespa/config/configgen/vector_inserter.h +++ b/config/src/vespa/config/configgen/vector_inserter.h @@ -5,22 +5,15 @@ #include <vespa/vespalib/data/slime/array_traverser.h> #include <vespa/vespalib/stllike/string.h> -namespace config { +namespace config::internal { -namespace internal { - -template<typename T, typename Converter = ::config::internal::ValueConverter<T> > +template<typename V, typename Converter = ::config::internal::ValueConverter<typename V::value_type> > class VectorInserter : public ::vespalib::slime::ArrayTraverser { public: - VectorInserter(std::vector<T> & vector); + VectorInserter(V & vector); void entry(size_t idx, const ::vespalib::slime::Inspector & inspector) override; private: - std::vector<T> & _vector; + V & _vector; }; -} // namespace internal - -} // namespace config - -#include "vector_inserter.hpp" - +} diff --git a/config/src/vespa/config/configgen/vector_inserter.hpp b/config/src/vespa/config/configgen/vector_inserter.hpp index ed43dda07d6..31c3c52a358 100644 --- a/config/src/vespa/config/configgen/vector_inserter.hpp +++ b/config/src/vespa/config/configgen/vector_inserter.hpp @@ -1,22 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace config { -namespace internal { +#pragma once -template<typename T, typename Converter> -VectorInserter<T, Converter>::VectorInserter(std::vector<T> & vector) +#include "vector_inserter.h" + +namespace config::internal { + +template<typename V, typename Converter> +VectorInserter<V, Converter>::VectorInserter(V & vector) : _vector(vector) {} -template<typename T, typename Converter> +template<typename V, typename Converter> void -VectorInserter<T, Converter>::entry(size_t idx, const ::vespalib::slime::Inspector & inspector) +VectorInserter<V, Converter>::entry(size_t idx, const ::vespalib::slime::Inspector & inspector) { (void) idx; Converter converter; _vector.push_back(converter(inspector)); } -} // namespace internal - } diff --git a/config/src/vespa/config/file/filesource.cpp b/config/src/vespa/config/file/filesource.cpp index 83a7e10716f..011868f7c06 100644 --- a/config/src/vespa/config/file/filesource.cpp +++ b/config/src/vespa/config/file/filesource.cpp @@ -3,34 +3,34 @@ #include "filesource.h" #include <vespa/config/subscription/sourcespec.h> #include <vespa/config/common/misc.h> -#include <vespa/vespalib/io/fileutil.h> +#include <vespa/config/common/iconfigholder.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <sys/types.h> #include <sys/stat.h> -#include <unistd.h> using vespalib::asciistream; namespace config { -FileSource::FileSource(const IConfigHolder::SP & holder, const vespalib::string & fileName) - : _holder(holder), +FileSource::FileSource(std::shared_ptr<IConfigHolder> holder, const vespalib::string & fileName) + : _holder(std::move(holder)), _fileName(fileName), _lastLoaded(-1), _generation(1) { } +FileSource::~FileSource() = default; + void FileSource::getConfig() { - std::vector<vespalib::string> lines(readConfigFile(_fileName)); + StringVector lines(readConfigFile(_fileName)); int64_t last = getLast(_fileName); if (last > _lastLoaded) { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), true, _generation))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(lines), true, _generation)); _lastLoaded = last; } else { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), false, _generation))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(lines), false, _generation)); } } @@ -49,11 +49,11 @@ FileSource::getLast(const vespalib::string & fileName) return filestat.st_mtime; } -std::vector<vespalib::string> +StringVector FileSource::readConfigFile(const vespalib::string & fileName) { asciistream is(asciistream::createFromFile(fileName)); - return is.getlines(); + return getlines(is); } void diff --git a/config/src/vespa/config/file/filesource.h b/config/src/vespa/config/file/filesource.h index 5ed673a445c..f355cd2159b 100644 --- a/config/src/vespa/config/file/filesource.h +++ b/config/src/vespa/config/file/filesource.h @@ -2,29 +2,30 @@ #pragma once #include <vespa/config/common/source.h> -#include <vespa/config/common/iconfigholder.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/noncopyable.hpp> +#include <vespa/config/common/types.h> namespace config { class FileSpec; class DirSpec; +class IConfigHolder; -class FileSource : public Source, - public vespalib::noncopyable +class FileSource : public Source { private: - IConfigHolder::SP _holder; + std::shared_ptr<IConfigHolder> _holder; const vespalib::string _fileName; int64_t _lastLoaded; int64_t _generation; - std::vector<vespalib::string> readConfigFile(const vespalib::string & fileName); + StringVector readConfigFile(const vespalib::string & fileName); int64_t getLast(const vespalib::string & fileName); public: - FileSource(const IConfigHolder::SP & holder, const vespalib::string & fileName); + FileSource(std::shared_ptr<IConfigHolder> holder, const vespalib::string & fileName); + FileSource(const FileSource &) = delete; + FileSource & operator = (const FileSource &) = delete; + ~FileSource() override; void getConfig() override; void close() override; void reload(int64_t generation) override; diff --git a/config/src/vespa/config/file/filesourcefactory.cpp b/config/src/vespa/config/file/filesourcefactory.cpp index 8d8979dd553..09f1d23ed17 100644 --- a/config/src/vespa/config/file/filesourcefactory.cpp +++ b/config/src/vespa/config/file/filesourcefactory.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "filesourcefactory.h" #include "filesource.h" -#include <vespa/config/common/exceptions.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -23,8 +22,8 @@ DirSourceFactory::DirSourceFactory(const DirSpec & dirSpec) } } -Source::UP -DirSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +std::unique_ptr<Source> +DirSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { vespalib::string fileId(key.getDefName()); if (!key.getConfigId().empty()) { @@ -45,7 +44,7 @@ DirSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey vespalib::string fName = _dirName; if (!fName.empty()) fName += "/"; fName += fileId; - return Source::UP(new FileSource(holder, fName)); + return std::make_unique<FileSource>(std::move(holder), fName); } FileSourceFactory::FileSourceFactory(const FileSpec & fileSpec) @@ -53,11 +52,11 @@ FileSourceFactory::FileSourceFactory(const FileSpec & fileSpec) { } -Source::UP -FileSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +std::unique_ptr<Source> +FileSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { (void) key; - return Source::UP(new FileSource(holder, _fileName)); + return std::make_unique<FileSource>(std::move(holder), _fileName); } } // namespace config diff --git a/config/src/vespa/config/file/filesourcefactory.h b/config/src/vespa/config/file/filesourcefactory.h index c5ba5769a53..d751d499d95 100644 --- a/config/src/vespa/config/file/filesourcefactory.h +++ b/config/src/vespa/config/file/filesourcefactory.h @@ -2,7 +2,7 @@ #pragma once #include <vespa/config/common/sourcefactory.h> -#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/types.h> namespace config { @@ -20,7 +20,7 @@ public: /** * Create source handling config described by key. */ - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: vespalib::string _fileName; }; @@ -36,10 +36,10 @@ public: /** * Create source handling config described by key. */ - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: vespalib::string _dirName; - std::vector<vespalib::string> _fileNames; + StringVector _fileNames; }; diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index 827ef75251b..b6014a1d668 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -2,14 +2,15 @@ #include "frtconfigagent.h" #include "frtconfigrequestv3.h" #include <vespa/config/common/trace.h> +#include <vespa/config/common/iconfigholder.h> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconfigagent"); namespace config { -FRTConfigAgent::FRTConfigAgent(const IConfigHolder::SP & holder, const TimingValues & timingValues) - : _holder(holder), +FRTConfigAgent::FRTConfigAgent(std::shared_ptr<IConfigHolder> holder, const TimingValues & timingValues) + : _holder(std::move(holder)), _timingValues(timingValues), _configState(), _latest(), diff --git a/config/src/vespa/config/frt/frtconfigagent.h b/config/src/vespa/config/frt/frtconfigagent.h index b4303537d38..c69f4204168 100644 --- a/config/src/vespa/config/frt/frtconfigagent.h +++ b/config/src/vespa/config/frt/frtconfigagent.h @@ -3,12 +3,14 @@ #include <vespa/config/common/configstate.h> #include <vespa/config/common/timingvalues.h> -#include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/configresponse.h> #include <vespa/config/common/configrequest.h> +#include <vespa/config/common/configvalue.h> namespace config { +class IConfigHolder; + class ConfigAgent { public: @@ -19,14 +21,14 @@ public: virtual uint64_t getWaitTime() const = 0; virtual const ConfigState & getConfigState() const = 0; - virtual ~ConfigAgent() { } + virtual ~ConfigAgent() = default; }; class FRTConfigAgent : public ConfigAgent { public: - FRTConfigAgent(const IConfigHolder::SP & holder, const TimingValues & timingValues); - ~FRTConfigAgent(); + FRTConfigAgent(std::shared_ptr<IConfigHolder> holder, const TimingValues & timingValues); + ~FRTConfigAgent() override; void handleResponse(const ConfigRequest & request, ConfigResponse::UP response) override; uint64_t getTimeout() const override; uint64_t getWaitTime() const override; @@ -37,14 +39,14 @@ private: void handleErrorResponse(const ConfigRequest & request, ConfigResponse::UP response); void setWaitTime(uint64_t delay, int multiplier); - IConfigHolder::SP _holder; + std::shared_ptr<IConfigHolder> _holder; const TimingValues _timingValues; - ConfigState _configState; - ConfigValue _latest; - uint64_t _waitTime; - uint64_t _numConfigured; - unsigned int _failedRequests; - uint64_t _nextTimeout; + ConfigState _configState; + ConfigValue _latest; + uint64_t _waitTime; + uint64_t _numConfigured; + unsigned int _failedRequests; + uint64_t _nextTimeout; }; } diff --git a/config/src/vespa/config/frt/frtsourcefactory.cpp b/config/src/vespa/config/frt/frtsourcefactory.cpp index c8331cfa815..259098edb90 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.cpp +++ b/config/src/vespa/config/frt/frtsourcefactory.cpp @@ -11,10 +11,11 @@ FRTSourceFactory::FRTSourceFactory(ConnectionFactory::UP connectionFactory, cons { } -Source::UP -FRTSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +std::unique_ptr<Source> +FRTSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { - return std::make_unique<FRTSource>(_connectionFactory, _requestFactory, std::make_unique<FRTConfigAgent>(holder, _timingValues), key); + return std::make_unique<FRTSource>(_connectionFactory, _requestFactory, + std::make_unique<FRTConfigAgent>(std::move(holder), _timingValues), key); } } // namespace config diff --git a/config/src/vespa/config/frt/frtsourcefactory.h b/config/src/vespa/config/frt/frtsourcefactory.h index 23595c0d70f..4331c6411dc 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.h +++ b/config/src/vespa/config/frt/frtsourcefactory.h @@ -19,7 +19,7 @@ public: /** * Create source handling config described by key. */ - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: ConnectionFactory::SP _connectionFactory; diff --git a/config/src/vespa/config/helper/configfetcher.cpp b/config/src/vespa/config/helper/configfetcher.cpp index d85308bbcbb..b2cf6e1955d 100644 --- a/config/src/vespa/config/helper/configfetcher.cpp +++ b/config/src/vespa/config/helper/configfetcher.cpp @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configfetcher.h" +#include "configpoller.h" #include <vespa/config/common/exceptions.h> #include <vespa/config/common/configcontext.h> #include <vespa/vespalib/util/thread.h> + #include <vespa/log/log.h> LOG_SETUP(".config.helper.configfetcher"); @@ -11,19 +13,16 @@ namespace config { VESPA_THREAD_STACK_TAG(config_fetcher_thread); -ConfigFetcher::ConfigFetcher(const IConfigContext::SP & context) - : _poller(context), - _thread(std::make_unique<vespalib::Thread>(_poller, config_fetcher_thread)), +ConfigFetcher::ConfigFetcher(std::shared_ptr<IConfigContext> context) + : _poller(std::make_unique<ConfigPoller>(std::move(context))), + _thread(std::make_unique<vespalib::Thread>(*_poller, config_fetcher_thread)), _closed(false), _started(false) { } ConfigFetcher::ConfigFetcher(const SourceSpec & spec) - : _poller(std::make_shared<ConfigContext>(spec)), - _thread(std::make_unique<vespalib::Thread>(_poller, config_fetcher_thread)), - _closed(false), - _started(false) + : ConfigFetcher(std::make_shared<ConfigContext>(spec)) { } @@ -32,8 +31,8 @@ ConfigFetcher::start() { if (!_closed) { LOG(debug, "Polling for config"); - _poller.poll(); - if (_poller.getGeneration() == -1) { + _poller->poll(); + if (_poller->getGeneration() == -1) { throw ConfigTimeoutException("ConfigFetcher::start timed out getting initial config"); } LOG(debug, "Starting fetcher thread..."); @@ -48,11 +47,16 @@ ConfigFetcher::~ConfigFetcher() close(); } +int64_t +ConfigFetcher::getGeneration() const { + return _poller->getGeneration(); +} + void ConfigFetcher::close() { if (!_closed) { - _poller.close(); + _poller->close(); if (_started) _thread->join(); } diff --git a/config/src/vespa/config/helper/configfetcher.h b/config/src/vespa/config/helper/configfetcher.h index e8cd6a66ebc..8891f0989e5 100644 --- a/config/src/vespa/config/helper/configfetcher.h +++ b/config/src/vespa/config/helper/configfetcher.h @@ -1,14 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "configpoller.h" #include <vespa/config/common/timingvalues.h> +#include <vespa/config/helper/ifetchercallback.h> +#include <vespa/config/subscription/sourcespec.h> #include <atomic> namespace vespalib { class Thread; } namespace config { +class ConfigPoller; +class IConfigContext; + /** * A config fetcher subscribes to a config and notifies a callback when done */ @@ -16,7 +20,7 @@ class ConfigFetcher { public: using milliseconds = std::chrono::milliseconds; - ConfigFetcher(const IConfigContext::SP & context); + ConfigFetcher(std::shared_ptr<IConfigContext> context); ConfigFetcher(const SourceSpec & spec = ServerSpec()); ~ConfigFetcher(); @@ -25,16 +29,12 @@ public: void start(); void close(); - int64_t getGeneration() const { return _poller.getGeneration(); } + int64_t getGeneration() const; private: - ConfigPoller _poller; + std::unique_ptr<ConfigPoller> _poller; std::unique_ptr<vespalib::Thread> _thread; std::atomic<bool> _closed; std::atomic<bool> _started; }; } // namespace config - - -#include "configfetcher.hpp" - diff --git a/config/src/vespa/config/helper/configfetcher.hpp b/config/src/vespa/config/helper/configfetcher.hpp index 9123c54797e..585a798b2ba 100644 --- a/config/src/vespa/config/helper/configfetcher.hpp +++ b/config/src/vespa/config/helper/configfetcher.hpp @@ -1,12 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configfetcher.h" +#include "configpoller.hpp" + namespace config { template <typename ConfigType> void ConfigFetcher::subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, milliseconds subscribeTimeout) { - _poller.subscribe<ConfigType>(configId, callback, subscribeTimeout); + _poller->subscribe<ConfigType>(configId, callback, subscribeTimeout); } } // namespace config diff --git a/config/src/vespa/config/helper/configgetter.h b/config/src/vespa/config/helper/configgetter.h index fab8ee98da4..dfb61da3c6f 100644 --- a/config/src/vespa/config/helper/configgetter.h +++ b/config/src/vespa/config/helper/configgetter.h @@ -19,9 +19,9 @@ class ConfigGetter public: using milliseconds = std::chrono::milliseconds; static std::unique_ptr<ConfigType> getConfig(int64_t &generation, const std::string & configId, const SourceSpec & spec = ServerSpec()); - static std::unique_ptr<ConfigType> getConfig(int64_t &generation, const std::string & configId, const std::shared_ptr<IConfigContext> & context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + static std::unique_ptr<ConfigType> getConfig(int64_t &generation, const std::string & configId, std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); static std::unique_ptr<ConfigType> getConfig(const std::string & configId, const SourceSpec & spec = ServerSpec()); - static std::unique_ptr<ConfigType> getConfig(const std::string & configId, const std::shared_ptr<IConfigContext> & context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + static std::unique_ptr<ConfigType> getConfig(const std::string & configId, std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); }; } // namespace config diff --git a/config/src/vespa/config/helper/configgetter.hpp b/config/src/vespa/config/helper/configgetter.hpp index 0a757ada83e..b01781ee313 100644 --- a/config/src/vespa/config/helper/configgetter.hpp +++ b/config/src/vespa/config/helper/configgetter.hpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configgetter.h" -#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/subscription/configsubscriber.hpp> namespace config { @@ -18,9 +18,9 @@ ConfigGetter<ConfigType>::getConfig(int64_t &generation, const std::string & con template <typename ConfigType> std::unique_ptr<ConfigType> -ConfigGetter<ConfigType>::getConfig(int64_t &generation, const std::string & configId, const IConfigContext::SP & context, milliseconds subscribeTimeout) +ConfigGetter<ConfigType>::getConfig(int64_t &generation, const std::string & configId, std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout) { - ConfigSubscriber s(context); + ConfigSubscriber s(std::move(context)); std::unique_ptr< ConfigHandle<ConfigType> > h = s.subscribe<ConfigType>(configId, subscribeTimeout); s.nextConfigNow(); generation = s.getGeneration(); @@ -37,10 +37,10 @@ ConfigGetter<ConfigType>::getConfig(const std::string & configId, const SourceSp template <typename ConfigType> std::unique_ptr<ConfigType> -ConfigGetter<ConfigType>::getConfig(const std::string & configId, const IConfigContext::SP & context, milliseconds subscribeTimeout) +ConfigGetter<ConfigType>::getConfig(const std::string & configId, std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout) { int64_t ignoreGeneration; - return getConfig(ignoreGeneration, configId, context, subscribeTimeout); + return getConfig(ignoreGeneration, configId, std::move(context), subscribeTimeout); } } // namespace config diff --git a/config/src/vespa/config/helper/configpoller.cpp b/config/src/vespa/config/helper/configpoller.cpp index 879c21bea8c..e9ed0af6b5e 100644 --- a/config/src/vespa/config/helper/configpoller.cpp +++ b/config/src/vespa/config/helper/configpoller.cpp @@ -2,15 +2,16 @@ #include "configpoller.h" #include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/configsubscriber.h> #include <vespa/log/log.h> LOG_SETUP(".config.helper.configpoller"); namespace config { -ConfigPoller::ConfigPoller(const IConfigContext::SP & context) +ConfigPoller::ConfigPoller(std::shared_ptr<IConfigContext> context) : _generation(-1), - _subscriber(context), + _subscriber(std::make_unique<ConfigSubscriber>(std::move(context))), _handleList(), _callbackList() { @@ -22,7 +23,7 @@ void ConfigPoller::run() { try { - while (!_subscriber.isClosed()) { + while (!_subscriber->isClosed()) { poll(); } } catch (config::InvalidConfigException & e) { @@ -35,11 +36,11 @@ void ConfigPoller::poll() { LOG(debug, "Checking for new config"); - if (_subscriber.nextGeneration()) { - if (_subscriber.isClosed()) + if (_subscriber->nextGeneration()) { + if (_subscriber->isClosed()) return; LOG(debug, "Got new config, reconfiguring"); - _generation = _subscriber.getGeneration(); + _generation = _subscriber->getGeneration(); for (size_t i = 0; i < _handleList.size(); i++) { ICallback * callback(_callbackList[i]); if (_handleList[i]->isChanged()) @@ -53,7 +54,7 @@ ConfigPoller::poll() void ConfigPoller::close() { - _subscriber.close(); + _subscriber->close(); } } diff --git a/config/src/vespa/config/helper/configpoller.h b/config/src/vespa/config/helper/configpoller.h index 40fa1784f59..44d3a51595d 100644 --- a/config/src/vespa/config/helper/configpoller.h +++ b/config/src/vespa/config/helper/configpoller.h @@ -3,12 +3,15 @@ #include "ifetchercallback.h" #include "ihandle.h" -#include <vespa/config/subscription/configsubscriber.h> #include <vespa/config/common/timingvalues.h> #include <vespa/vespalib/util/runnable.h> +#include <vector> namespace config { +class IConfigContext; +class ConfigSubscriber; + /** * A config poller runs a polling sequence on a set of configs that it has * subscribed to. @@ -16,8 +19,8 @@ namespace config { class ConfigPoller : public vespalib::Runnable { public: using milliseconds = std::chrono::milliseconds; - ConfigPoller(const IConfigContext::SP & context); - ~ConfigPoller(); + ConfigPoller(std::shared_ptr<IConfigContext> context); + ~ConfigPoller() override; void run() override; template <typename ConfigType> void subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); @@ -26,11 +29,9 @@ public: int64_t getGeneration() const { return _generation; } private: int64_t _generation; - ConfigSubscriber _subscriber; - std::vector<IHandle::UP> _handleList; - std::vector<ICallback *> _callbackList; + std::unique_ptr<ConfigSubscriber> _subscriber; + std::vector<IHandle::UP> _handleList; + std::vector<ICallback *> _callbackList; }; } // namespace config - -#include "configpoller.hpp" diff --git a/config/src/vespa/config/helper/configpoller.hpp b/config/src/vespa/config/helper/configpoller.hpp index 4088a311609..540ea160e7e 100644 --- a/config/src/vespa/config/helper/configpoller.hpp +++ b/config/src/vespa/config/helper/configpoller.hpp @@ -1,12 +1,34 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configpoller.h" +#include <vespa/config/subscription/configsubscriber.hpp> + namespace config { template <typename ConfigType> +class GenericHandle : public IHandle +{ +public: + GenericHandle(std::unique_ptr<ConfigHandle<ConfigType> > handle) + : _handle(std::move(handle)) + { + } + + std::unique_ptr<const ConfigInstance> getConfig() override { + return std::unique_ptr<const ConfigInstance>(_handle->getConfig().release()); + } + bool isChanged() override { return _handle->isChanged(); } +private: + std::unique_ptr<ConfigHandle <ConfigType> > _handle; +}; + +template <typename ConfigType> void ConfigPoller::subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, milliseconds subscribeTimeout) { - std::unique_ptr<ConfigHandle<ConfigType> > handle(_subscriber.subscribe<ConfigType>(configId, subscribeTimeout)); + std::unique_ptr<ConfigHandle<ConfigType> > handle(_subscriber->subscribe<ConfigType>(configId, subscribeTimeout)); _handleList.emplace_back(std::make_unique<GenericHandle<ConfigType>>(std::move(handle))); _callbackList.push_back(callback); } diff --git a/config/src/vespa/config/helper/ihandle.h b/config/src/vespa/config/helper/ihandle.h index 4d8de8243d0..271b9f9051d 100644 --- a/config/src/vespa/config/helper/ihandle.h +++ b/config/src/vespa/config/helper/ihandle.h @@ -1,34 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/config/subscription/confighandle.h> +#include <memory> namespace config { +class ConfigInstance; + class IHandle { public: typedef std::unique_ptr<IHandle> UP; virtual std::unique_ptr<const ConfigInstance> getConfig() = 0; virtual bool isChanged() = 0; - virtual ~IHandle() { } -}; - -template <typename ConfigType> -class GenericHandle : public IHandle -{ -public: - GenericHandle(std::unique_ptr<ConfigHandle<ConfigType> > handle) - : _handle(std::move(handle)) - { - } - - std::unique_ptr<const ConfigInstance> getConfig() override { - return std::unique_ptr<const ConfigInstance>(_handle->getConfig().release()); - } - bool isChanged() override { return _handle->isChanged(); } -private: - std::unique_ptr<ConfigHandle <ConfigType> > _handle; + virtual ~IHandle() = default; }; } diff --git a/config/src/vespa/config/helper/legacysubscriber.h b/config/src/vespa/config/helper/legacysubscriber.h index 5e370a3a6c2..0389c346096 100644 --- a/config/src/vespa/config/helper/legacysubscriber.h +++ b/config/src/vespa/config/helper/legacysubscriber.h @@ -3,7 +3,6 @@ #include "legacy.h" #include "configfetcher.h" -#include <vespa/config/subscription/sourcespec.h> namespace config { @@ -27,6 +26,3 @@ private: }; } // namespace config - -#include "legacysubscriber.hpp" - diff --git a/config/src/vespa/config/helper/legacysubscriber.hpp b/config/src/vespa/config/helper/legacysubscriber.hpp index 9610f8f78db..c11f97e81b7 100644 --- a/config/src/vespa/config/helper/legacysubscriber.hpp +++ b/config/src/vespa/config/helper/legacysubscriber.hpp @@ -1,5 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "legacysubscriber.h" +#include "configfetcher.hpp" #include <vespa/config/common/configcontext.h> namespace config { diff --git a/config/src/vespa/config/print/asciiconfigreader.h b/config/src/vespa/config/print/asciiconfigreader.h index 11a9da2da4e..5be777386b4 100644 --- a/config/src/vespa/config/print/asciiconfigreader.h +++ b/config/src/vespa/config/print/asciiconfigreader.h @@ -2,11 +2,11 @@ #pragma once #include "configreader.h" -#include "configformatter.h" -#include <vespa/vespalib/stllike/asciistream.h> namespace config { +class ConfigFormatter; + /** * Read a config from istream */ @@ -22,6 +22,3 @@ private: }; } // namespace config - -#include "asciiconfigreader.hpp" - diff --git a/config/src/vespa/config/print/asciiconfigreader.hpp b/config/src/vespa/config/print/asciiconfigreader.hpp index 3a76f7536b9..6f9919e6f71 100644 --- a/config/src/vespa/config/print/asciiconfigreader.hpp +++ b/config/src/vespa/config/print/asciiconfigreader.hpp @@ -1,5 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "asciiconfigreader.h" +#include <vespa/config/common/types.h> +#include <vespa/config/common/configvalue.h> + namespace config { template <typename ConfigType> @@ -15,19 +21,19 @@ AsciiConfigReader<ConfigType>::read(const ConfigFormatter & formatter) ConfigDataBuffer buffer; buffer.setEncodedString(_is.str()); formatter.decode(buffer); - return std::unique_ptr<ConfigType>(new ConfigType(buffer)); + return std::make_unique<ConfigType>(buffer); } template <typename ConfigType> std::unique_ptr<ConfigType> AsciiConfigReader<ConfigType>::read() { - std::vector<vespalib::string> lines; + StringVector lines; vespalib::string line; while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); + return std::make_unique<ConfigType>(ConfigValue(std::move(lines))); } } // namespace config diff --git a/config/src/vespa/config/print/fileconfigreader.h b/config/src/vespa/config/print/fileconfigreader.h index 1d47d9c04a7..ca794f6c8a6 100644 --- a/config/src/vespa/config/print/fileconfigreader.h +++ b/config/src/vespa/config/print/fileconfigreader.h @@ -24,6 +24,3 @@ private: }; } // namespace config - -#include "fileconfigreader.hpp" - diff --git a/config/src/vespa/config/print/fileconfigreader.hpp b/config/src/vespa/config/print/fileconfigreader.hpp index dec390ef358..b4d0dac86f5 100644 --- a/config/src/vespa/config/print/fileconfigreader.hpp +++ b/config/src/vespa/config/print/fileconfigreader.hpp @@ -29,14 +29,14 @@ FileConfigReader<ConfigType>::read(const ConfigFormatter & formatter) buf << file.rdbuf(); buffer.setEncodedString(buf.str()); formatter.decode(buffer); - return std::unique_ptr<ConfigType>(new ConfigType(buffer)); + return std::make_unique<ConfigType>(buffer); } template <typename ConfigType> std::unique_ptr<ConfigType> FileConfigReader<ConfigType>::read() { - std::vector<vespalib::string> lines; + StringVector lines; std::ifstream f(_fileName.c_str()); if (f.fail()) throw vespalib::IllegalArgumentException(std::string("Unable to open file ") + _fileName); @@ -44,7 +44,7 @@ FileConfigReader<ConfigType>::read() for (std::getline(f, line); f; std::getline(f, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); + return std::make_unique<ConfigType>(ConfigValue(std::move(lines))); } } // namespace config diff --git a/config/src/vespa/config/print/istreamconfigreader.h b/config/src/vespa/config/print/istreamconfigreader.h index 511fbe2f538..d941d1582ac 100644 --- a/config/src/vespa/config/print/istreamconfigreader.h +++ b/config/src/vespa/config/print/istreamconfigreader.h @@ -21,6 +21,3 @@ private: }; } // namespace config - -#include "istreamconfigreader.hpp" - diff --git a/config/src/vespa/config/print/istreamconfigreader.hpp b/config/src/vespa/config/print/istreamconfigreader.hpp index 11b41ddbe2f..1cc0d975919 100644 --- a/config/src/vespa/config/print/istreamconfigreader.hpp +++ b/config/src/vespa/config/print/istreamconfigreader.hpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <istream> +#include "istreamconfigreader.h" +#include <sstream> namespace config { @@ -19,19 +20,19 @@ IstreamConfigReader<ConfigType>::read(const ConfigFormatter & formatter) buf << _is.rdbuf(); buffer.setEncodedString(buf.str()); formatter.decode(buffer); - return std::unique_ptr<ConfigType>(new ConfigType(buffer)); + return std::make_unique<ConfigType>(buffer); } template <typename ConfigType> std::unique_ptr<ConfigType> IstreamConfigReader<ConfigType>::read() { - std::vector<vespalib::string> lines; + StringVector lines; std::string line; while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); + return std::make_unique<ConfigType>(ConfigValue(std::move(lines))); } } // namespace config diff --git a/config/src/vespa/config/raw/rawsource.cpp b/config/src/vespa/config/raw/rawsource.cpp index af3d7f6f7f5..cf551106405 100644 --- a/config/src/vespa/config/raw/rawsource.cpp +++ b/config/src/vespa/config/raw/rawsource.cpp @@ -1,13 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rawsource.h" #include <vespa/config/common/misc.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configvalue.h> #include <vespa/vespalib/stllike/asciistream.h> namespace config { +RawSource::~RawSource() = default; -RawSource::RawSource(const IConfigHolder::SP & holder, const vespalib::string & payload) - : _holder(holder), +RawSource::RawSource(std::shared_ptr<IConfigHolder> holder, const vespalib::string & payload) + : _holder(std::move(holder)), _payload(payload) { } @@ -15,9 +18,7 @@ RawSource::RawSource(const IConfigHolder::SP & holder, const vespalib::string & void RawSource::getConfig() { - auto lines(readConfig()); - ConfigValue value(lines, calculateContentXxhash64(lines)); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 1))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(readConfig()), true, 1)); } void @@ -31,11 +32,11 @@ RawSource::close() { } -std::vector<vespalib::string> +StringVector RawSource::readConfig() { vespalib::asciistream is(_payload); - return is.getlines(); + return getlines(is); } } diff --git a/config/src/vespa/config/raw/rawsource.h b/config/src/vespa/config/raw/rawsource.h index 2b15e459de4..a6e1e806cbf 100644 --- a/config/src/vespa/config/raw/rawsource.h +++ b/config/src/vespa/config/raw/rawsource.h @@ -2,24 +2,26 @@ #pragma once #include <vespa/config/common/source.h> -#include <vespa/config/common/iconfigholder.h> -#include <vespa/vespalib/stllike/string.h> - +#include <vespa/config/common/types.h> namespace config { +class IConfigHolder; + /** * Class for sending and receiving config request from a raw string. */ class RawSource : public Source { public: - RawSource(const IConfigHolder::SP & holder, const vespalib::string & payload); - + RawSource(std::shared_ptr<IConfigHolder> holder, const vespalib::string & payload); + RawSource(const RawSource &) = delete; + RawSource & operator = (const RawSource &) = delete; + ~RawSource() override; void getConfig() override; void reload(int64_t generation) override; void close() override; private: - IConfigHolder::SP _holder; - std::vector<vespalib::string> readConfig(); + std::shared_ptr<IConfigHolder> _holder; + StringVector readConfig(); const vespalib::string _payload; }; diff --git a/config/src/vespa/config/raw/rawsourcefactory.cpp b/config/src/vespa/config/raw/rawsourcefactory.cpp index d6f3dfd6be9..396ee8b7c27 100644 --- a/config/src/vespa/config/raw/rawsourcefactory.cpp +++ b/config/src/vespa/config/raw/rawsourcefactory.cpp @@ -6,11 +6,11 @@ namespace config { -Source::UP -RawSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +std::unique_ptr<Source> +RawSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { (void) key; - return Source::UP(new RawSource(holder, _payload)); + return std::make_unique<RawSource>(std::move(holder), _payload); } } diff --git a/config/src/vespa/config/raw/rawsourcefactory.h b/config/src/vespa/config/raw/rawsourcefactory.h index 191547b5d8c..3b8d8986625 100644 --- a/config/src/vespa/config/raw/rawsourcefactory.h +++ b/config/src/vespa/config/raw/rawsourcefactory.h @@ -14,7 +14,7 @@ public: : _payload(payload) { } - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: const vespalib::string _payload; }; diff --git a/config/src/vespa/config/retriever/configretriever.cpp b/config/src/vespa/config/retriever/configretriever.cpp index 20c81742821..125d8ef3012 100644 --- a/config/src/vespa/config/retriever/configretriever.cpp +++ b/config/src/vespa/config/retriever/configretriever.cpp @@ -13,7 +13,7 @@ namespace config { const milliseconds ConfigRetriever::DEFAULT_NEXTGENERATION_TIMEOUT(60000); ConfigRetriever::ConfigRetriever(const ConfigKeySet & bootstrapSet, - const IConfigContext::SP & context, + std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout) : _bootstrapSubscriber(bootstrapSet, context, subscribeTimeout), _configSubscriber(), diff --git a/config/src/vespa/config/retriever/configretriever.h b/config/src/vespa/config/retriever/configretriever.h index ad57f4887fc..8a47ad825cb 100644 --- a/config/src/vespa/config/retriever/configretriever.h +++ b/config/src/vespa/config/retriever/configretriever.h @@ -27,7 +27,7 @@ class ConfigRetriever public: using milliseconds = std::chrono::milliseconds; ConfigRetriever(const ConfigKeySet & bootstrapSet, - const IConfigContext::SP & context, + std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); ~ConfigRetriever(); @@ -93,13 +93,13 @@ public: static const milliseconds DEFAULT_SUBSCRIBE_TIMEOUT; static const milliseconds DEFAULT_NEXTGENERATION_TIMEOUT; private: - FixedConfigSubscriber _bootstrapSubscriber; - std::unique_ptr<GenericConfigSubscriber> _configSubscriber; - std::mutex _lock; - std::vector<ConfigSubscription::SP> _subscriptionList; - ConfigKeySet _lastKeySet; - IConfigContext::SP _context; - std::unique_ptr<SourceSpec> _spec; + FixedConfigSubscriber _bootstrapSubscriber; + std::unique_ptr<GenericConfigSubscriber> _configSubscriber; + std::mutex _lock; + std::vector<std::shared_ptr<ConfigSubscription>> _subscriptionList; + ConfigKeySet _lastKeySet; + std::shared_ptr<IConfigContext> _context; + std::unique_ptr<SourceSpec> _spec; bool _closed; int64_t _generation; milliseconds _subscribeTimeout; diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp index e999538e713..293a90ebbba 100644 --- a/config/src/vespa/config/retriever/configsnapshot.cpp +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -5,6 +5,7 @@ #include <vespa/config/print/configdatabuffer.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/common/misc.h> +#include <vespa/config/frt/protocol.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -41,8 +42,9 @@ ConfigSnapshot::swap(ConfigSnapshot &rhs) { } ConfigSnapshot::ConfigSnapshot(const SubscriptionList &subscriptionList, int64_t generation) - : _valueMap(), - _generation(generation) { + : _valueMap(), + _generation(generation) +{ for (SubscriptionList::const_iterator it(subscriptionList.begin()), mt(subscriptionList.end()); it != mt; it++) { _valueMap[(*it)->getKey()] = Value((*it)->getLastGenerationChanged(), (*it)->getConfig()); } @@ -66,10 +68,10 @@ ConfigSnapshot ConfigSnapshot::subset(const ConfigKeySet & keySet) const { ValueMap subSet; - for (ConfigKeySet::const_iterator it(keySet.begin()), mt(keySet.end()); it != mt; it++) { - ValueMap::const_iterator found(_valueMap.find(*it)); + for (const ConfigKey & key : keySet) { + ValueMap::const_iterator found(_valueMap.find(key)); if (found != _valueMap.end()) { - subSet[*it] = found->second; + subSet[key] = found->second; } } return ConfigSnapshot(subSet, _generation); @@ -127,15 +129,13 @@ ConfigSnapshot::serializeV2(Cursor & root) const void ConfigSnapshot::serializeKeyV1(Cursor & cursor, const ConfigKey & key) const { - typedef std::vector<vespalib::string> SchemaVector; cursor.setString("configId", Memory(key.getConfigId())); cursor.setString("defName", Memory(key.getDefName())); cursor.setString("defNamespace", Memory(key.getDefNamespace())); cursor.setString("defMd5", Memory(key.getDefMd5())); Cursor & defSchema(cursor.setArray("defSchema")); - const SchemaVector & vec(key.getDefSchema()); - for (SchemaVector::const_iterator it(vec.begin()), mt(vec.end()); it != mt; it++) { - defSchema.addString(vespalib::Memory(*it)); + for (const vespalib::string & line : key.getDefSchema()) { + defSchema.addString(vespalib::Memory(line)); } } @@ -203,7 +203,7 @@ ConfigSnapshot::deserializeV2(Inspector & root) ConfigKey ConfigSnapshot::deserializeKeyV1(Inspector & inspector) const { - std::vector<vespalib::string> schema; + StringVector schema; Inspector & s(inspector["defSchema"]); for (size_t i = 0; i < s.children(); i++) { schema.push_back(s[i].asString().make_string()); @@ -219,13 +219,13 @@ ConfigSnapshot::deserializeKeyV1(Inspector & inspector) const std::pair<int64_t, ConfigValue> ConfigSnapshot::deserializeValueV1(Inspector & inspector) const { - std::vector<vespalib::string> payload; + StringVector payload; int64_t lastChanged = static_cast<int64_t>(inspector["lastChanged"].asDouble()); Inspector & s(inspector["lines"]); for (size_t i = 0; i < s.children(); i++) { payload.push_back(s[i].asString().make_string()); } - return Value(lastChanged, ConfigValue(payload, calculateContentXxhash64(payload))); + return Value(lastChanged, ConfigValue(payload)); } namespace { diff --git a/config/src/vespa/config/retriever/configsnapshot.h b/config/src/vespa/config/retriever/configsnapshot.h index b7d49ca2584..8393fd38ed3 100644 --- a/config/src/vespa/config/retriever/configsnapshot.h +++ b/config/src/vespa/config/retriever/configsnapshot.h @@ -3,8 +3,7 @@ #include "configkeyset.h" #include <vespa/config/common/configvalue.h> -#include <vespa/vespalib/stllike/string.h> -#include <map> +#include <vespa/config/common/misc.h> namespace config { @@ -18,7 +17,7 @@ class ConfigDataBuffer; class ConfigSnapshot { public: - typedef std::vector<std::shared_ptr<ConfigSubscription>> SubscriptionList; + using SubscriptionList = std::vector<std::shared_ptr<ConfigSubscription>>; /** * Construct an empty config snapshot. @@ -95,8 +94,8 @@ public: void serialize(ConfigDataBuffer & buffer) const; void deserialize(const ConfigDataBuffer & buffer); private: - typedef std::pair<int64_t, ConfigValue> Value; - typedef std::map<ConfigKey, Value> ValueMap; + using Value = std::pair<int64_t, ConfigValue>; + using ValueMap = std::map<ConfigKey, Value>; const static int64_t SNAPSHOT_FORMAT_VERSION; ConfigSnapshot(const ValueMap & valueMap, int64_t generation); @@ -121,5 +120,3 @@ private: }; } // namespace config - -#include "configsnapshot.hpp" diff --git a/config/src/vespa/config/retriever/configsnapshot.hpp b/config/src/vespa/config/retriever/configsnapshot.hpp index 387fc9712e0..5513eaa9d65 100644 --- a/config/src/vespa/config/retriever/configsnapshot.hpp +++ b/config/src/vespa/config/retriever/configsnapshot.hpp @@ -1,5 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configsnapshot.h" +#include <vespa/config/common/configvalue.hpp> + namespace config { template <typename ConfigType> diff --git a/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp b/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp index 07049a8ac2b..da199c2e298 100644 --- a/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp +++ b/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp @@ -3,9 +3,9 @@ namespace config { FixedConfigSubscriber::FixedConfigSubscriber(const ConfigKeySet & keySet, - const IConfigContext::SP & context, + std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout) - : _set(context), + : _set(std::move(context)), _subscriptionList() { for (const ConfigKey & key : keySet) { diff --git a/config/src/vespa/config/retriever/fixedconfigsubscriber.h b/config/src/vespa/config/retriever/fixedconfigsubscriber.h index 5ae684db782..1ca4d8cf0b3 100644 --- a/config/src/vespa/config/retriever/fixedconfigsubscriber.h +++ b/config/src/vespa/config/retriever/fixedconfigsubscriber.h @@ -15,14 +15,14 @@ class FixedConfigSubscriber { public: using milliseconds = std::chrono::milliseconds; - FixedConfigSubscriber(const ConfigKeySet & keySet, const IConfigContext::SP & context, milliseconds subscribeTimeout); + FixedConfigSubscriber(const ConfigKeySet & keySet, std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout); bool nextGeneration(milliseconds timeoutInMillis); void close(); int64_t getGeneration() const; ConfigSnapshot getConfigSnapshot() const; private: ConfigSubscriptionSet _set; - std::vector<ConfigSubscription::SP> _subscriptionList; + std::vector<std::shared_ptr<ConfigSubscription>> _subscriptionList; }; } // namespace config diff --git a/config/src/vespa/config/retriever/genericconfigsubscriber.cpp b/config/src/vespa/config/retriever/genericconfigsubscriber.cpp index 96146bb8927..4190c404d04 100644 --- a/config/src/vespa/config/retriever/genericconfigsubscriber.cpp +++ b/config/src/vespa/config/retriever/genericconfigsubscriber.cpp @@ -3,8 +3,8 @@ namespace config { -GenericConfigSubscriber::GenericConfigSubscriber(const IConfigContext::SP & context) - : _set(context) +GenericConfigSubscriber::GenericConfigSubscriber(std::shared_ptr<IConfigContext> context) + : _set(std::move(context)) { } bool @@ -13,7 +13,7 @@ GenericConfigSubscriber::nextGeneration(milliseconds timeoutInMillis) return _set.acquireSnapshot(timeoutInMillis, true); } -ConfigSubscription::SP +std::shared_ptr<ConfigSubscription> GenericConfigSubscriber::subscribe(const ConfigKey & key, milliseconds timeoutInMillis) { return _set.subscribe(key, timeoutInMillis); diff --git a/config/src/vespa/config/retriever/genericconfigsubscriber.h b/config/src/vespa/config/retriever/genericconfigsubscriber.h index 17d37ffb7ff..b2b7be5b9e4 100644 --- a/config/src/vespa/config/retriever/genericconfigsubscriber.h +++ b/config/src/vespa/config/retriever/genericconfigsubscriber.h @@ -14,9 +14,9 @@ class GenericConfigSubscriber { public: using milliseconds = std::chrono::milliseconds; - GenericConfigSubscriber(const IConfigContext::SP & context); + GenericConfigSubscriber(std::shared_ptr<IConfigContext> context); bool nextGeneration(milliseconds timeoutInMillis); - ConfigSubscription::SP subscribe(const ConfigKey & key, milliseconds timeoutInMillis); + std::shared_ptr<ConfigSubscription> subscribe(const ConfigKey & key, milliseconds timeoutInMillis); void close(); int64_t getGeneration() const; private: diff --git a/config/src/vespa/config/retriever/simpleconfigretriever.cpp b/config/src/vespa/config/retriever/simpleconfigretriever.cpp index fc4f21718af..43dad7d9429 100644 --- a/config/src/vespa/config/retriever/simpleconfigretriever.cpp +++ b/config/src/vespa/config/retriever/simpleconfigretriever.cpp @@ -3,7 +3,7 @@ namespace config { SimpleConfigRetriever::SimpleConfigRetriever(const ConfigKeySet & keySet, - const IConfigContext::SP & context, + std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout) : _set(context), _subscriptionList() diff --git a/config/src/vespa/config/retriever/simpleconfigretriever.h b/config/src/vespa/config/retriever/simpleconfigretriever.h index 1e19789db9f..2650532cda2 100644 --- a/config/src/vespa/config/retriever/simpleconfigretriever.h +++ b/config/src/vespa/config/retriever/simpleconfigretriever.h @@ -20,7 +20,7 @@ public: using milliseconds = std::chrono::milliseconds; SimpleConfigRetriever(const ConfigKeySet & keySet, - const IConfigContext::SP & context, + std::shared_ptr<IConfigContext> context, milliseconds subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); /** @@ -35,7 +35,7 @@ public: private: ConfigSubscriptionSet _set; - std::vector<ConfigSubscription::SP> _subscriptionList; + std::vector<std::shared_ptr<ConfigSubscription>> _subscriptionList; }; } // namespace config diff --git a/config/src/vespa/config/set/configinstancesourcefactory.cpp b/config/src/vespa/config/set/configinstancesourcefactory.cpp index 3ed0fdc932e..15a6125d096 100644 --- a/config/src/vespa/config/set/configinstancesourcefactory.cpp +++ b/config/src/vespa/config/set/configinstancesourcefactory.cpp @@ -1,26 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configinstancesourcefactory.h" +#include <vespa/config/common/iconfigholder.h> namespace { class ConfigInstanceSource : public config::Source { public: - ConfigInstanceSource(const config::IConfigHolder::SP & holder, const vespalib::asciistream & buffer) - : _holder(holder), - _buffer(buffer), + ConfigInstanceSource(std::shared_ptr<config::IConfigHolder> holder, vespalib::asciistream buffer) + : _holder(std::move(holder)), + _buffer(std::move(buffer)), _generation(-1) { } void close() override { } void getConfig() override { - std::vector<vespalib::string> lines(_buffer.getlines()); - std::string currentXxhash64(config::calculateContentXxhash64(lines)); - _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentXxhash64), true, _generation))); + _holder->handle(std::make_unique<config::ConfigUpdate>(config::ConfigValue(config::getlines(_buffer)), true, _generation)); } void reload(int64_t generation) override { _generation = generation; } private: - config::IConfigHolder::SP _holder; + std::shared_ptr<config::IConfigHolder> _holder; vespalib::asciistream _buffer; int64_t _generation; }; @@ -29,18 +28,18 @@ private: namespace config { -ConfigInstanceSourceFactory::ConfigInstanceSourceFactory(const ConfigKey & key, const vespalib::asciistream & buffer) +ConfigInstanceSourceFactory::ConfigInstanceSourceFactory(const ConfigKey & key, vespalib::asciistream buffer) : _key(key), - _buffer(buffer) + _buffer(std::move(buffer)) { } -Source::UP -ConfigInstanceSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +std::unique_ptr<Source> +ConfigInstanceSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { (void) key; // TODO: Check key against _key - return Source::UP(new ConfigInstanceSource(holder, _buffer)); + return std::make_unique<ConfigInstanceSource>(std::move(holder), _buffer); } } // namespace config diff --git a/config/src/vespa/config/set/configinstancesourcefactory.h b/config/src/vespa/config/set/configinstancesourcefactory.h index 279da055d91..bff81a457c4 100644 --- a/config/src/vespa/config/set/configinstancesourcefactory.h +++ b/config/src/vespa/config/set/configinstancesourcefactory.h @@ -17,12 +17,12 @@ class ConfigKey; class ConfigInstanceSourceFactory : public SourceFactory { public: - ConfigInstanceSourceFactory(const ConfigKey & key, const vespalib::asciistream & buffer); + ConfigInstanceSourceFactory(const ConfigKey & key, vespalib::asciistream buffer); /** * Create source handling config described by key. */ - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: const ConfigKey _key; vespalib::asciistream _buffer; diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index c60126b17e7..41886a12d01 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -2,6 +2,7 @@ #include "configsetsource.h" #include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/exceptions.h> #include <vespa/log/log.h> @@ -9,17 +10,17 @@ LOG_SETUP(".config.set.configsetsource"); namespace config { -ConfigSetSource::ConfigSetSource(const IConfigHolder::SP & holder, const ConfigKey & key, const BuilderMapSP & builderMap) - : _holder(holder), +ConfigSetSource::ConfigSetSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key, BuilderMapSP builderMap) + : _holder(std::move(holder)), _key(key), _generation(1), - _builderMap(builderMap) + _builderMap(std::move(builderMap)) { if (!validRequest(key)) throw ConfigRuntimeException("Invalid subscribe for key " + key.toString() + ", not builder found"); } -ConfigSetSource::~ConfigSetSource() { } +ConfigSetSource::~ConfigSetSource() = default; void ConfigSetSource::getConfig() @@ -29,17 +30,17 @@ ConfigSetSource::getConfig() vespalib::asciistream ss; AsciiConfigWriter writer(ss); writer.write(*instance); - std::vector<vespalib::string> lines(ss.getlines()); - std::string currentXxhash64(calculateContentXxhash64(lines)); + StringVector lines(getlines(ss)); + vespalib::string currentXxhash64(calculateContentXxhash64(lines)); if (isGenerationNewer(_generation, _lastState.generation) && currentXxhash64.compare(_lastState.xxhash64) != 0) { LOG(debug, "New generation, updating"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), true, _generation))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(std::move(lines), currentXxhash64), true, _generation)); _lastState.xxhash64 = currentXxhash64; _lastState.generation = _generation; } else { LOG(debug, "Sending timestamp update"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), false, _generation))); + _holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(std::move(lines), currentXxhash64), false, _generation)); _lastState.generation = _generation; } } diff --git a/config/src/vespa/config/set/configsetsource.h b/config/src/vespa/config/set/configsetsource.h index aa9e8425f96..95cc0049a7a 100644 --- a/config/src/vespa/config/set/configsetsource.h +++ b/config/src/vespa/config/set/configsetsource.h @@ -3,13 +3,13 @@ #include <vespa/config/common/source.h> #include <vespa/config/common/configkey.h> -#include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/configstate.h> #include <map> namespace config { class ConfigInstance; +class IConfigHolder; /** * Class for sending and receiving config request from a raw string. @@ -18,18 +18,20 @@ class ConfigSetSource : public Source { public: typedef std::map<ConfigKey, ConfigInstance *> BuilderMap; typedef std::shared_ptr<BuilderMap> BuilderMapSP; - ConfigSetSource(const IConfigHolder::SP & holder, const ConfigKey & key, const BuilderMapSP & builderMap); + ConfigSetSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key, BuilderMapSP builderMap); + ConfigSetSource(const ConfigSetSource &) = delete; + ConfigSetSource & operator =(const ConfigSetSource &) = delete; ~ConfigSetSource(); void getConfig() override; void reload(int64_t generation) override; void close() override; private: - IConfigHolder::SP _holder; + std::shared_ptr<IConfigHolder> _holder; const ConfigKey _key; - int64_t _generation; - BuilderMapSP _builderMap; - ConfigState _lastState; + int64_t _generation; + BuilderMapSP _builderMap; + ConfigState _lastState; bool validRequest(const ConfigKey & key); }; diff --git a/config/src/vespa/config/set/configsetsourcefactory.cpp b/config/src/vespa/config/set/configsetsourcefactory.cpp index b2ca99879cb..9d443cdaabc 100644 --- a/config/src/vespa/config/set/configsetsourcefactory.cpp +++ b/config/src/vespa/config/set/configsetsourcefactory.cpp @@ -3,15 +3,17 @@ namespace config { -ConfigSetSourceFactory::ConfigSetSourceFactory(const BuilderMapSP & builderMap) - : _builderMap(builderMap) +ConfigSetSourceFactory::ConfigSetSourceFactory(BuilderMapSP builderMap) + : _builderMap(std::move(builderMap)) { } -Source::UP -ConfigSetSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +ConfigSetSourceFactory::~ConfigSetSourceFactory() = default; + +std::unique_ptr<Source> +ConfigSetSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { - return Source::UP(new ConfigSetSource(holder, key, _builderMap)); + return std::make_unique<ConfigSetSource>(std::move(holder), key, _builderMap); } } // namespace config diff --git a/config/src/vespa/config/set/configsetsourcefactory.h b/config/src/vespa/config/set/configsetsourcefactory.h index 88e2eedb255..c42e14a7c84 100644 --- a/config/src/vespa/config/set/configsetsourcefactory.h +++ b/config/src/vespa/config/set/configsetsourcefactory.h @@ -16,14 +16,10 @@ class ConfigKey; class ConfigSetSourceFactory : public SourceFactory { public: - typedef ConfigSetSource::BuilderMap BuilderMap; - typedef ConfigSetSource::BuilderMapSP BuilderMapSP; - ConfigSetSourceFactory(const BuilderMapSP & builderMap); - - /** - * Create source handling config described by key. - */ - Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const override; + using BuilderMapSP = ConfigSetSource::BuilderMapSP; + explicit ConfigSetSourceFactory(BuilderMapSP builderMap); + ~ConfigSetSourceFactory() override; + std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: BuilderMapSP _builderMap; }; diff --git a/config/src/vespa/config/subscription/confighandle.h b/config/src/vespa/config/subscription/confighandle.h index 69532e5ba48..98a3e844acd 100644 --- a/config/src/vespa/config/subscription/confighandle.h +++ b/config/src/vespa/config/subscription/confighandle.h @@ -2,10 +2,11 @@ #pragma once #include <memory> -#include <vespa/config/subscription/configsubscription.h> namespace config { +class ConfigSubscription; + /** * A ConfigHandle is a subscription handle that is capable of looking up config * objects of a generic type. @@ -16,7 +17,8 @@ class ConfigHandle public: typedef std::unique_ptr<ConfigHandle <ConfigType> > UP; - ConfigHandle(const ConfigSubscription::SP & subscription); + explicit ConfigHandle(std::shared_ptr<ConfigSubscription> subscription); + ~ConfigHandle(); /** * Return the currently available config known to the ConfigHandle. Throws @@ -38,10 +40,8 @@ public: */ bool isChanged() const; private: - ConfigSubscription::SP _subscription; + std::shared_ptr<ConfigSubscription> _subscription; }; } // namespace config -#include "confighandle.hpp" - diff --git a/config/src/vespa/config/subscription/confighandle.hpp b/config/src/vespa/config/subscription/confighandle.hpp index 9ce94808601..9ad27a5cb3f 100644 --- a/config/src/vespa/config/subscription/confighandle.hpp +++ b/config/src/vespa/config/subscription/confighandle.hpp @@ -1,15 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "confighandle.h" +#include <vespa/config/common/configvalue.hpp> namespace config { template <typename ConfigType> -ConfigHandle<ConfigType>::ConfigHandle(const ConfigSubscription::SP & subscription) - : _subscription(subscription) +ConfigHandle<ConfigType>::ConfigHandle(std::shared_ptr<ConfigSubscription> subscription) + : _subscription(std::move(subscription)) { } template <typename ConfigType> +ConfigHandle<ConfigType>::~ConfigHandle() = default; + +template <typename ConfigType> std::unique_ptr<ConfigType> ConfigHandle<ConfigType>::getConfig() const { diff --git a/config/src/vespa/config/subscription/configinstancespec.h b/config/src/vespa/config/subscription/configinstancespec.h index 2294b37e1d1..32ee78b1d71 100644 --- a/config/src/vespa/config/subscription/configinstancespec.h +++ b/config/src/vespa/config/subscription/configinstancespec.h @@ -15,6 +15,9 @@ class ConfigInstanceSpec : public SourceSpec { public: ConfigInstanceSpec(const ConfigInstance & instance); + ConfigInstanceSpec(const ConfigInstanceSpec &) = delete; + ConfigInstanceSpec & operator =(const ConfigInstanceSpec &) = delete; + ~ConfigInstanceSpec() override; std::unique_ptr<SourceFactory> createSourceFactory(const TimingValues & timingValues) const override; private: const ConfigKey _key; diff --git a/config/src/vespa/config/subscription/configsubscriber.cpp b/config/src/vespa/config/subscription/configsubscriber.cpp index 2627b479df9..0645481a1f5 100644 --- a/config/src/vespa/config/subscription/configsubscriber.cpp +++ b/config/src/vespa/config/subscription/configsubscriber.cpp @@ -6,8 +6,8 @@ namespace config { -ConfigSubscriber::ConfigSubscriber(const IConfigContext::SP & context) - : _set(context) +ConfigSubscriber::ConfigSubscriber(std::shared_ptr<IConfigContext> context) + : _set(std::move(context)) { } @@ -15,6 +15,8 @@ ConfigSubscriber::ConfigSubscriber(const SourceSpec & spec) : _set(std::make_shared<ConfigContext>(spec)) { } +ConfigSubscriber::~ConfigSubscriber() = default; + bool ConfigSubscriber::nextConfig(milliseconds timeoutInMillis) { diff --git a/config/src/vespa/config/subscription/configsubscriber.h b/config/src/vespa/config/subscription/configsubscriber.h index 1cc82b6db8c..fd6497155fc 100644 --- a/config/src/vespa/config/subscription/configsubscriber.h +++ b/config/src/vespa/config/subscription/configsubscriber.h @@ -45,7 +45,10 @@ public: * * @param context A ConfigContext shared between all subscribers. */ - ConfigSubscriber(const IConfigContext::SP & context); + explicit ConfigSubscriber(std::shared_ptr<IConfigContext> context); + ConfigSubscriber(const ConfigSubscriber &) = delete; + ConfigSubscriber & operator= (const ConfigSubscriber &) = delete; + ~ConfigSubscriber(); /** * Checks if one or more of the configs in the set is updated or not. @@ -106,6 +109,3 @@ private: }; } // namespace config - -#include "configsubscriber.hpp" - diff --git a/config/src/vespa/config/subscription/configsubscriber.hpp b/config/src/vespa/config/subscription/configsubscriber.hpp index 28c54240748..ca890f8e35d 100644 --- a/config/src/vespa/config/subscription/configsubscriber.hpp +++ b/config/src/vespa/config/subscription/configsubscriber.hpp @@ -1,5 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configsubscriber.h" +#include "confighandle.hpp" + namespace config { template <typename ConfigType> @@ -7,7 +10,7 @@ std::unique_ptr<ConfigHandle<ConfigType> > ConfigSubscriber::subscribe(const std::string & configId, milliseconds timeoutInMillis) { const ConfigKey key(ConfigKey::create<ConfigType>(configId)); - return std::unique_ptr<ConfigHandle<ConfigType> >(new ConfigHandle<ConfigType>(_set.subscribe(key, timeoutInMillis))); + return std::make_unique<ConfigHandle<ConfigType> >(_set.subscribe(key, timeoutInMillis)); } } diff --git a/config/src/vespa/config/subscription/configsubscription.cpp b/config/src/vespa/config/subscription/configsubscription.cpp index 9790541906b..d49c5c32478 100644 --- a/config/src/vespa/config/subscription/configsubscription.cpp +++ b/config/src/vespa/config/subscription/configsubscription.cpp @@ -1,16 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configsubscription.h" +#include <vespa/config/common/configupdate.h> +#include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/common/misc.h> -#include "configsubscription.h" namespace config { -ConfigSubscription::ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, const IConfigHolder::SP & holder, Source::UP source) +ConfigSubscription::ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, + std::shared_ptr<IConfigHolder> holder, std::unique_ptr<Source> source) : _id(id), _key(key), _source(std::move(source)), - _holder(holder), + _holder(std::move(holder)), _next(), _current(), _isChanged(false), diff --git a/config/src/vespa/config/subscription/configsubscription.h b/config/src/vespa/config/subscription/configsubscription.h index 08dcac6a580..05c03736fa3 100644 --- a/config/src/vespa/config/subscription/configsubscription.h +++ b/config/src/vespa/config/subscription/configsubscription.h @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include "subscriptionid.h" -#include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/configkey.h> #include <vespa/config/common/source.h> - #include <atomic> #include <chrono> namespace config { +class IConfigHolder; +class ConfigUpdate; +class ConfigValue; + /** * A subscription can be polled for config updates, and handles interruption of * the nextUpdate call. @@ -20,7 +22,7 @@ public: typedef std::unique_ptr<ConfigSubscription> UP; typedef std::shared_ptr<ConfigSubscription> SP; - ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, const IConfigHolder::SP & holder, Source::UP source); + ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, std::shared_ptr<IConfigHolder> holder, std::unique_ptr<Source> source); ~ConfigSubscription(); /** @@ -57,15 +59,15 @@ public: void reload(int64_t generation); private: - const SubscriptionId _id; - const ConfigKey _key; - Source::UP _source; - IConfigHolder::SP _holder; - ConfigUpdate::UP _next; - ConfigUpdate::UP _current; - bool _isChanged; - int64_t _lastGenerationChanged; - std::atomic<bool> _closed; + const SubscriptionId _id; + const ConfigKey _key; + std::unique_ptr<Source> _source; + std::shared_ptr<IConfigHolder> _holder; + std::unique_ptr<ConfigUpdate> _next; + std::unique_ptr<ConfigUpdate> _current; + bool _isChanged; + int64_t _lastGenerationChanged; + std::atomic<bool> _closed; }; typedef std::vector<ConfigSubscription::SP> SubscriptionList; diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 63c9f287350..9ad4dfe3f9d 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -1,8 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configsubscriptionset.h" +#include "configsubscription.h" #include <vespa/config/common/exceptions.h> #include <vespa/config/common/misc.h> +#include <vespa/config/common/iconfigmanager.h> +#include <vespa/config/common/iconfigcontext.h> #include <thread> #include <vespa/log/log.h> @@ -13,9 +16,9 @@ using namespace std::chrono; namespace config { -ConfigSubscriptionSet::ConfigSubscriptionSet(const IConfigContext::SP & context) - : _context(context), - _mgr(context->getManagerInstance()), +ConfigSubscriptionSet::ConfigSubscriptionSet(std::shared_ptr<IConfigContext> context) + : _context(std::move(context)), + _mgr(_context->getManagerInstance()), _currentGeneration(-1), _subscriptionList(), _state(OPEN) @@ -119,7 +122,7 @@ ConfigSubscriptionSet::isClosed() const return (_state.load(std::memory_order_relaxed) == CLOSED); } -ConfigSubscription::SP +std::shared_ptr<ConfigSubscription> ConfigSubscriptionSet::subscribe(const ConfigKey & key, milliseconds timeoutInMillis) { if (_state != OPEN) { @@ -127,7 +130,7 @@ ConfigSubscriptionSet::subscribe(const ConfigKey & key, milliseconds timeoutInMi } LOG(debug, "Subscribing with config Id(%s), defName(%s)", key.getConfigId().c_str(), key.getDefName().c_str()); - ConfigSubscription::SP s = _mgr.subscribe(key, timeoutInMillis); + std::shared_ptr<ConfigSubscription> s = _mgr.subscribe(key, timeoutInMillis); _subscriptionList.push_back(s); return s; } diff --git a/config/src/vespa/config/subscription/configsubscriptionset.h b/config/src/vespa/config/subscription/configsubscriptionset.h index ea37a133ef8..89c74a97be2 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.h +++ b/config/src/vespa/config/subscription/configsubscriptionset.h @@ -2,16 +2,19 @@ // #pragma once -#include "confighandle.h" #include "subscriptionid.h" -#include "configsubscription.h" -#include "configprovider.h" -#include <vespa/config/common/iconfigcontext.h> -#include <vespa/config/common/iconfigmanager.h> #include <atomic> +#include <chrono> +#include <memory> +#include <vector> namespace config { +class IConfigContext; +class IConfigManager; +class ConfigSubscription; +class ConfigKey; + /** * A ConfigSubscriptionSet is a set of configs that can be subscribed to. */ @@ -25,8 +28,10 @@ public: * * @param context A ConfigContext shared between all subscriptions. */ - ConfigSubscriptionSet(const IConfigContext::SP & context); + explicit ConfigSubscriptionSet(std::shared_ptr<IConfigContext> context); + ConfigSubscriptionSet(const ConfigSubscriptionSet &) = delete; + ConfigSubscriptionSet & operator= (const ConfigSubscriptionSet &) = delete; ~ConfigSubscriptionSet(); /** @@ -48,7 +53,7 @@ public: bool isClosed() const; // Helpers for doing the subscription - ConfigSubscription::SP subscribe(const ConfigKey & key, milliseconds timeoutInMillis); + std::shared_ptr<ConfigSubscription> subscribe(const ConfigKey & key, milliseconds timeoutInMillis); // Tries to acquire a new snapshot of config within the timeout bool acquireSnapshot(milliseconds timeoutInMillis, bool requireDifference); @@ -56,13 +61,13 @@ public: private: // Describes the state of the subscriber. enum SubscriberState { OPEN, FROZEN, CONFIGURED, CLOSED }; + using SubscriptionList = std::vector<std::shared_ptr<ConfigSubscription>>; - IConfigContext::SP _context; // Context to keep alive managers. - IConfigManager & _mgr; // The config manager that we use. - int64_t _currentGeneration; // Holds the current config generation. - SubscriptionList _subscriptionList; // List of current subscriptions. - - std::atomic<SubscriberState> _state; // Current state of this subscriber. + std::shared_ptr<IConfigContext> _context; // Context to keep alive managers. + IConfigManager & _mgr; // The config manager that we use. + int64_t _currentGeneration; // Holds the current config generation. + SubscriptionList _subscriptionList; // List of current subscriptions. + std::atomic<SubscriberState> _state; // Current state of this subscriber. }; } // namespace config diff --git a/config/src/vespa/config/subscription/configuri.cpp b/config/src/vespa/config/subscription/configuri.cpp index 001e7adbb8b..edb06f8b237 100644 --- a/config/src/vespa/config/subscription/configuri.cpp +++ b/config/src/vespa/config/subscription/configuri.cpp @@ -18,7 +18,7 @@ ConfigUri::ConfigUri(const vespalib::string &configId) { } -ConfigUri::ConfigUri(const vespalib::string &configId, IConfigContext::SP context) +ConfigUri::ConfigUri(const vespalib::string &configId, std::shared_ptr<IConfigContext> context) : _configId(configId), _context(std::move(context)), _empty(false) @@ -34,7 +34,7 @@ ConfigUri::createWithNewId(const vespalib::string & configId) const } const vespalib::string & ConfigUri::getConfigId() const { return _configId; } -const IConfigContext::SP & ConfigUri::getContext() const { return _context; } +const std::shared_ptr<IConfigContext> & ConfigUri::getContext() const { return _context; } ConfigUri ConfigUri::createFromInstance(const ConfigInstance & instance) diff --git a/config/src/vespa/config/subscription/configuri.h b/config/src/vespa/config/subscription/configuri.h index 7fe32312d30..d8472d9a637 100644 --- a/config/src/vespa/config/subscription/configuri.h +++ b/config/src/vespa/config/subscription/configuri.h @@ -46,7 +46,7 @@ public: * @param configId The config id. * @param context A context object that can be shared with multiple URIs. */ - ConfigUri(const vespalib::string &configId, IConfigContext::SP context); + ConfigUri(const vespalib::string &configId, std::shared_ptr<IConfigContext> context); ~ConfigUri(); @@ -90,7 +90,7 @@ public: * Get the context for this uri. Used by subscriber. * @return The context. */ - const IConfigContext::SP & getContext() const; + const std::shared_ptr<IConfigContext> & getContext() const; /** * Empty if the original id was empty or created with createEmpty @@ -99,9 +99,9 @@ public: bool empty() const { return _empty; } private: - vespalib::string _configId; - IConfigContext::SP _context; - bool _empty; + vespalib::string _configId; + std::shared_ptr<IConfigContext> _context; + bool _empty; }; } // namespace config diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index c8fd577935d..8009cdfd217 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -153,7 +153,9 @@ ConfigInstanceSpec::ConfigInstanceSpec(const ConfigInstance& instance) writer.write(instance); } -SourceFactory::UP +ConfigInstanceSpec::~ConfigInstanceSpec() = default; + +std::unique_ptr<SourceFactory> ConfigInstanceSpec::createSourceFactory(const TimingValues& ) const { return std::make_unique<ConfigInstanceSourceFactory>(_key, _buffer); diff --git a/config/src/vespa/config/subscription/sourcespec.h b/config/src/vespa/config/subscription/sourcespec.h index 040c16204e3..a57b22ca322 100644 --- a/config/src/vespa/config/subscription/sourcespec.h +++ b/config/src/vespa/config/subscription/sourcespec.h @@ -37,7 +37,7 @@ public: * @return An std::unique_ptr<Source> that can be used to ask for config. */ virtual SourceFactorySP createSourceFactory(const TimingValues & timingValues) const = 0; - virtual ~SourceSpec() { } + virtual ~SourceSpec() = default; }; @@ -131,7 +131,7 @@ class ServerSpec : public SourceSpec { public: /// A list of host specifications - typedef std::vector<vespalib::string> HostSpecList; + using HostSpecList = std::vector<vespalib::string>; /** * Construct a ServerSpec that fetches the host specs from the diff --git a/configd/src/apps/sentinel/config-owner.cpp b/configd/src/apps/sentinel/config-owner.cpp index 40fe5267dae..90ceb705dc7 100644 --- a/configd/src/apps/sentinel/config-owner.cpp +++ b/configd/src/apps/sentinel/config-owner.cpp @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "config-owner.h" -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/config/common/exceptions.h> -#include <string> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/log/log.h> LOG_SETUP(".sentinel.config-owner"); diff --git a/configd/src/apps/sentinel/config-owner.h b/configd/src/apps/sentinel/config-owner.h index a9083b553fa..2e44f8cd338 100644 --- a/configd/src/apps/sentinel/config-owner.h +++ b/configd/src/apps/sentinel/config-owner.h @@ -4,7 +4,7 @@ #include <vespa/config-sentinel.h> #include <vespa/config-model.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configsubscriber.h> using cloud::config::SentinelConfig; diff --git a/configd/src/apps/sentinel/manager.h b/configd/src/apps/sentinel/manager.h index 8aaa406d578..48c95fe7a3d 100644 --- a/configd/src/apps/sentinel/manager.h +++ b/configd/src/apps/sentinel/manager.h @@ -8,7 +8,6 @@ #include "service.h" #include "state-api.h" #include <vespa/config-sentinel.h> -#include <vespa/config/config.h> #include <vespa/vespalib/net/state_server.h> #include <sys/types.h> #include <sys/select.h> diff --git a/configd/src/apps/sentinel/model-owner.cpp b/configd/src/apps/sentinel/model-owner.cpp index 8b1db60c73d..5dd7a033933 100644 --- a/configd/src/apps/sentinel/model-owner.cpp +++ b/configd/src/apps/sentinel/model-owner.cpp @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "model-owner.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/config/common/exceptions.h> -#include <string> -#include <chrono> -#include <vespa/log/log.h> +#include <vespa/config/subscription/configsubscriber.hpp> +#include <vespa/log/log.h> LOG_SETUP(".sentinel.model-owner"); using namespace std::chrono_literals; diff --git a/configd/src/apps/sentinel/model-owner.h b/configd/src/apps/sentinel/model-owner.h index 6c3038b4b55..762846e1ccd 100644 --- a/configd/src/apps/sentinel/model-owner.h +++ b/configd/src/apps/sentinel/model-owner.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/config-model.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configsubscriber.h> #include <optional> #include <mutex> diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 18b40e8ec34..3565c3c1a39 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/defaults.h> #include <chrono> +#include <clocale> #include <string> #include <unistd.h> #include <sys/time.h> diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def index 6f08ad75f6e..bff33eaf034 100644 --- a/configdefinitions/src/vespa/fleetcontroller.def +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -25,7 +25,7 @@ zookeeper_session_timeout double default=30.0 master_zookeeper_cooldown_period double default=60.0 ## Sets how many fleetcontrollers will gather state. A fleetcontroller -## gathering state can take over quicker should ## the master fail. +## gathering state can take over quicker should the master fail. ## If set to 1, only master will gather state. If set higher, others will ## also do so, prioritizing those fleetcontrollers likely to be the ones to ## take over if the master fails. @@ -40,14 +40,14 @@ rpc_port int default=6500 restart ## Port where fleetcontroller listens for HTTP status request http_port int default=0 restart -## Maximum number of milliseconds a storage will be automatically reported in -## maintenance due to node recently being availble. (If 0, no time at all) +## Maximum number of milliseconds a storage node will be automatically reported +## in maintenance due to node recently being available. (If 0, no time at all) storage_transition_time int default=30000 ## Maximum number of milliseconds a distributor will be automatically reported -## in maintenance due to node recently being availble. (If 0, no time at all) +## in maintenance due to node recently being available. (If 0, no time at all) ## -## Currently default is 0.. Should probably be more then we know it is working +## Currently default is 0.. Should probably be more when we know it is working ## correctly distributor_transition_time int default=0 @@ -190,7 +190,7 @@ determine_buckets_from_bucket_space_metric bool default=true # If one resource category from one content node is above the configured limit, feed is blocked. # This information is pushed to all distributor nodes via a new cluster state bundle. # The distributor nodes handle the actual feed blocking based on this information. -enable_cluster_feed_block bool default=false +enable_cluster_feed_block bool default=true # Contains all resource categories (key) with its corresponding feed block limit (value) # used when the cluster controller decides whether external feed should be blocked (or unblocked) in the entire cluster. diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 5a277eaed9d..5e9460ddfc4 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,24 +7,10 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=true -tenants{}.applications{}.generateNonMtlsEndpoint bool default=true - -tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" -tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" -tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" -tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" -tenants{}.applications{}.hosts{}.services{}.configId string -tenants{}.applications{}.hosts{}.services{}.index int default=0 -tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 -tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" -tenants{}.applications{}.hosts{}.services{}.servicealiases[] string -tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string - tenants{}.applications{}.endpoints[].dnsName string tenants{}.applications{}.endpoints[].clusterId string tenants{}.applications{}.endpoints[].scope enum {application, global, zone} tenants{}.applications{}.endpoints[].routingMethod enum {shared, sharedLayer4} tenants{}.applications{}.endpoints[].weight int default=1 -tenants{}.applications{}.endpoints[].hosts[] string
\ No newline at end of file +tenants{}.applications{}.endpoints[].hosts[] string diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 1aaa7f71f8e..f1fc667ca9d 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -39,13 +39,8 @@ common_merge_chain_optimalization_minimum_size int default=64 restart ## Chunksize to use while merging buckets between nodes. ## -## Default is set to 4 MB. -## Note that this will gradually be increased to reach stor-distributormanager:splitsize which is currently at 32M -bucket_merge_chunk_size int default=33554432 restart - -## If set, portions of apply bucket diff handling will be performed asynchronously -## with persistence thread not waiting for local writes to complete. -async_apply_bucket_diff bool default=true +## Should follow stor-distributormanager:splitsize (16MB). +bucket_merge_chunk_size int default=16772216 restart ## When merging, it is possible to send more metadata than needed in order to ## let local nodes in merge decide which entries fits best to add this time @@ -85,6 +80,22 @@ resource_usage_reporter_noise_level double default=0.001 ## - DYNAMIC uses DynamicThrottlePolicy under the hood and will block if the window ## is full (if a blocking throttler API call is invoked). ## +async_operation_throttler.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED restart +## Internal throttler tuning parameters that only apply when type == DYNAMIC: +async_operation_throttler.window_size_increment int default=20 +async_operation_throttler.window_size_decrement_factor double default=1.2 +async_operation_throttler.window_size_backoff double default=0.95 + +## Specify throttling used for async persistence operations. This throttling takes place +## before operations are dispatched to Proton and serves as a limiter for how many +## operations may be in flight in Proton's internal queues. +## +## - UNLIMITED is, as it says on the tin, unlimited. Offers no actual throttling, but +## has near zero overhead and never blocks. +## - DYNAMIC uses DynamicThrottlePolicy under the hood and will block if the window +## is full (if a blocking throttler API call is invoked). +## +## TODO deprecate in favor of the async_operation_throttler struct instead. async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED restart ## Specifies the extent the throttling window is increased by when the async throttle @@ -93,4 +104,5 @@ async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED res ## value, number of threads). ## ## Only applies if async_operation_throttler_type == DYNAMIC. +## DEPRECATED! use the async_operation_throttler struct instead async_operation_dynamic_throttling_window_increment int default=20 restart diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index 04632ffd35f..a2c4ec3b2db 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -32,10 +32,13 @@ juteMaxBuffer int default=52428800 myid int restart server[].id int server[].hostname string +server[].clientPort int default=2181 server[].quorumPort int default=2182 server[].electionPort int default=2183 # Whether this server is joining an existing cluster server[].joining bool default=false +# Whether this server is retired, and about to be removed +server[].retired bool default=false # Needed when upgrading from ZooKeeper 3.4 to 3.5, see https://issues.apache.org/jira/browse/ZOOKEEPER-3056, # and in general where there is a zookeeper ensemble running that has had few transactions. @@ -51,4 +54,4 @@ tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIF jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" dynamicReconfiguration bool default=false -snapshotMethod string default="" +snapshotMethod string default="gz" diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index 149bcced29c..d3a0a79c737 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -3,14 +3,10 @@ package com.yahoo.config.codegen; import java.io.FileWriter; import java.io.File; -import java.io.BufferedReader; import java.io.IOException; -import java.io.FileReader; import java.io.StringWriter; import java.io.Writer; import java.util.Map; -import java.util.HashMap; -import java.util.Collections; import java.util.Arrays; import java.util.StringTokenizer; import java.util.stream.Collectors; @@ -24,39 +20,31 @@ public class CppClassBuilder implements ClassBuilder { private final NormalizedDefinition nd; private final File rootDir; private final String relativePathUnderRoot; - private static final Map<String, String> vectorTypeDefs; - static { - Map<String, String> map = new HashMap<String, String>(); - map.put("bool", "BoolVector"); - map.put("int32_t", "IntVector"); - map.put("int64_t", "LongVector"); - map.put("double", "DoubleVector"); - map.put("vespalib::string", "StringVector"); - vectorTypeDefs = Collections.unmodifiableMap(map); - } - private static final Map<String, String> mapTypeDefs; - static { - Map<String, String> map = new HashMap<>(); - map.put("bool", "BoolMap"); - map.put("int32_t", "IntMap"); - map.put("int64_t", "LongMap"); - map.put("double", "DoubleMap"); - map.put("vespalib::string", "StringMap"); - mapTypeDefs = Collections.unmodifiableMap(map); - } - private static final Map<String, String> slimeTypeMap; - static { - Map<String, String> map = new HashMap<String, String>(); - map.put("bool", "Bool"); - map.put("int", "Long"); - map.put("long", "Long"); - map.put("double", "Double"); - map.put("string", "String"); - map.put("enum", "String"); - map.put("file", "String"); - map.put("reference", "String"); - slimeTypeMap = Collections.unmodifiableMap(map); - } + private static final Map<String, String> vectorTypeDefs = Map.of( + "bool", "::config::BoolVector", + "int32_t", "::config::IntVector", + "int64_t", "::config::LongVector", + "double", "::config::DoubleVector", + "vespalib::string", "::config::StringVector" + ); + private static final Map<String, String> mapTypeDefs = Map.of( + "bool", "::config::BoolMap", + "int32_t", "::config::IntMap", + "int64_t", "::config::LongMap", + "double", "::config::DoubleMap", + "vespalib::string", "::config::StringMap" + ); + + private static final Map<String, String> slimeTypeMap = Map.of( + "bool", "Bool", + "int", "Long", + "long", "Long", + "double", "Double", + "string", "String", + "enum", "String", + "file", "String", + "reference", "String" + ); public CppClassBuilder(CNode root, NormalizedDefinition nd, File rootDir, String relativePathUnderRoot) { this.root = root; @@ -69,19 +57,6 @@ public class CppClassBuilder implements ClassBuilder { generateConfig(root, nd); } - String readFile(File f) throws IOException { - if (!f.isFile()) return null; - StringBuilder sb = new StringBuilder(); - try (BufferedReader sr = new BufferedReader(new FileReader(f))) { - while (true) { - String line = sr.readLine(); - if (line == null) break; - sb.append(line).append("\n"); - } - return sb.toString(); - } - } - void writeFile(File f, String content) throws IOException { FileWriter fw = new FileWriter(f); fw.write(content); @@ -118,7 +93,7 @@ public class CppClassBuilder implements ClassBuilder { static String removeDashesAndUpperCaseAllFirstChars(String source, boolean capitalizeFirst) { // Create upper case chars after each dash - String parts[] = source.split("[-_]"); + String [] parts = source.split("[-_]"); StringBuilder sb = new StringBuilder(); for (String s : parts) { sb.append(s.substring(0, 1).toUpperCase()).append(s.substring(1)); @@ -149,7 +124,7 @@ public class CppClassBuilder implements ClassBuilder { * Class to generate noexcept specifier if default constructor, copy constructor or copy assignment is non-throwing */ private static class NoExceptSpecifier { - private boolean enabled; + private final boolean enabled; public NoExceptSpecifier(CNode node) { enabled = checkNode(node); @@ -267,9 +242,7 @@ public class CppClassBuilder implements ClassBuilder { + "#define CLOUD_CONFIG_" + defineName + "_H\n" + "\n" + "#include <vespa/config/configgen/configinstance.h>\n" - + "#include <vespa/vespalib/stllike/string.h>\n" - + "#include <vector>\n" - + "#include <map>\n" + + "#include <vespa/config/common/types.h>\n" + "\n"); w.write("namespace config {\n"); w.write(" class ConfigValue;\n"); @@ -295,7 +268,7 @@ public class CppClassBuilder implements ClassBuilder { void writeTypeDeclarations(Writer w, CNode node, String indent) throws IOException { - java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + java.util.Set<String> declaredTypes = new java.util.HashSet<>(); for (CNode child : node.getChildren()) { boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); if (complexType && !declaredTypes.contains(child.getName())) { @@ -357,7 +330,7 @@ public class CppClassBuilder implements ClassBuilder { } void writeStructFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { - w.write(indent + className + "(const std::vector<vespalib::string> & __lines);\n"); + w.write(indent + className + "(const " + vectorTypeDefs.get("vespalib::string") + " & __lines);\n"); w.write(indent + className + "(const vespalib::slime::Inspector & __inspector);\n"); w.write(indent + className + "(const ::config::ConfigPayload & __payload);\n"); writeCommonFunctionDeclarations(w, className, node, indent); @@ -370,6 +343,12 @@ public class CppClassBuilder implements ClassBuilder { void writeClassAssignmentOperatorDeclaration(Writer w, String className, NoExceptSpecifier noexcept, String indent) throws IOException { w.write(indent + className + " & operator = (const " + className + " & __rhs)" + noexcept + ";\n"); } + void writeClassMoveConstructorDeclaration(Writer w, String className, NoExceptSpecifier noexcept, String indent) throws IOException { + w.write(indent + className + "(" + className + " && __rhs)" + noexcept + ";\n"); + } + void writeClassMoveOperatorDeclaration(Writer w, String className, NoExceptSpecifier noexcept, String indent) throws IOException { + w.write(indent + className + " & operator = (" + className + " && __rhs)" + noexcept + ";\n"); + } void writeConfigClassCopyConstructorDefinition(Writer w, String parent, String className, NoExceptSpecifier noexcept) throws IOException { w.write(parent + "::" + className + "(const " + className + " & __rhs)" + noexcept + " = default;\n"); @@ -377,12 +356,23 @@ public class CppClassBuilder implements ClassBuilder { void writeConfigClassAssignmentOperatorDefinition(Writer w, String parent, String className, NoExceptSpecifier noexcept) throws IOException { w.write(parent + " & " + parent + "::" + "operator =(const " + className + " & __rhs)" + noexcept + " = default;\n"); } + void writeConfigClassMoveConstructorDefinition(Writer w, String parent, String className, NoExceptSpecifier noexcept) throws IOException { + w.write(parent + "::" + className + "(" + className + " && __rhs)" + noexcept + " = default;\n"); + } + void writeConfigClassMoveOperatorDefinition(Writer w, String parent, String className, NoExceptSpecifier noexcept) throws IOException { + w.write(parent + " & " + parent + "::" + "operator =(" + className + " && __rhs)" + noexcept + " = default;\n"); + } void writeClassCopyConstructorDefinition(Writer w, String parent, CNode node) throws IOException { String typeName = getTypeName(node, false); NoExceptSpecifier noexcept = new NoExceptSpecifier(node); w.write(parent + "::" + typeName + "(const " + typeName + " & __rhs)" + noexcept + " = default;\n"); } + void writeClassMoveConstructorDefinition(Writer w, String parent, CNode node) throws IOException { + String typeName = getTypeName(node, false); + NoExceptSpecifier noexcept = new NoExceptSpecifier(node); + w.write(parent + "::" + typeName + "(" + typeName + " && __rhs)" + noexcept + " = default;\n"); + } void writeClassAssignmentOperatorDefinition(Writer w, String parent, CNode node) throws IOException { String typeName = getTypeName(node, false); @@ -391,8 +381,15 @@ public class CppClassBuilder implements ClassBuilder { w.write(parent + " & " + parent + "::" + "operator = (const " + typeName + " & __rhs)" + noexcept + " = default;\n"); } + void writeClassMoveOperatorDefinition(Writer w, String parent, CNode node) throws IOException { + String typeName = getTypeName(node, false); + NoExceptSpecifier noexcept = new NoExceptSpecifier(node); + // Write empty constructor + w.write(parent + " & " + parent + "::" + "operator = (" + typeName + " && __rhs)" + noexcept + " = default;\n"); + } + void writeDestructor(Writer w, String parent, String className) throws IOException { - w.write(parent + "~" + className + "() { } \n"); + w.write(parent + "~" + className + "() = default; \n"); } void writeCommonFunctionDeclarations(Writer w, String className, CNode node, String indent) throws IOException { @@ -400,6 +397,8 @@ public class CppClassBuilder implements ClassBuilder { w.write("" + indent + className + "() " + noexcept + ";\n"); writeClassCopyConstructorDeclaration(w, className, noexcept, indent); writeClassAssignmentOperatorDeclaration(w, className, noexcept, indent); + writeClassMoveConstructorDeclaration(w, className, noexcept, indent); + writeClassMoveOperatorDeclaration(w, className, noexcept, indent); w.write("" + indent + "~" + className + "();\n"); w.write("\n" + indent + "bool operator==(const " + className + "& __rhs) const noexcept;\n" @@ -460,7 +459,7 @@ public class CppClassBuilder implements ClassBuilder { + indent + "static const vespalib::string CONFIG_DEF_VERSION;\n" + indent + "static const vespalib::string CONFIG_DEF_NAME;\n" + indent + "static const vespalib::string CONFIG_DEF_NAMESPACE;\n" - + indent + "static const std::vector<vespalib::string> CONFIG_DEF_SCHEMA;\n" + + indent + "static const ::config::StringVector CONFIG_DEF_SCHEMA;\n" + indent + "static const int64_t CONFIG_DEF_SERIALIZE_VERSION;\n" + "\n" ); @@ -469,7 +468,7 @@ public class CppClassBuilder implements ClassBuilder { void writeComment(Writer w, String indent, String comment, boolean javadoc) throws IOException { - /** If simple one liner comment, write on one line. */ + // If simple one liner comment, write on one line. if (javadoc && comment.indexOf('\n') == -1 && comment.length() <= 80 - (indent.length() + 7)) { @@ -481,7 +480,7 @@ public class CppClassBuilder implements ClassBuilder { w.write(indent + "// " + comment + "\n"); return; } - /** If not we need to write multi line comment. */ + // If not we need to write multi line comment. int maxLineLen = 80 - (indent.length() + 3); if (javadoc) w.write(indent + "/**\n"); do { @@ -543,18 +542,6 @@ public class CppClassBuilder implements ClassBuilder { void writeHeaderTypeDefs(Writer w, CNode root, String indent) throws IOException { w.write(indent + "typedef std::unique_ptr<const " + getInternalClassName(root) + "> UP;\n"); - for (Map.Entry<String, String> entry : vectorTypeDefs.entrySet()) { - String typeName = entry.getKey(); - String vectorName = entry.getValue(); - String typeDef = "typedef std::vector<" + typeName + "> " + vectorName; - w.write(indent + typeDef + ";\n"); - } - for (Map.Entry<String, String> entry : mapTypeDefs.entrySet()) { - String typeName = entry.getKey(); - String mapName = entry.getValue(); - String typeDef = "typedef std::map<vespalib::string, " + typeName + "> " + mapName; - w.write(indent + typeDef + ";\n"); - } } private static String getInternalClassName(CNode root) { @@ -599,11 +586,11 @@ public class CppClassBuilder implements ClassBuilder { w.write("#include <vespa/config/configgen/configpayload.h>\n"); w.write("#include <vespa/config/print/configdatabuffer.h>\n"); w.write("#include <vespa/config/common/configparser.h>\n"); - w.write("#include <vespa/config/configgen/vector_inserter.h>\n"); - w.write("#include <vespa/config/configgen/map_inserter.h>\n"); w.write("#include <vespa/vespalib/data/slime/convenience.h>\n"); w.write("#include <vespa/vespalib/data/slime/slime.h>\n"); w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); + w.write("#include <vespa/config/configgen/vector_inserter.hpp>\n"); + w.write("#include <vespa/config/configgen/map_inserter.hpp>\n"); w.write("\n"); writeNameSpaceBegin(w, generateCppNameSpace(root)); w.write("\nnamespace internal {\n\n"); @@ -627,7 +614,7 @@ public class CppClassBuilder implements ClassBuilder { w.write("\"" + line.replace("\"", "\\\"") + "\",\n"); } w.write("};\n"); - w.write("const std::vector<vespalib::string> " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n"); + w.write("const ::config::StringVector " + typeName + "::CONFIG_DEF_SCHEMA(__internalDefSchema,\n"); w.write(" __internalDefSchema + (sizeof(__internalDefSchema) / \n"); w.write(" sizeof(__internalDefSchema[0])));\n"); w.write("\n"); @@ -640,7 +627,7 @@ public class CppClassBuilder implements ClassBuilder { root = true; } final String parent = fullClassName + "::"; - java.util.Set<String> declaredTypes = new java.util.HashSet<String>(); + java.util.Set<String> declaredTypes = new java.util.HashSet<>(); for (CNode child : node.getChildren()) { boolean complexType = (child instanceof InnerCNode || child instanceof LeafCNode.EnumLeaf); if (complexType && !declaredTypes.contains(child.getName())) { @@ -755,9 +742,13 @@ public class CppClassBuilder implements ClassBuilder { if (root) { writeConfigClassCopyConstructorDefinition(w, fullClassName, typeName, noexcept); writeConfigClassAssignmentOperatorDefinition(w, fullClassName, typeName, noexcept); + writeConfigClassMoveConstructorDefinition(w, fullClassName, typeName, noexcept); + writeConfigClassMoveOperatorDefinition(w, fullClassName, typeName, noexcept); } else { writeClassCopyConstructorDefinition(w, fullClassName, node); writeClassAssignmentOperatorDefinition(w, fullClassName, node); + writeClassMoveConstructorDefinition(w, fullClassName, node); + writeClassMoveOperatorDefinition(w, fullClassName, node); } writeDestructor(w, parent, typeName); @@ -768,30 +759,20 @@ public class CppClassBuilder implements ClassBuilder { + "{\n" + indent + "try {\n"); indent = " "; - w.write(indent + "const std::vector<vespalib::string> & __lines(__value.getLines());\n"); + w.write(indent + "const " + vectorTypeDefs.get("vespalib::string") + " & __lines(__value.getLines());\n"); } else { - w.write(parent + typeName + "(const std::vector<vespalib::string> & __lines)\n" + w.write(parent + typeName + "(const " + vectorTypeDefs.get("vespalib::string") +" & __lines)\n" + "{\n"); } - w.write("" - + indent + "std::set<vespalib::string> __remainingValuesToParse(" - + "__lines.begin(), __lines.end());\n"); - w.write(indent + "for(std::set<vespalib::string>::iterator __rVTPiter = __remainingValuesToParse.begin();\n" - + indent + " __rVTPiter != __remainingValuesToParse.end();)\n" - + indent + "{\n" - + indent + " if (ConfigParser::stripWhitespace(*__rVTPiter).empty()) {\n" - + indent + " std::set<vespalib::string>::iterator __rVTPiter2 = __rVTPiter++;\n" - + indent + " __remainingValuesToParse.erase(__rVTPiter2);\n" - + indent + " } else {\n" - + indent + " ++__rVTPiter;\n" - + indent + " }\n" - + indent + "}\n"); + w.write(indent + "std::set<vespalib::string> __remainingValuesToParse = ConfigParser::getUniqueNonWhiteSpaceLines(__lines);\n"); for (CNode child : node.getChildren()) { String childType = getTypeName(child, false); String childName = getIdentifier(child.getName()); + String childVectorType = null; if (child instanceof LeafCNode.EnumLeaf) { if (child.isArray) { - w.write(indent + "std::vector<vespalib::string> " + childName + "__ValueList(\n "); + childVectorType = "::config::StringVector"; + w.write(indent + childVectorType + " " + childName + "__ValueList(\n "); } else if (child.isMap) { w.write(indent + "std::map<vespalib::string, vespalib::string> " + childName + "__ValueMap(\n "); } else { @@ -802,18 +783,17 @@ public class CppClassBuilder implements ClassBuilder { w.write(indent + childName + " = "); } if (child.isArray) { - w.write("ConfigParser::parseArray<" + childType + ">(\"" - + child.getName() + "\", __lines)"); + if (childVectorType == null) { + childVectorType = getTypeName(child, true); + } + w.write("ConfigParser::parseArray<" + childVectorType + ">(\"" + child.getName() + "\", __lines)"); } else if (child.isMap) { - w.write("ConfigParser::parseMap<" + childType + ">(\"" - + child.getName() + "\", __lines)"); + w.write("ConfigParser::parseMap<" + childType + ">(\"" + child.getName() + "\", __lines)"); } else { if (child instanceof LeafCNode) { - w.write("ConfigParser::parse<" + childType + ">(\"" - + child.getName() + "\", __lines"); + w.write("ConfigParser::parse<" + childType + ">(\"" + child.getName() + "\", __lines"); } else { - w.write("ConfigParser::parseStruct<" + childType + ">(\"" - + child.getName() + "\", __lines"); + w.write("ConfigParser::parseStruct<" + childType + ">(\"" + child.getName() + "\", __lines"); } if (child instanceof LeafCNode && ((LeafCNode) child).getDefaultValue() != null) { LeafCNode leaf = (LeafCNode) child; @@ -832,26 +812,20 @@ public class CppClassBuilder implements ClassBuilder { w.write(");\n"); if (child.isArray) { w.write(indent + childName + ".reserve(" + childName + "__ValueList.size());\n" - + indent + "for (std::vector<vespalib::string>::const_iterator __it\n" - + indent + " = " + childName + "__ValueList.begin();\n" - + indent + " __it != " + childName + "__ValueList.end(); ++__it)\n" - + indent + "{\n" - + indent + " " + childName + ".push_back(get" + childType + "(*__it));\n" + + indent + "for (const auto & item : " + childName + "__ValueList) {\n" + + indent + " " + childName + ".push_back(get" + childType + "(item));\n" + indent + "}\n" ); } else if (child.isMap) { - w.write(indent + "typedef std::map<vespalib::string, vespalib::string> __ValueMap;\n"); - w.write(indent + "for (__ValueMap::iterator __it(" + childName + "__ValueMap.begin()), __mt(" + childName + "__ValueMap.end()); __it != __mt; __it++) {\n" - + " " + childName + "[__it->first] = get" + childType + "(__it->second);\n" + w.write(indent + "for (const auto & entry : " + childName + "__ValueMap) {\n" + + " " + childName + "[entry.first] = get" + childType + "(entry.second);\n" + "}\n" ); } } else { w.write(";\n"); } - w.write(indent + "ConfigParser::stripLinesForKey(\"" - + child.getName() + "\", " - + "__remainingValuesToParse);\n"); + w.write(indent + "ConfigParser::stripLinesForKey(\"" + child.getName() + "\", " + "__remainingValuesToParse);\n"); } if (root) { indent = " "; @@ -864,9 +838,8 @@ public class CppClassBuilder implements ClassBuilder { + "\n" ); // Write operator== - String lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); w.write("bool\n" - + parent + lineBreak + "operator==(const " + typeName + "& __rhs) const noexcept\n" + + parent + lineBreak(parent, typeName) + "operator==(const " + typeName + "& __rhs) const noexcept\n" + "{\n" + " return (" ); @@ -883,9 +856,8 @@ public class CppClassBuilder implements ClassBuilder { + "\n" ); // Write operator!= - lineBreak = (parent.length() + typeName.length() < 50 ? "" : "\n"); w.write("bool\n" - + parent + lineBreak + "operator!=(const " + typeName + "& __rhs) const noexcept\n" + + parent + lineBreak(parent, typeName) + "operator!=(const " + typeName + "& __rhs) const noexcept\n" + "{\n" + " return !(operator==(__rhs));\n" + "}\n" @@ -896,6 +868,10 @@ public class CppClassBuilder implements ClassBuilder { writeSlimeConstructor(w, node, parent, root); } + private static String lineBreak(String parent, String typeName) { + return (parent.length() + typeName.length() < 50 ? "" : "\n"); + } + public void writeSlimeEncoder(Writer w, CNode node, String parent, boolean root) throws IOException { String indent = " "; @@ -929,55 +905,54 @@ public class CppClassBuilder implements ClassBuilder { if (child.isArray) { w.write(indent + "__c.setString(\"type\", \"array\");\n"); w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); - w.write(indent + "for (size_t __i = 0; __i < " + childName + ".size(); __i++) {\n"); + w.write(indent + "for (const auto & child : " + childName +") {\n"); w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); if (child instanceof LeafCNode.EnumLeaf) { String repType = slimeTypeMap.get("enum"); w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); w.write(indent + " __c3.set" + repType); - w.write("(\"value\", vespalib::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); + w.write("(\"value\", vespalib::Memory(get" + childType + "Name(child)));\n"); } else if (child instanceof LeafCNode) { String type = ((LeafCNode) child).getType(); String repType = slimeTypeMap.get(type); w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); w.write(indent + " __c3.set" + repType); if ("String".equals(repType)) { - w.write("(\"value\", vespalib::Memory(" + childName + "[__i]));\n"); + w.write("(\"value\", vespalib::Memory(child));\n"); } else { - w.write("(\"value\", " + childName + "[__i]);\n"); + w.write("(\"value\", child);\n"); } } else { w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); - w.write(indent + " " + childName + "[__i].serialize(__c4);\n"); + w.write(indent + " child.serialize(__c4);\n"); } w.write(indent + "}\n"); } else if (child.isMap) { w.write(indent + "__c.setString(\"type\", \"map\");\n"); w.write(indent + "vespalib::slime::Cursor & __c2 = __c.setArray(\"value\");\n"); - String childMapType = getTypeName(child, true); - w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n"); + w.write(indent + "for (const auto & entry : " + childName + ") {\n"); w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); - w.write(indent + " __c3.setString(\"key\", vespalib::Memory(it->first));\n"); + w.write(indent + " __c3.setString(\"key\", vespalib::Memory(entry.first));\n"); if (child instanceof LeafCNode.EnumLeaf) { String repType = slimeTypeMap.get("enum"); w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); w.write(indent + " __c3.set" + repType); - w.write("(\"value\", vespalib::Memory(get" + childType + "Name(it->second)));\n"); + w.write("(\"value\", vespalib::Memory(get" + childType + "Name(entry.second)));\n"); } else if (child instanceof LeafCNode) { String type = ((LeafCNode) child).getType(); String repType = slimeTypeMap.get(type); w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); w.write(indent + " __c3.set" + repType); if ("String".equals(repType)) { - w.write("(\"value\", vespalib::Memory(it->second));\n"); + w.write("(\"value\", vespalib::Memory(entry.second));\n"); } else { - w.write("(\"value\", it->second);\n"); + w.write("(\"value\", entry.second);\n"); } } else { w.write(indent + " __c3.setString(\"type\", \"struct\");\n"); w.write(indent + " Cursor & __c4 = __c3.setObject(\"value\");\n"); - w.write(indent + " it->second.serialize(__c4);\n"); + w.write(indent + " entry.second.serialize(__c4);\n"); } w.write(indent + "}\n"); } else { @@ -1033,39 +1008,13 @@ public class CppClassBuilder implements ClassBuilder { if (child.isArray) { w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); w.write(indent + " " + childName + ".push_back("); - if (child instanceof LeafCNode.EnumLeaf) { - String repType = slimeTypeMap.get("enum"); - w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); - } else if (child instanceof LeafCNode) { - String type = ((LeafCNode) child).getType(); - String repType = slimeTypeMap.get(type); - if ("String".equals(repType)) { - w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); - } else { - w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); - } - } else { - w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); - } + writeSlimeChild(w, child, childType, inspectorLine); w.write(");\n"); w.write(indent + "}\n"); } else if (child.isMap) { w.write(indent + "for (size_t __i = 0; __i < " + inspectorLine + ".children(); __i++) {\n"); w.write(indent + " " + childName + "[" + inspectorLine + "[__i][\"key\"].asString().make_string()] = "); - if (child instanceof LeafCNode.EnumLeaf) { - String repType = slimeTypeMap.get("enum"); - w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); - } else if (child instanceof LeafCNode) { - String type = ((LeafCNode) child).getType(); - String repType = slimeTypeMap.get(type); - if ("String".equals(repType)) { - w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); - } else { - w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); - } - } else { - w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); - } + writeSlimeChild(w, child, childType, inspectorLine); w.write(";\n"); w.write(indent + "}\n"); } else { @@ -1090,6 +1039,23 @@ public class CppClassBuilder implements ClassBuilder { w.write("}\n\n"); } + private void writeSlimeChild(Writer w, CNode child, String childType, String inspectorLine) throws IOException { + if (child instanceof LeafCNode.EnumLeaf) { + String repType = slimeTypeMap.get("enum"); + w.write("get" + childType + "(" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string())"); + } else if (child instanceof LeafCNode) { + String type = ((LeafCNode) child).getType(); + String repType = slimeTypeMap.get(type); + if ("String".equals(repType)) { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "().make_string()"); + } else { + w.write("" + inspectorLine + "[__i][\"value\"].as" + repType + "()"); + } + } else { + w.write(childType + "(" + inspectorLine + "[__i][\"value\"])"); + } + } + public void writeSlimeConstructor(Writer w, CNode node, String parent, boolean root) throws IOException { String tmpName = getTypeName(node, false); String typeName = root ? getInternalClassName(node) : tmpName; @@ -1110,7 +1076,8 @@ public class CppClassBuilder implements ClassBuilder { String childInspector = "__inspector[\"" + child.getName() + "\"]"; if (child.isArray) { String inserterName = "__" + childName + "Inserter"; - w.write(indent + "::config::internal::VectorInserter<" + childType); + String childVectorType = getTypeName(child, true); + w.write(indent + "::config::internal::VectorInserter<" + childVectorType); if (child instanceof LeafCNode.EnumLeaf) { w.write(", Internal" + childType + "Converter"); } diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java index 17ebeaf78a5..f77c0cbf932 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.configserver.flags.http; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import java.util.logging.Level; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.vespa.configserver.flags.FlagsDb; @@ -18,18 +17,19 @@ import com.yahoo.yolean.Exceptions; import java.io.UncheckedIOException; import java.net.URI; import java.util.Objects; +import java.util.logging.Level; /** * Handles /flags/v1 requests * * @author hakonhall */ -public class FlagsHandler extends LoggingRequestHandler { +public class FlagsHandler extends ThreadedHttpRequestHandler { private final FlagsDb flagsDb; @Inject - public FlagsHandler(LoggingRequestHandler.Context context, FlagsDb flagsDb) { + public FlagsHandler(ThreadedHttpRequestHandler.Context context, FlagsDb flagsDb) { super(context); this.flagsDb = flagsDb; } diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java index b078b9a42f9..3ca0567a1c5 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java @@ -41,7 +41,7 @@ public class FlagsHandlerTest { private static final String FLAGS_V1_URL = "https://foo.com:4443/flags/v1"; private final FlagsDb flagsDb = new FlagsDbImpl(new MockCurator()); - private final FlagsHandler handler = new FlagsHandler(FlagsHandler.testOnlyContext(), flagsDb); + private final FlagsHandler handler = new FlagsHandler(FlagsHandler.testContext(), flagsDb); @Test public void testV1() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 80194337daa..ccd9f0e7044 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -88,7 +88,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; @@ -111,8 +110,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; -import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; @@ -147,6 +146,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Metric metric; private final SecretStoreValidator secretStoreValidator; private final ClusterReindexingStatusClient clusterReindexingStatusClient; + private final FlagSource flagSource; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -203,8 +203,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.metric = Objects.requireNonNull(metric); this.secretStoreValidator = Objects.requireNonNull(secretStoreValidator); this.clusterReindexingStatusClient = clusterReindexingStatusClient; + this.flagSource = flagSource; } + // Should be used by tests only (first constructor in this class makes sure we use injectable components where possible) public static class Builder { private TenantRepository tenantRepository; private Optional<Provisioner> hostProvisioner; @@ -217,6 +219,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private Metric metric = new NullMetric(); private SecretStoreValidator secretStoreValidator = new SecretStoreValidator(new SecretStoreProvider().get()); private FlagSource flagSource = new InMemoryFlagSource(); + private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker(); public Builder withTenantRepository(TenantRepository tenantRepository) { this.tenantRepository = tenantRepository; @@ -280,11 +283,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } + public Builder withConfigConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) { + this.configConvergenceChecker = configConvergenceChecker; + return this; + } + public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, hostProvisioner, InfraDeployerProvider.empty().getInfraDeployer(), - new ConfigConvergenceChecker(), + configConvergenceChecker, httpProxy, configserverConfig, orchestrator, @@ -631,7 +639,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) { Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); - Instant instant = Instant.now().minus(keepFileReferences); + Instant instant = clock.instant().minus(keepFileReferences); return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) @@ -747,10 +755,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public ServiceListResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, - URI uri, Duration timeoutPerService, Optional<Version> vespaVersion) { - return convergeChecker.getServiceConfigGenerations(getApplication(applicationId, vespaVersion), uri, timeoutPerService); + return convergeChecker.checkConvergenceForAllServices(getApplication(applicationId, vespaVersion), timeoutPerService); } public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; } @@ -1016,6 +1023,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return applicationId.orElse(null); } + public FlagSource flagSource() { return flagSource; } + private Session validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { Session session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { @@ -1042,6 +1051,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getTenant(appId).getSessionRepository().getActiveApplicationSet(appId); } + public Application getActiveApplication(ApplicationId applicationId) { + return getActiveApplicationSet(applicationId) + .map(a -> a.getForVersionOrLatest(Optional.empty(), clock.instant())) + .orElseThrow(() -> new RuntimeException("Found no active application for " + applicationId)); + } + private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 57e49ef3e8d..8d51a91bc3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.yolean.Exceptions; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -73,6 +74,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; private final ConfigServerMaintenance configServerMaintenance; + private final Clock clock; @SuppressWarnings("unused") // Injected component @Inject @@ -83,21 +85,22 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, - flagSource, convergence); + flagSource, convergence, Clock.systemUTC()); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, VipStatusMode vipStatusMode, - FlagSource flagSource, ConfigConvergenceChecker convergence) { + FlagSource flagSource, ConfigConvergenceChecker convergence, Clock clock) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, - FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode, flagSource, convergence); + FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode, flagSource, convergence, clock); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode, FlagSource flagSource, ConfigConvergenceChecker convergence) { + VipStatusMode vipStatusMode, FlagSource flagSource, ConfigConvergenceChecker convergence, + Clock clock) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; @@ -107,6 +110,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; + this.clock = clock; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); configServerMaintenance = new ConfigServerMaintenance(configserverConfig, @@ -197,8 +201,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private void startRpcServerWithFileDistribution() { rpcServerExecutor.execute(server); - Instant end = Instant.now().plus(Duration.ofSeconds(10)); - while (!server.isRunning() && Instant.now().isBefore(end)) { + Instant end = clock.instant().plus(Duration.ofSeconds(10)); + while (!server.isRunning() && clock.instant().isBefore(end)) { try { Thread.sleep(10); } catch (InterruptedException e) { @@ -220,7 +224,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void redeployAllApplications() throws InterruptedException { - Instant end = Instant.now().plus(maxDurationOfRedeployment); + Instant end = clock.instant().plus(maxDurationOfRedeployment); List<ApplicationId> applicationsToRedeploy = applicationRepository.listApplications(); Collections.shuffle(applicationsToRedeploy); long failCount = 0; @@ -233,7 +237,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(Level.INFO, "Redeployment of " + applicationsToRedeploy + " not finished, will retry in " + sleepTime); Thread.sleep(sleepTime.toMillis()); } - } while ( ! applicationsToRedeploy.isEmpty() && Instant.now().isBefore(end)); + } while ( ! applicationsToRedeploy.isEmpty() && clock.instant().isBefore(end)); if ( ! applicationsToRedeploy.isEmpty()) throw new RuntimeException("Redeploying applications not finished after " + maxDurationOfRedeployment + @@ -296,11 +300,11 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void logProgress(LogState logState, int failedDeployments, int finishedDeployments) { - if ( ! Duration.between(logState.lastLogged, Instant.now()).minus(Duration.ofSeconds(10)).isNegative() + if ( ! Duration.between(logState.lastLogged, clock.instant()).minus(Duration.ofSeconds(10)).isNegative() && (logState.failedDeployments != failedDeployments || logState.finishedDeployments != finishedDeployments)) { log.log(Level.INFO, () -> finishedDeployments + " of " + logState.applicationCount + " apps redeployed " + "(" + failedDeployments + " failed)"); - logState.update(Instant.now(), failedDeployments, finishedDeployments); + logState.update(clock.instant(), failedDeployments, finishedDeployments); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java index d3295c023b0..a96f9db855c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import java.time.Clock; import java.time.Duration; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index ad14cf4aab6..a49af0a0e51 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.model.api.ApplicationClusterInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; @@ -39,14 +40,17 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; /** @@ -65,13 +69,14 @@ public class ConfigConvergenceChecker extends AbstractComponent { QRSERVER.serviceName, LOGSERVER_CONTAINER.serviceName, CLUSTERCONTROLLER_CONTAINER.serviceName, + METRICS_PROXY_CONTAINER.serviceName, "searchnode", "storagenode", "distributor" ); - private final Executor responseHandlerExecutor = + private final ExecutorService responseHandlerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config-convergence-checker-response-handler-")); private final ObjectMapper jsonMapper = new ObjectMapper(); @@ -80,20 +85,43 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Fetches the active config generation for all services in the given application. */ public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) { + return getServiceConfigGenerations(application, timeoutPerService, true); + } + + /** + * Fetches the active config generation for all services in the given application. Will not check services + * which defer config changes until restart if checkAll is false. + */ + private Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService, boolean checkAll) { List<ServiceInfo> servicesToCheck = new ArrayList<>(); application.getModel().getHosts() .forEach(host -> host.getServices().stream() .filter(service -> serviceTypesToCheck.contains(service.getServiceType())) + .filter(serviceInfo -> shouldCheckService(checkAll, application, serviceInfo)) .forEach(service -> getStatePort(service).ifPresent(port -> servicesToCheck.add(service)))); + log.log(Level.FINE, "Services to check for config convergence: " + servicesToCheck); return getServiceGenerations(servicesToCheck, timeoutPerService); } - /** Check all services in given application. Returns the minimum current generation of all services */ - public ServiceListResponse getServiceConfigGenerations(Application application, URI uri, Duration timeoutPerService) { - Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService); + /** Checks all services in given application. Returns the minimum current generation of all services */ + public ServiceListResponse checkConvergenceForAllServices(Application application, Duration timeoutPerService) { + return checkConvergence(application, timeoutPerService, true); + } + + /** + * Checks services except those which defer config changes until restart in the given application. + * Returns the minimum current generation of those services. + */ + public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application) { + Duration timeoutPerService = Duration.ofSeconds(10); + return checkConvergence(application, timeoutPerService, false); + } + + private ServiceListResponse checkConvergence(Application application, Duration timeoutPerService, boolean checkAll) { + Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService, checkAll); long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1); - return new ServiceListResponse(currentGenerations, uri, application.getApplicationGeneration(), currentGeneration); + return new ServiceListResponse(currentGenerations, application.getApplicationGeneration(), currentGeneration); } /** Check service identified by host and port in given application */ @@ -113,6 +141,26 @@ public class ConfigConvergenceChecker extends AbstractComponent { } } + private boolean shouldCheckService(boolean checkServicesWithDeferChangesUntilRestart, Application application, ServiceInfo serviceInfo) { + if (checkServicesWithDeferChangesUntilRestart) return true; + if (isNotContainer(serviceInfo)) return true; + return serviceIsInClusterWhichShouldBeChecked(application, serviceInfo); + } + + private boolean isNotContainer(ServiceInfo serviceInfo) { + return ! List.of(CONTAINER.serviceName, QRSERVER.serviceName, METRICS_PROXY_CONTAINER).contains(serviceInfo.getServiceType()); + } + + // Don't check service in a cluster which uses restartOnDeploy (new config will not be used until service is restarted) + private boolean serviceIsInClusterWhichShouldBeChecked(Application application, ServiceInfo serviceInfo) { + Set<ApplicationClusterInfo> excludeFromChecking = application.getModel().applicationClusterInfo() + .stream() + .filter(ApplicationClusterInfo::getDeferChangesUntilRestart) + .collect(Collectors.toSet()); + + return excludeFromChecking.stream().noneMatch(info -> info.name().equals(serviceInfo.getProperty("clustername").orElse(""))); + } + /** Gets service generation for a list of services (in parallel). */ private Map<ServiceInfo, Long> getServiceGenerations(List<ServiceInfo> services, Duration timeout) { try (CloseableHttpAsyncClient client = createHttpClient()) { @@ -196,6 +244,16 @@ public class ConfigConvergenceChecker extends AbstractComponent { .findFirst(); } + @Override + public void deconstruct() { + responseHandlerExecutor.shutdown(); + try { + responseHandlerExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Unable to shutdown executor", e); + } + } + private static long generationFromContainerState(JsonNode state) { return state.get("config").get("generation").asLong(-1); } @@ -256,23 +314,23 @@ public class ConfigConvergenceChecker extends AbstractComponent { public final boolean converged; public final Optional<String> errorMessage; - public ServiceResponse(Status status, Long wantedGeneration) { - this(status, wantedGeneration, 0L); + public ServiceResponse(Status status, long wantedGeneration) { + this(status, wantedGeneration, 0); } - public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration) { + public ServiceResponse(Status status, long wantedGeneration, long currentGeneration) { this(status, wantedGeneration, currentGeneration, false); } - public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged) { + public ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged) { this(status, wantedGeneration, currentGeneration, converged, Optional.empty()); } - public ServiceResponse(Status status, Long wantedGeneration, String errorMessage) { - this(status, wantedGeneration, 0L, false, Optional.ofNullable(errorMessage)); + public ServiceResponse(Status status, long wantedGeneration, String errorMessage) { + this(status, wantedGeneration, 0, false, Optional.ofNullable(errorMessage)); } - private ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged, Optional<String> errorMessage) { + private ServiceResponse(Status status, long wantedGeneration, long currentGeneration, boolean converged, Optional<String> errorMessage) { this.status = status; this.wantedGeneration = wantedGeneration; this.currentGeneration = currentGeneration; @@ -285,15 +343,15 @@ public class ConfigConvergenceChecker extends AbstractComponent { public static class ServiceListResponse { public final List<Service> services = new ArrayList<>(); - public final URI uri; public final long wantedGeneration; public final long currentGeneration; + public final boolean converged; - public ServiceListResponse(Map<ServiceInfo, Long> services, URI uri, long wantedGeneration, long currentGeneration) { + public ServiceListResponse(Map<ServiceInfo, Long> services, long wantedGeneration, long currentGeneration) { services.forEach((key, value) -> this.services.add(new Service(key, value))); - this.uri = uri; this.wantedGeneration = wantedGeneration; this.currentGeneration = currentGeneration; + this.converged = currentGeneration >= wantedGeneration; } public List<Service> services() { return services; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java index f0711e5c238..88cddb93d9d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java @@ -5,7 +5,13 @@ package com.yahoo.vespa.config.server.application; * @author Ulf Lilleengen */ public class ConfigNotConvergedException extends RuntimeException { + + public ConfigNotConvergedException(Throwable t) { + super(t); + } + public ConfigNotConvergedException(String message) { super(message); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 98ddf702a7f..937ec2bb0e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -472,7 +472,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica // If some are missing, quorum is enough, but wait for all up to 5 seconds before returning if (respondents.size() >= barrierMemberCount()) { if (gotQuorumTime.isBefore(startTime)) - gotQuorumTime = Instant.now(); + gotQuorumTime = clock.instant(); // Give up if more than some time has passed since we got quorum, otherwise continue if (Duration.between(Instant.now(), gotQuorumTime.plus(waitForAll)).isNegative()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 71e5489fb2e..d7977477f30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -36,6 +36,7 @@ public class DeployHandlerLogger implements DeployLogger { } @Override + @SuppressWarnings("deprecation") public void log(Level level, String message) { if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return; @@ -46,6 +47,7 @@ public class DeployHandlerLogger implements DeployLogger { } @Override + @SuppressWarnings("deprecation") public void logApplicationPackage(Level level, String message) { if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 4b7e39f48a4..e6159cbfa33 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ServiceInfo; @@ -19,6 +20,9 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.ApplicationRepository.Activation; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; +import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ReindexActions; import com.yahoo.vespa.config.server.configchange.RestartActions; @@ -35,6 +39,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; + /** * The process of deploying an application. * Deployments are created by an {@link ApplicationRepository}. @@ -149,6 +155,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); if ( ! restartActions.isEmpty()) { + + waitForConfigToConverge(applicationId); + Set<String> hostnames = restartActions.getEntries().stream() .flatMap(entry -> entry.getServices().stream()) .map(ServiceInfo::getHostName) @@ -165,6 +174,30 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } + private void waitForConfigToConverge(ApplicationId applicationId) { + deployLogger.log(Level.INFO, "Wait for all services to use new config generation before restarting"); + while (true) { + try { + params.get().getTimeoutBudget().assertNotTimedOut( + () -> "Timeout exceeded while waiting for config convergence for " + applicationId); + } catch (UncheckedTimeoutException e) { + throw new ConfigNotConvergedException(e); + } + + ConfigConvergenceChecker convergenceChecker = applicationRepository.configConvergenceChecker(); + Application app = applicationRepository.getActiveApplication(applicationId); + ServiceListResponse response = convergenceChecker.checkConvergenceUnlessDeferringChangesUntilRestart(app); + if (response.converged) { + deployLogger.log(Level.INFO, "Services converged on new config generation " + response.currentGeneration); + return; + } else { + deployLogger.log(Level.INFO, "Services did not converge on new config generation " + + response.wantedGeneration + ", current generation: " + response.currentGeneration + ", will retry"); + try { Thread.sleep(10_000); } catch (InterruptedException e) { /* ignore */ } + } + } + } + private void storeReindexing(ApplicationId applicationId, long requiredSession) { applicationRepository.modifyReindexing(applicationId, reindexing -> { if (configChangeActions != null) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 8c1fcf9d9f2..38245f26a18 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -177,24 +177,20 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; - private final boolean enableFeedBlockInDistributor; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; - private final boolean ignoreMergeQueueLimit; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; private final int metricsproxyNumThreads; + private final int availableProcessors; private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; - private final int distributorMergeBusyWait; - private final boolean distributorEnhancedMaintenanceScheduling; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; - private final boolean asyncApplyBucketDiff; private final boolean ignoreThreadStackSizes; private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; @@ -204,6 +200,11 @@ public class ModelContextImpl implements ModelContext { private final List<String> ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; + private final String mergeThrottlingPolicy; + private final double persistenceThrottlingWsDecrementFactor; + private final double persistenceThrottlingWsBackoff; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; + private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -220,24 +221,20 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); - this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT); this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); this.metricsproxyNumThreads = flagValue(source, appId, Flags.METRICSPROXY_NUM_THREADS); + this.availableProcessors = flagValue(source, appId, Flags.AVAILABLE_PROCESSORS); this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); - this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); - this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING); - this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; + this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY); this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); - this.asyncApplyBucketDiff = flagValue(source, appId, Flags.ASYNC_APPLY_BUCKET_DIFF); this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES); this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); @@ -247,6 +244,11 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); + this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); + this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); + this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); + this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES); } @@ -263,7 +265,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { @@ -271,18 +272,15 @@ public class ModelContextImpl implements ModelContext { } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } - @Override public int metricsproxyNumThreads() { return metricsproxyNumThreads; } + @Override public int defaultPoolNumThreads() { return metricsproxyNumThreads; } + @Override public int availableProcessors() { return availableProcessors; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } - @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } - @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @@ -292,6 +290,11 @@ public class ModelContextImpl implements ModelContext { @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } + @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } + @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } + @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { @@ -354,6 +357,7 @@ public class ModelContextImpl implements ModelContext { private final List<X509Certificate> operatorCertificates; private final List<String> tlsCiphersOverride; private final List<String> zoneDnsSuffixes; + private final List<String> environmentVariables; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -386,13 +390,15 @@ public class ModelContextImpl implements ModelContext { this.tenantSecretStores = tenantSecretStores; this.secretStore = secretStore; this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.operatorCertificates = operatorCertificates; this.tlsCiphersOverride = PermanentFlags.TLS_CIPHERS_OVERRIDE.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes(); + this.environmentVariables = PermanentFlags.ENVIRONMENT_VARIABLES.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -474,6 +480,9 @@ public class ModelContextImpl implements ModelContext { .value(); } + @Override + public List<String> environmentVariables() { return environmentVariables; } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index ca822b7a99d..0acf32d79a7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -90,15 +90,15 @@ public class ZooKeeperClient { } private void writeSchemas(ApplicationPackage app) throws IOException { - Collection<NamedReader> sds = app.getSchemas(); - if (sds.isEmpty()) return; + Collection<NamedReader> schemas = app.getSchemas(); + if (schemas.isEmpty()) return; Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); curator.create(zkPath); // Ensures that ranking expressions and other files are also written - writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); - writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); - for (NamedReader sd : sds) { + writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, true); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, true); + for (NamedReader sd : schemas) { curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader()))); sd.getReader().close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 8e5eee2104c..a3f9dbfad11 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -223,6 +223,7 @@ public class FileServer { } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { + @SuppressWarnings("removal") // TODO Vespa 8: remove ConfigSourceSet configSourceSet = new ConfigSourceSet(configServers); if (configServers.size() == 0) return FileDownloader.emptyConnectionPool(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 96e1767b462..c414d8dfa9a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -2,13 +2,13 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import java.io.IOException; import java.io.OutputStream; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; @@ -50,7 +50,8 @@ public class HttpErrorResponse extends HttpResponse { UNKNOWN_VESPA_VERSION, PARENT_HOST_NOT_READY, CERTIFICATE_NOT_READY, - LOAD_BALANCER_NOT_READY + LOAD_BALANCER_NOT_READY, + CONFIG_NOT_CONVERGED } public static HttpErrorResponse notFoundError(String msg) { @@ -101,6 +102,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(CONFLICT, ErrorCode.CERTIFICATE_NOT_READY.name(), msg); } + public static HttpErrorResponse configNotConverged(String msg) { + return new HttpErrorResponse(CONFLICT, ErrorCode.CONFIG_NOT_CONVERGED.name(), msg); + } + public static HttpErrorResponse loadBalancerNotReady(String msg) { return new HttpErrorResponse(CONFLICT, ErrorCode.LOAD_BALANCER_NOT_READY.name(), msg); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 2dad2c060cc..190005771c7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -9,7 +9,8 @@ import com.yahoo.config.provision.exception.ActivationConflictException; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; @@ -24,7 +25,7 @@ import java.util.logging.Level; * * @author hmusum */ -public class HttpHandler extends LoggingRequestHandler { +public class HttpHandler extends ThreadedHttpRequestHandler { public HttpHandler(HttpHandler.Context ctx) { super(ctx); @@ -68,6 +69,8 @@ public class HttpHandler extends LoggingRequestHandler { return HttpErrorResponse.parentHostNotReady(getMessage(e, request)); } catch (CertificateNotReadyException e) { return HttpErrorResponse.certificateNotReady(getMessage(e, request)); + } catch (ConfigNotConvergedException e) { + return HttpErrorResponse.configNotConverged(getMessage(e, request)); } catch (LoadBalancerServiceException e) { return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request)); } catch (Exception e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java index d9cbc7bc533..569ed1525b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java @@ -63,6 +63,7 @@ public class RoutingStatusApiHandler extends RestApiRequestHandler<RoutingStatus private static RestApi createRestApiDefinition(RoutingStatusApiHandler self) { return RestApi.builder() + // TODO(mpolden): Remove this route when clients have migrated to v2 .addRoute(RestApi.route("/routing/v1/status") .get(self::listInactiveDeployments)) .addRoute(RestApi.route("/routing/v1/status/zone") @@ -72,9 +73,27 @@ public class RoutingStatusApiHandler extends RestApiRequestHandler<RoutingStatus .addRoute(RestApi.route("/routing/v1/status/{upstreamName}") .get(self::getDeploymentStatus) .put(self::changeDeploymentStatus)) + .addRoute(RestApi.route("/routing/v2/status") + .get(self::getDeploymentStatusV2)) .build(); } + /* Get inactive deployments and zone status */ + private SlimeJsonResponse getDeploymentStatusV2(RestApi.RequestContext context) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor inactiveDeploymentsArray = root.setArray("inactiveDeployments"); + curator.getChildren(DEPLOYMENT_STATUS_ROOT).stream() + .filter(upstreamName -> deploymentStatus(upstreamName).status() == RoutingStatus.out) + .sorted() + .forEach(upstreamName -> { + Cursor deploymentObject = inactiveDeploymentsArray.addObject(); + deploymentObject.setString("upstreamName", upstreamName); + }); + root.setBool("zoneActive", zoneStatus() == RoutingStatus.in); + return new SlimeJsonResponse(slime); + } + /** Get upstream of all deployments with status OUT */ private SlimeJsonResponse listInactiveDeployments(RestApi.RequestContext context) { List<String> inactiveDeployments = curator.getChildren(DEPLOYMENT_STATUS_ROOT).stream() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 0131517818d..df4df234ed0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -117,10 +117,9 @@ public class ApplicationHandler extends HttpHandler { private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) { ServiceListResponse response = applicationRepository.servicesToCheckForConfigConvergence(applicationId, - request.getUri(), getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); - return new HttpServiceListResponse(response); + return new HttpServiceListResponse(response, request.getUri()); } private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) { @@ -370,7 +369,7 @@ public class ApplicationHandler extends HttpHandler { static class HttpServiceListResponse extends JSONResponse { // Pre-condition: servicesToCheck has a state port - public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response) { + public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response, URI uri) { super(200); Cursor serviceArray = object.setArray("services"); response.services().forEach((service) -> { @@ -381,13 +380,13 @@ public class ApplicationHandler extends HttpHandler { serviceObject.setString("host", hostName); serviceObject.setLong("port", statePort); serviceObject.setString("type", serviceInfo.getServiceType()); - serviceObject.setString("url", response.uri.toString() + "/" + hostName + ":" + statePort); + serviceObject.setString("url", uri.toString() + "/" + hostName + ":" + statePort); serviceObject.setLong("currentGeneration", service.currentGeneration); }); - object.setString("url", response.uri.toString()); + object.setString("url", uri.toString()); object.setLong("currentGeneration", response.currentGeneration); object.setLong("wantedGeneration", response.wantedGeneration); - object.setBool("converged", response.currentGeneration >= response.wantedGeneration); + object.setBool("converged", response.converged); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index a0e8d83fba1..9d8a3d87811 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -96,6 +96,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { File downloadDirectory, Supervisor supervisor) { List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(configserverConfig); + @SuppressWarnings("removal") // TODO Vespa 8: remove ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = (otherConfigServersInCluster.isEmpty()) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index dccabca2858..7677417f1a9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.model; -import com.google.common.base.Joiner; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; @@ -11,19 +10,14 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; @@ -37,12 +31,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag generateNonMtlsEndpoint; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - generateNonMtlsEndpoint = Flags.GENERATE_NON_MTLS_ENDPOINT.bindTo(flagSource); } @Override @@ -76,13 +68,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { // TODO: read active rotation from ApplicationClusterInfo ab.activeRotation(getActiveRotation(app)); - ab.usePowerOfTwoChoicesLb(true); - ab.generateNonMtlsEndpoint(generateNonMtlsEndpoint(app)); - - // TODO: Remove when endpoints-config is read by all load balancers - app.getModel().getHosts().stream() - .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) - .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); Set<ApplicationClusterInfo> applicationClusterInfos = app.getModel().applicationClusterInfo(); List<LbServicesConfig.Tenants.Applications.Endpoints.Builder> endpointBuilder = applicationClusterInfos.stream() @@ -123,38 +108,4 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return activeRotation; } - private boolean generateNonMtlsEndpoint(ApplicationInfo app) { - return generateNonMtlsEndpoint.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value(); - } - - private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { - LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); - hb.hostname(hostInfo.getHostname()); - hostInfo.getServices().forEach(serviceInfo -> hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo))); - return hb; - } - - private LbServicesConfig.Tenants.Applications.Hosts.Services.Builder getServiceConfig(ServiceInfo serviceInfo) { - List<String> endpointAliases = Stream.of(serviceInfo.getProperty("endpointaliases").orElse("").split(",")). - filter(prop -> !"".equals(prop)).collect(Collectors.toList()); - endpointAliases.addAll(Stream.of(serviceInfo.getProperty("rotations").orElse("").split(",")).filter(prop -> !"".equals(prop)).collect(Collectors.toList())); - Collections.sort(endpointAliases); - - LbServicesConfig.Tenants.Applications.Hosts.Services.Builder sb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Builder() - .type(serviceInfo.getServiceType()) - .clustertype(serviceInfo.getProperty("clustertype").orElse("")) - .clustername(serviceInfo.getProperty("clustername").orElse("")) - .configId(serviceInfo.getConfigId()) - .servicealiases(Stream.of(serviceInfo.getProperty("servicealiases").orElse("").split(",")). - filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList())) - .endpointaliases(endpointAliases) - .index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999"))); - serviceInfo.getPorts().forEach(portInfo -> { - LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() - .number(portInfo.getPort()) - .tags(Joiner.on(" ").join(portInfo.getTags())); - sb.ports(pb); - }); - return sb; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index e4a0fa81f94..f8ee3e5e0c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 42ccdffb2af..ab02594d164 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -578,7 +578,7 @@ public class SessionRepository { // Skip sessions newly added (we might have a session in the file system, but not in ZooKeeper, // we don't want to touch any of them) if (newSessions.contains(candidate.getSessionId())) { - log.log(Level.INFO, () -> "Skipping session " + candidate.getSessionId() + ", newly created: "); + log.log(Level.FINE, () -> "Skipping expiring newly created session " + candidate.getSessionId()); continue; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index b3c2fa2e300..7d53c383cf5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -330,7 +330,7 @@ public class TenantRepository { private Tenant createTenant(TenantName tenantName, Instant created) { if (tenants.containsKey(tenantName)) return getTenant(tenantName); - Instant start = Instant.now(); + Instant start = clock.instant(); log.log(Level.FINE, () -> "Adding tenant '" + tenantName); TenantApplications applicationRepo = new TenantApplications(tenantName, @@ -375,7 +375,7 @@ public class TenantRepository { configDefinitionRepo, zookeeperServerConfig.juteMaxBuffer()); log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created + - ". Bootstrapping in " + Duration.between(start, Instant.now())); + ". Bootstrapping in " + Duration.between(start, clock.instant())); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created); createAndWriteTenantMetaData(tenant); tenants.putIfAbsent(tenantName, tenant); diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java index d81297a87e9..d3c19bb0ba5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.serviceview; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.restapi.UriBuilder; @@ -57,7 +57,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl } @Inject - public StateRequestHandler(LoggingRequestHandler.Context context, + public StateRequestHandler(ThreadedHttpRequestHandler.Context context, ConfigserverConfig configserverConfig) { super(context, StateRequestHandler::createRestApiDefinition); this.restApiPort = configserverConfig.httpport(); diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 729b50419d7..b3a37c6f669 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0" xmlns:preprocess="properties"> - <container id="configserver" jetty="true" version="1.0"> + <container id="configserver" version="1.0"> <config name="container.jdisc.config.health-monitor"> <initialStatus>initializing</initialStatus> </config> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 7baad75ebc5..284a0aa0a62 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantMetaData; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.config.util.ConfigUtils; @@ -94,7 +95,6 @@ public class ApplicationRepositoryTest { private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); - private final static TenantName tenant3 = TenantName.from("test3"); private final static ManualClock clock = new ManualClock(Instant.now()); private ApplicationRepository applicationRepository; @@ -103,6 +103,7 @@ public class ApplicationRepositoryTest { private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; private Curator curator; + private ConfigserverConfig configserverConfig; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -114,24 +115,22 @@ public class ApplicationRepositoryTest { @Before public void setup() throws IOException { curator = new MockCurator(); - ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + configserverConfig = new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - InMemoryFlagSource flagSource = new InMemoryFlagSource(); tenantRepository = new TestTenantRepository.Builder() .withClock(clock) .withConfigserverConfig(configserverConfig) .withCurator(curator) .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) - .withFlagSource(flagSource) + .withFlagSource(new InMemoryFlagSource()) .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); - tenantRepository.addTenant(tenant3); orchestrator = new OrchestratorMock(); provisioner = new MockProvisioner(); applicationRepository = new ApplicationRepository.Builder() @@ -146,47 +145,42 @@ public class ApplicationRepositoryTest { } @Test - public void prepareAndActivate() { - PrepareResult result = prepareAndActivate(testApp); - assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); - assertTrue(result.configChangeActions().getReindexActions().isEmpty()); - assertTrue(result.configChangeActions().getRestartActions().isEmpty()); - - Tenant tenant = applicationRepository.getTenant(applicationId()); - Session session = applicationRepository.getActiveLocalSession(tenant, applicationId()); - session.getAllocatedHosts(); - } - - @Test public void prepareAndActivateWithTenantMetaData() { - Instant startTime = clock.instant(); + long startTime = clock.instant().toEpochMilli(); Duration duration = Duration.ofHours(1); clock.advance(duration); - Instant deployTime = clock.instant(); + long deployTime = clock.instant().toEpochMilli(); PrepareResult result = prepareAndActivate(testApp); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getReindexActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); - Tenant tenant = applicationRepository.getTenant(applicationId()); + Session session = applicationRepository.getActiveLocalSession(tenant(), applicationId()); + session.getAllocatedHosts(); - assertEquals(startTime.toEpochMilli(), - applicationRepository.getTenantMetaData(tenant).createdTimestamp().toEpochMilli()); - assertEquals(deployTime.toEpochMilli(), - applicationRepository.getTenantMetaData(tenant).lastDeployTimestamp().toEpochMilli()); + assertEquals(startTime, tenantMetaData(tenant()).createdTimestamp().toEpochMilli()); + assertEquals(deployTime, tenantMetaData(tenant()).lastDeployTimestamp().toEpochMilli()); // Creating a new tenant will have metadata with timestamp equal to current time clock.advance(duration); - Instant createTenantTime = clock.instant(); + long createTenantTime = clock.instant().toEpochMilli(); Tenant fooTenant = tenantRepository.addTenant(TenantName.from("foo")); - assertEquals(createTenantTime.toEpochMilli(), - applicationRepository.getTenantMetaData(fooTenant).createdTimestamp().toEpochMilli()); - assertEquals(createTenantTime.toEpochMilli(), - applicationRepository.getTenantMetaData(fooTenant).lastDeployTimestamp().toEpochMilli()); + assertEquals(createTenantTime, tenantMetaData(fooTenant).createdTimestamp().toEpochMilli()); + assertEquals(createTenantTime, tenantMetaData(fooTenant).lastDeployTimestamp().toEpochMilli()); } @Test public void prepareAndActivateWithRestart() { + applicationRepository = new ApplicationRepository.Builder() + .withTenantRepository(tenantRepository) + .withProvisioner(provisioner) + .withConfigserverConfig(configserverConfig) + .withOrchestrator(orchestrator) + .withLogRetriever(new MockLogRetriever()) + .withClock(clock) + .withConfigConvergenceChecker(new MockConfigConvergenceChecker(2)) + .build(); + prepareAndActivate(testAppJdiscOnly); PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); @@ -217,24 +211,20 @@ public class ApplicationRepositoryTest { @Test public void redeploy() { - PrepareResult result = deployApp(testApp); + long firstSessionId = deployApp(testApp).sessionId(); - long firstSessionId = result.sessionId(); - - PrepareResult result2 = deployApp(testApp); - long secondSessionId = result2.sessionId(); + long secondSessionId = deployApp(testApp).sessionId(); assertNotEquals(firstSessionId, secondSessionId); - Tenant tenant = applicationRepository.getTenant(applicationId()); - Session session = applicationRepository.getActiveLocalSession(tenant, applicationId()); + Session session = applicationRepository.getActiveLocalSession(tenant(), applicationId()); assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); } @Test public void createFromActiveSession() { - PrepareResult result = deployApp(testApp); - long sessionId = applicationRepository.createSessionFromExisting(applicationId(), false, timeoutBudget); - long originalSessionId = result.sessionId(); + long originalSessionId = deployApp(testApp).sessionId(); + + long sessionId = createSessionFromExisting(applicationId(), timeoutBudget); ApplicationMetaData originalApplicationMetaData = getApplicationMetaData(applicationId(), originalSessionId); ApplicationMetaData applicationMetaData = getApplicationMetaData(applicationId(), sessionId); @@ -269,20 +259,24 @@ public class ApplicationRepositoryTest { } @Test - public void deleteUnusedFileReferences() throws IOException, InterruptedException { + public void deleteUnusedFileReferences() throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); Duration keepFileReferencesDuration = Duration.ofSeconds(4); // Add file reference that is not in use and should be deleted (older than 'keepFileReferencesDuration') File filereferenceDirOldest = createFilereferenceOnDisk(new File(fileReferencesDir, "foo")); - //Thread.sleep(Duration.ofSeconds(1).toMillis()); + clock.advance(Duration.ofSeconds(1)); // Add file references that are not in use and could be deleted IntStream.range(0, 3).forEach(i -> { - createFilereferenceOnDisk(new File(fileReferencesDir, "bar" + i)); - try { Thread.sleep(Duration.ofSeconds(1).toMillis()); } catch (InterruptedException e) { /* ignore */ } + try { + createFilereferenceOnDisk(new File(fileReferencesDir, "bar" + i)); + } catch (IOException e) { + fail(e.getMessage()); + } + clock.advance(Duration.ofSeconds(1)); }); - Thread.sleep(keepFileReferencesDuration.toMillis()); + clock.advance(keepFileReferencesDuration); // Add file reference that is not in use, but should not be deleted (newer than 'keepFileReferencesDuration') File filereferenceDirNewest = createFilereferenceOnDisk(new File(fileReferencesDir, "baz")); @@ -309,17 +303,17 @@ public class ApplicationRepositoryTest { assertTrue(filereferenceDirNewest.exists()); } - private File createFilereferenceOnDisk(File filereferenceDir) { + private File createFilereferenceOnDisk(File filereferenceDir) throws IOException { assertTrue(filereferenceDir.mkdir()); - File bar = new File(filereferenceDir, "file"); - IOUtils.writeFile(bar, Utf8.toBytes("test")); + File file = new File(filereferenceDir, "bar"); + IOUtils.writeFile(file, Utf8.toBytes("test")); + Files.setAttribute(filereferenceDir.toPath(), "lastAccessTime", FileTime.from(clock.instant())); return filereferenceDir; } @Test public void delete() { - Tenant tenant = applicationRepository.getTenant(applicationId()); - SessionRepository sessionRepository = tenant.getSessionRepository(); + SessionRepository sessionRepository = tenant().getSessionRepository(); { PrepareResult result = deployApp(testApp); long sessionId = result.sessionId(); @@ -330,7 +324,7 @@ public class ApplicationRepositoryTest { assertNotNull(applicationRepository.getActiveSession(applicationId())); Path sessionNode = sessionRepository.getSessionPath(sessionId); assertTrue(curator.exists(sessionNode)); - TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs(); + TenantFileSystemDirs tenantFileSystemDirs = tenant().getApplicationRepo().getTenantFileSystemDirs(); File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); assertTrue(sessionFile.exists()); @@ -339,7 +333,7 @@ public class ApplicationRepositoryTest { assertNull(applicationRepository.getActiveSession(applicationId())); assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationSet()); assertTrue(provisioner.removed()); - assertEquals(tenant.getName(), provisioner.lastApplicationId().tenant()); + assertEquals(tenant().getName(), provisioner.lastApplicationId().tenant()); assertEquals(applicationId(), provisioner.lastApplicationId()); assertTrue(curator.exists(sessionNode)); assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); @@ -367,10 +361,9 @@ public class ApplicationRepositoryTest { assertTrue(applicationRepository.delete(applicationId())); } - // If delete fails, a retry should work if the failure is transient and zookeeper state should be constistent + // If delete fails, a retry should work if the failure is transient and zookeeper state should be consistent { - PrepareResult result = deployApp(testApp); - long sessionId = result.sessionId(); + long sessionId = deployApp(testApp).sessionId(); assertNotNull(sessionRepository.getRemoteSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); assertEquals(sessionId, applicationRepository.getActiveSession(applicationId()).getSessionId()); @@ -488,8 +481,7 @@ public class ApplicationRepositoryTest { // ... but it should be deleted if some time has passed clock.advance(Duration.ofSeconds(60)); - tester.applicationRepository().deleteExpiredLocalSessions(); - assertEquals(1, sessionRepository.getLocalSessions().size()); + deleteExpiredLocalSessionsAndAssertNumberOfSessions(1, tester, sessionRepository); // Set older created timestamp for session dir for local session without any data in zookeeper, should be deleted setCreatedTime(dir, Instant.now().minus(Duration.ofDays(31))); @@ -564,10 +556,10 @@ public class ApplicationRepositoryTest { long firstSession = result.sessionId(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); - long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); + long sessionId = createSession(applicationId(), timeoutBudget, testAppJdiscOnly); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("tenant:test1 Session 3 is not prepared"); - applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, timeoutBudget, false); + activate(applicationId(), sessionId, timeoutBudget); Session activeSession = applicationRepository.getActiveSession(applicationId()); assertEquals(firstSession, activeSession.getSessionId()); @@ -577,14 +569,13 @@ public class ApplicationRepositoryTest { @Test public void testActivationTimesOut() { // Needed so we can test that the original active session is still active after a failed activation - PrepareResult result = deployApp(testAppJdiscOnly); - long firstSession = result.sessionId(); + long firstSession = deployApp(testAppJdiscOnly).sessionId(); - long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); + long sessionId = createSession(applicationId(), timeoutBudget, testAppJdiscOnly); applicationRepository.prepare(sessionId, prepareParams()); exceptionRule.expect(RuntimeException.class); exceptionRule.expectMessage("Timeout exceeded when trying to activate 'test1.testapp'"); - applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); + activate(applicationId(), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0))); Session activeSession = applicationRepository.getActiveSession(applicationId()); assertEquals(firstSession, activeSession.getSessionId()); @@ -595,18 +586,16 @@ public class ApplicationRepositoryTest { public void testActivationOfSessionCreatedFromNoLongerActiveSessionFails() { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); - PrepareResult result1 = deployApp(testAppJdiscOnly); - result1.sessionId(); + deployApp(testAppJdiscOnly); - long sessionId2 = applicationRepository.createSessionFromExisting(applicationId(), false, timeoutBudget); + long sessionId2 = createSessionFromExisting(applicationId(), timeoutBudget); // Deploy and activate another session - PrepareResult result2 = deployApp(testAppJdiscOnly); - result2.sessionId(); + deployApp(testAppJdiscOnly); applicationRepository.prepare(sessionId2, prepareParams()); exceptionRule.expect(ActivationConflictException.class); exceptionRule.expectMessage("app:test1.testapp.default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)"); - applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId2, timeoutBudget, false); + activate(applicationId(), sessionId2, timeoutBudget); } @Test @@ -620,7 +609,7 @@ public class ApplicationRepositoryTest { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("app:test1.testapp.default Session 2 is already active"); - applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, timeoutBudget, false); + activate(applicationId(), sessionId, timeoutBudget); } @Test @@ -641,9 +630,8 @@ public class ApplicationRepositoryTest { .vespaVersion(vespaVersion) .build()); - RequestHandler requestHandler = getRequestHandler(applicationId()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); - assertEquals(1337 , config.intval()); + SimpletypesConfig config = resolve(applicationId(), vespaVersion); + assertEquals(1337, config.intval()); } @Test @@ -664,18 +652,17 @@ public class ApplicationRepositoryTest { .vespaVersion(vespaVersion) .build()); - RequestHandler requestHandler = getRequestHandler(applicationId()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + SimpletypesConfig config = resolve(applicationId(), vespaVersion); assertEquals(1337, config.intval()); - RequestHandler requestHandler2 = getRequestHandler(appId2); - SimpletypesConfig config2 = resolve(SimpletypesConfig.class, requestHandler2, appId2, vespaVersion); + SimpletypesConfig config2 = resolve(appId2, vespaVersion); assertEquals(1330, config2.intval()); + RequestHandler requestHandler = getRequestHandler(applicationId()); assertTrue(requestHandler.hasApplication(applicationId(), Optional.of(vespaVersion))); assertNull(requestHandler.resolveApplicationId("doesnotexist")); assertEquals(new ApplicationId.Builder().tenant(tenant1).applicationName("testapp").build(), - requestHandler.resolveApplicationId("mytesthost")); // Host set in application package. + requestHandler.resolveApplicationId("mytesthost")); // Host set in application package. } @Test @@ -686,12 +673,11 @@ public class ApplicationRepositoryTest { .vespaVersion(vespaVersion) .build()); - RequestHandler requestHandler = getRequestHandler(applicationId()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + SimpletypesConfig config = resolve(applicationId(), vespaVersion); assertEquals(1337, config.intval()); // TODO: Revisit this test, I cannot see that we create a model for version 3.2.1 - config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), new Version(3, 2, 1)); + config = resolve(applicationId(), new Version(3, 2, 1)); assertEquals(1337, config.intval()); } @@ -703,15 +689,14 @@ public class ApplicationRepositoryTest { .vespaVersion(vespaVersion) .build()); - RequestHandler requestHandler = getRequestHandler(applicationId()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); - assertEquals(1337 , config.intval()); + SimpletypesConfig config = resolve(applicationId(), vespaVersion); + assertEquals(1337, config.intval()); applicationRepository.delete(applicationId()); exceptionRule.expect(com.yahoo.vespa.config.server.NotFoundException.class); exceptionRule.expectMessage("No such application id: test1.testapp"); - resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + resolve(applicationId(), vespaVersion); } private PrepareResult prepareAndActivate(File application) { @@ -730,14 +715,14 @@ public class ApplicationRepositoryTest { return new PrepareParams.Builder().applicationId(applicationId()).build(); } - private ApplicationId applicationId() { - return applicationId(tenant1); - } + private ApplicationId applicationId() { return applicationId(tenant1); } private ApplicationId applicationId(TenantName tenantName) { return ApplicationId.from(tenantName, ApplicationName.from("testapp"), InstanceName.defaultName()); } + private Tenant tenant() { return applicationRepository.getTenant(applicationId()); } + private ApplicationMetaData getApplicationMetaData(ApplicationId applicationId, long sessionId) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); return applicationRepository.getMetadataFromLocalSession(tenant, sessionId); @@ -786,12 +771,11 @@ public class ApplicationRepositoryTest { } - private <T extends ConfigInstance> T resolve(Class<T> clazz, - RequestHandler applications, - ApplicationId appId, - Version vespaVersion) { + private SimpletypesConfig resolve(ApplicationId applicationId, Version vespaVersion) { String configId = ""; - ConfigResponse response = getConfigResponse(clazz, applications, appId, vespaVersion, configId); + RequestHandler requestHandler = getRequestHandler(applicationId); + Class<SimpletypesConfig> clazz = SimpletypesConfig.class; + ConfigResponse response = getConfigResponse(clazz, requestHandler, applicationId, vespaVersion, configId); return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); } @@ -841,4 +825,20 @@ public class ApplicationRepositoryTest { assertEquals(expectedNumberOfSessions, sessionRepository.getLocalSessions().size()); } + private void activate(ApplicationId applicationId, long sessionId, TimeoutBudget timeoutBudget) { + applicationRepository.activate(applicationRepository.getTenant(applicationId), sessionId, timeoutBudget, false); + } + + private TenantMetaData tenantMetaData(Tenant tenant) { + return applicationRepository.getTenantMetaData(tenant); + } + + private long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File app) { + return applicationRepository.createSession(applicationId, timeoutBudget, app); + } + + private long createSessionFromExisting(ApplicationId applicationId, TimeoutBudget timeoutBudget) { + return applicationRepository.createSessionFromExisting(applicationId, false, timeoutBudget); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index babc7b79b65..67613d5a806 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -18,6 +18,7 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.path.Path; +import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.deploy.DeployTester; @@ -42,6 +43,7 @@ import java.util.Optional; import java.util.function.BooleanSupplier; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_FILE; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_PROGRAMMATICALLY; import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; @@ -56,6 +58,7 @@ import static org.junit.Assert.assertTrue; public class ConfigServerBootstrapTest { private final MockCurator curator = new MockCurator(); + private final ManualClock clock = new ManualClock(); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -177,7 +180,7 @@ public class ConfigServerBootstrapTest { waitUntil(() -> bootstrap.vipStatus().isInRotation(), "failed waiting for server to be in rotation"); } - private ConfigServerBootstrap createBootstrap(DeployTester tester, RpcServer rpcServer, ConfigServerBootstrap.VipStatusMode vipStatusProgrammatically) throws IOException { + private ConfigServerBootstrap createBootstrap(DeployTester tester, RpcServer rpcServer, VipStatusMode vipStatusMode) throws IOException { VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); @@ -188,9 +191,10 @@ public class ConfigServerBootstrapTest { versionState, stateMonitor, vipStatus, - vipStatusProgrammatically, + vipStatusMode, new InMemoryFlagSource(), - new ConfigConvergenceChecker()); + new ConfigConvergenceChecker(), + clock); } private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException { @@ -223,7 +227,7 @@ public class ConfigServerBootstrapTest { .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .hostedVespa(hosted) .multitenant(hosted) - .maxDurationOfBootstrap(2) /* seconds */ + .maxDurationOfBootstrap(0) /* seconds, 0 => it will not retry deployment if bootstrap fails */ .sleepTimeWhenRedeployingFails(0)); /* seconds */ } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java new file mode 100644 index 00000000000..b4892caa05f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.config.server; + +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; + +import java.time.Duration; +import java.util.Map; + +public class MockConfigConvergenceChecker extends ConfigConvergenceChecker { + + private final long wantedGeneration; + + public MockConfigConvergenceChecker(long wantedGeneration) { + this.wantedGeneration = wantedGeneration; + } + + @Override + public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) { + return Map.of(); + } + + @Override + public ServiceListResponse checkConvergenceForAllServices(Application application, Duration timeoutPerService) { + return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration); + } + + @Override + public ServiceResponse getServiceConfigGeneration(Application application, String hostAndPortToCheck, Duration timeout) { + return new ServiceResponse(ServiceResponse.Status.ok, wantedGeneration); + } + + @Override + public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application) { + return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index 2c738e2d519..57060e10282 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -13,9 +13,9 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; @@ -35,10 +35,9 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3.createWithParams; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; /** * @author Ulf Lilleengen @@ -67,10 +66,9 @@ public class SuperModelControllerTest { assertEquals(1, lbc.tenants().size()); assertEquals(1, lbc.tenants("a").applications().size()); Applications app = lbc.tenants("a").applications("foo:prod:default:default"); - assertTrue(app.hosts().size() > 0); + assertNotNull(app); } - @Test(expected = UnknownConfigDefinitionException.class) public void test_unknown_config_definition() { PayloadChecksums payloadChecksums = PayloadChecksums.empty(); @@ -107,31 +105,12 @@ public class SuperModelControllerTest { assertEquals(2, lbc.tenants().size()); assertEquals(2, lbc.tenants("t1").applications().size()); assertEquals(1, lbc.tenants("t2").applications().size()); - assertEquals(1, lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default").hosts().size()); - assertQrServer(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default")); } private ApplicationId applicationId(String applicationName, TenantName tenantName) { return ApplicationId.from(tenantName, ApplicationName.from(applicationName), InstanceName.defaultName()); } - private void assertQrServer(Applications app) { - String host = app.hosts().keySet().iterator().next(); - Applications.Hosts hosts = app.hosts(host); - assertEquals(host, hosts.hostname()); - for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { - if (QRSERVER.serviceName.equals(e.getKey())) { - Applications.Hosts.Services s = e.getValue(); - assertEquals("qrserver", s.type()); - assertEquals(4, s.ports().size()); - assertEquals(8000, s.ports().get(0).number()); - assertEquals(0, s.index()); - return; - } - } - org.junit.Assert.fail("No container service in config"); - } - private DeployState createDeployState(File applicationPackage, ApplicationId applicationId) { return new DeployState.Builder() .applicationPackage(FilesApplicationPackage.fromFile(applicationPackage)) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 6afb9ef086d..6016ce991d0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -28,6 +28,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -88,12 +89,12 @@ public class ConfigConvergenceCheckerTest { @Test public void service_list_convergence() { { - URI requestUrl = testServer().resolve("/serviceconverge"); wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}"))); - ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + ServiceListResponse response = checker.checkConvergenceForAllServices(application, clientTimeout); assertEquals(3, response.wantedGeneration); assertEquals(3, response.currentGeneration); + assertTrue(response.converged); List<ServiceListResponse.Service> services = response.services; assertEquals(1, services.size()); assertService(this.service, services.get(0), 3); @@ -113,9 +114,10 @@ public class ConfigConvergenceCheckerTest { URI requestUrl = testServer().resolve("/serviceconverge"); - ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + ServiceListResponse response = checker.checkConvergenceForAllServices(application, clientTimeout); assertEquals(4, response.wantedGeneration); assertEquals(3, response.currentGeneration); + assertFalse(response.converged); List<ServiceListResponse.Service> services = response.services; assertEquals(2, services.size()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 582b7f2c071..ea1f6a845b0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -39,6 +39,7 @@ public class DeployHandlerLoggerTest { assertTrue(Pattern.matches(expectedPattern, baos.toString())); } + @SuppressWarnings("deprecation") private void logMessages(DeployLogger logger) { logger.log(LogLevel.DEBUG, "foobar"); logger.log(LogLevel.SPAM, "foobar"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index f16b1102f2a..7f223fbce6f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -24,6 +24,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; @@ -37,9 +38,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; -import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.nio.file.Files; @@ -253,7 +255,8 @@ public class DeployTester { private Curator curator = new MockCurator(); private Metrics metrics; private List<ModelFactory> modelFactories; - private Orchestrator orchestrator; + private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker(); + private FlagSource flagSource = new InMemoryFlagSource(); public DeployTester build() { Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC); @@ -285,9 +288,11 @@ public class DeployTester { ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withConfigserverConfig(configserverConfig) - .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new)) + .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withProvisioner(provisioner) + .withConfigConvergenceChecker(configConvergenceChecker) + .withFlagSource(flagSource) .build(); return new DeployTester(clock, tenantRepository, applicationRepository); @@ -336,10 +341,16 @@ public class DeployTester { return this; } - public Builder orchestrator(Orchestrator orchestrator) { - this.orchestrator = orchestrator; + public Builder configConvergenceChecker(ConfigConvergenceChecker configConvergenceChecker) { + this.configConvergenceChecker = configConvergenceChecker; return this; } + + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 33e18843738..a6d7cf02bc7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.server.MockConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.ApplicationReindexing; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.InvalidApplicationException; @@ -432,7 +433,14 @@ public class HostedDeployTest { "reindex please", services, "music"), new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); - DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); + DeployTester tester = new DeployTester.Builder() + .modelFactories(modelFactories) + .configserverConfig(createConfigserverConfig(prodZone)) + .clock(clock) + .zone(prodZone) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)) + .configConvergenceChecker(new MockConfigConvergenceChecker(2)) + .build(); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0"); assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); @@ -480,7 +488,9 @@ public class HostedDeployTest { .configserverConfig(createConfigserverConfig(prodZone)) .clock(clock) .zone(prodZone) - .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)).build(); + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)) + .configConvergenceChecker(new MockConfigConvergenceChecker(2)) + .build(); } private static class ConfigChangeActionsModelFactory extends TestModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 8816e695e64..279f3a237e8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -63,7 +63,7 @@ public class HttpGetConfigHandlerTest { .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); - handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); + handler = new HttpGetConfigHandler(HttpGetConfigHandler.testContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java index 8f092ec9d54..40671294b4c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java @@ -45,7 +45,7 @@ public class HttpHandlerTest { private static class HttpTestHandler extends HttpHandler { private final RuntimeException exception; HttpTestHandler(RuntimeException exception) { - super(HttpHandler.testOnlyContext()); + super(HttpHandler.testContext()); this.exception = exception; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 45c74f0d49f..520b4d0edc5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -70,7 +70,7 @@ public class HttpListConfigsHandlerTest { .build(); applicationRepository.deploy(testApp, prepareParams()); - HttpListConfigsHandler.Context ctx = HttpListConfigsHandler.testOnlyContext(); + HttpListConfigsHandler.Context ctx = HttpListConfigsHandler.testContext(); handler = new HttpListConfigsHandler(ctx, tenantRepository); namedHandler = new HttpListNamedConfigsHandler(ctx, tenantRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java index 18fa307a9d3..ed739d73860 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java @@ -29,7 +29,7 @@ public class StatusHandlerTest { public void require_that_handler_works() throws IOException { ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(List.of(new VespaModelFactory(new NullConfigModelRegistry()))); ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder().build(); - StatusHandler handler = new StatusHandler(StatusHandler.testOnlyContext(), modelFactoryRegistry, configserverConfig); + StatusHandler handler = new StatusHandler(StatusHandler.testContext(), modelFactoryRegistry, configserverConfig); HttpResponse response = handler.handle(HttpRequest.createTestRequest("/status", GET)); JsonNode jsonNode = mapper.readTree(SessionHandlerTest.getRenderedString(response)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java index e2b45d33cbc..f389829a160 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java @@ -62,6 +62,22 @@ public class RoutingStatusApiHandlerTest { } @Test + public void get_deployment_status_v2() { + String response = responseAsString(executeRequest(Method.GET, "/routing/v2/status/", null)); + assertEquals("{\"inactiveDeployments\":[],\"zoneActive\":true}", response); + + // Set deployment out + executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), statusOut()); + response = responseAsString(executeRequest(Method.GET, "/routing/v2/status/", null)); + assertEquals("{\"inactiveDeployments\":[{\"upstreamName\":\"test-upstream-name\"}],\"zoneActive\":true}", response); + + // Set zone out + executeRequest(Method.PUT, "/routing/v1/status/zone", null); + response = responseAsString(executeRequest(Method.GET, "/routing/v2/status/", null)); + assertEquals("{\"inactiveDeployments\":[{\"upstreamName\":\"test-upstream-name\"}],\"zoneActive\":false}", response); + } + + @Test public void set_deployment_status() { String response = responseAsString(executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), statusOut())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 322a2b924bd..1d891c0547c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -69,7 +69,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { .build(); applicationRepository.deploy(testApp, prepareParams(appId1)); - handler = new ApplicationHandler(ApplicationHandler.testOnlyContext(), + handler = new ApplicationHandler(ApplicationHandler.testContext(), Zone.defaultZone(), applicationRepository); pathPrefix = createPath(appId1, Zone.defaultZone()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 04483e0191d..005dd715dd4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -556,8 +556,8 @@ public class ApplicationHandlerTest { { // Known service HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.ok, - 3L, - 3L, + 3, + 3, true), hostAndPort, uri); @@ -601,9 +601,9 @@ public class ApplicationHandlerTest { { HttpServiceListResponse response = new HttpServiceListResponse(new ServiceListResponse(Map.of(createServiceInfo(hostname, port), 3L), - requestUrl, 3L, - 3L)); + 3L), + requestUrl); assertResponse("{\n" + " \"services\": [\n" + " {\n" + @@ -635,9 +635,9 @@ public class ApplicationHandlerTest { HttpServiceListResponse response = new HttpServiceListResponse(new ServiceListResponse(serviceInfos, - requestUrl, 4L, - 3L)); + 3L), + requestUrl); assertResponse("{\n" + " \"services\": [\n" + " {\n" + @@ -756,7 +756,7 @@ public class ApplicationHandlerTest { "/environment/" + zone.environment().value() + "/region/" + zone.region().value() + "/instance/" + applicationId.instance().value() + "\"]"; - ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), + ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testContext(), tenantRepository, Zone.defaultZone()); ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, @@ -808,7 +808,7 @@ public class ApplicationHandlerTest { } private ApplicationHandler createApplicationHandler(ApplicationRepository applicationRepository) { - return new ApplicationHandler(ApplicationHandler.testOnlyContext(), Zone.defaultZone(), applicationRepository); + return new ApplicationHandler(ApplicationHandler.testContext(), Zone.defaultZone(), applicationRepository); } private PrepareParams prepareParams(ApplicationId applicationId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 6bbbc451094..fbc5e87c329 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -62,7 +62,7 @@ public class HostHandlerTest { .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); - handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); + handler = new HostHandler(HostHandler.testContext(), applicationRepository); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 3d207f9f64a..9aae64cb884 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -72,7 +72,7 @@ public class HttpGetConfigHandlerTest { .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); - handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); + handler = new HttpGetConfigHandler(HttpGetConfigHandler.testContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 141f8a52f13..2ee1064f614 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -76,10 +76,10 @@ public class HttpListConfigsHandlerTest { .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); - handler = new HttpListConfigsHandler(HttpListConfigsHandler.testOnlyContext(), + handler = new HttpListConfigsHandler(HttpListConfigsHandler.testContext(), tenantRepository, Zone.defaultZone()); - namedHandler = new HttpListNamedConfigsHandler(HttpListConfigsHandler.testOnlyContext(), + namedHandler = new HttpListNamedConfigsHandler(HttpListConfigsHandler.testContext(), tenantRepository, Zone.defaultZone()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index e2ad65786a4..76790e6264d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -55,7 +55,7 @@ public class ListApplicationsHandlerTest { tenantRepository.addTenant(foobar); applicationRepo = tenantRepository.getTenant(mytenant).getApplicationRepo(); applicationRepo2 = tenantRepository.getTenant(foobar).getApplicationRepo(); - handler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), + handler = new ListApplicationsHandler(ListApplicationsHandler.testContext(), tenantRepository, new Zone(Environment.dev, RegionName.from("us-east"))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index c6f84c2d6ae..ecb8d7603f6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -164,7 +164,7 @@ public class SessionActiveHandlerTest { } private SessionActiveHandler createHandler() { - return new SessionActiveHandler(SessionActiveHandler.testOnlyContext(), + return new SessionActiveHandler(SessionActiveHandler.testContext(), applicationRepository, Zone.defaultZone()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index f13ec6cd5c8..ec96be0d0c8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -183,7 +183,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private SessionContentHandler createHandler() { return new SessionContentHandler( - SessionContentHandler.testOnlyContext(), + SessionContentHandler.testContext(), new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new MockProvisioner()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 4cb69ce5d20..702dd2792da 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -195,7 +195,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { } private SessionCreateHandler createHandler() { - return new SessionCreateHandler(SessionCreateHandler.testOnlyContext(), + return new SessionCreateHandler(SessionCreateHandler.testContext(), applicationRepository, configserverConfig); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 3c762b7c2e5..3d2e108a75e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationName; @@ -247,7 +247,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void test_out_of_capacity_response() throws IOException { long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); String exceptionMessage = "Out of capacity"; - FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testContext(), applicationRepository, configserverConfig, new OutOfCapacityException(exceptionMessage)); @@ -262,7 +262,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void test_that_nullpointerexception_gives_internal_server_error() throws IOException { long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); String exceptionMessage = "nullpointer thrown in test handler"; - FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testContext(), applicationRepository, configserverConfig, new NullPointerException(exceptionMessage)); @@ -277,7 +277,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void test_application_lock_failure() throws IOException { String exceptionMessage = "Timed out after waiting PT1M to acquire lock '/provision/v1/locks/foo/bar/default'"; long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); - FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testContext(), applicationRepository, configserverConfig, new ApplicationLockException(new UncheckedTimeoutException(exceptionMessage))); @@ -314,7 +314,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { } private SessionHandler createHandler() { - return new SessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), applicationRepository, configserverConfig); + return new SessionPrepareHandler(SessionPrepareHandler.testContext(), applicationRepository, configserverConfig); } private HttpResponse request(HttpRequest.Method put, long l) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 402e2576591..ee35ca572e1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; @@ -43,7 +42,6 @@ import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpo import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.application; import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.global; import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.zone; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -56,8 +54,6 @@ import static org.junit.Assume.assumeFalse; @RunWith(Parameterized.class) public class LbServicesProducerTest { - private static final String rotation1 = "rotation-1"; - private static final String rotation2 = "rotation-2"; private static final Set<ContainerEndpoint> endpoints = Set.of( new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) @@ -91,20 +87,6 @@ public class LbServicesProducerTest { } @Test - public void testConfigAliases() { - Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); - LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); - LbServicesConfig.Tenants.Applications.Hosts.Services services = - conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); - assertEquals(1, services.servicealiases().size()); - assertEquals(2, services.endpointaliases().size()); - - assertEquals("service1", services.servicealiases(0)); - assertEquals("foo1.bar1.com", services.endpointaliases(0)); - assertEquals("foo2.bar2.com", services.endpointaliases(1)); - } - - @Test public void testConfigActiveRotation() { { RegionName regionName = RegionName.from("us-east-1"); @@ -119,18 +101,6 @@ public class LbServicesProducerTest { } } - @Test - public void generate_non_mtls_endpoints_from_feature_flag() { - RegionName regionName = RegionName.from("us-east-1"); - - LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); - assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").generateNonMtlsEndpoint()); - - flagSource.withBooleanFlag(Flags.GENERATE_NON_MTLS_ENDPOINT.id(), false); - conf = createModelAndGetLbServicesConfig(regionName); - assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").generateNonMtlsEndpoint()); - } - private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) { Zone zone = new Zone(Environment.prod, regionName); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().zone(zone)); @@ -153,20 +123,12 @@ public class LbServicesProducerTest { .properties(new TestProperties().setHostedVespa(true))); RegionName regionName = RegionName.from("us-east-1"); LbServicesConfig config = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); - LbServicesConfig.Tenants.Applications.Hosts.Services services = config - .tenants("foo") - .applications("foo:prod:" + regionName.value() + ":default") - .hosts("foo.foo.yahoo.com") - .services(QRSERVER.serviceName); - - assertTrue(services.servicealiases().contains("service1")); - assertTrue("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases().containsAll(List.of("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2))); List<Endpoints> endpointList = config.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").endpoints(); - // Expect 4 zone endpoints (2 suffixes), 2 global endpoints and 1 application endpoint - assertEquals(7, endpointList.size()); + // Expect 2 zone endpoints (2 suffixes), 2 global endpoints and 1 application endpoint + assertEquals(5, endpointList.size()); List<Endpoints> zoneEndpoints = endpointList.stream().filter(e -> e.scope() == zone).collect(Collectors.toList()); - assertEquals(4, zoneEndpoints.size()); + assertEquals(2, zoneEndpoints.size()); assertTrue(zoneEndpoints.stream() .filter(e -> e.routingMethod() == sharedLayer4) .map(Endpoints::dnsName).collect(Collectors.toList()) @@ -186,16 +148,11 @@ public class LbServicesProducerTest { assertContainsEndpoint(applicationEndpoints, "app-endpoint", "mydisc", application, sharedLayer4, 1, List.of("foo.foo.yahoo.com")); } - @Test public void testRoutingConfigForTesterApplication() { assumeFalse(useGlobalServiceId); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); - LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); - LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); - assertEquals(1, services.servicealiases().size()); - assertEquals(2, services.endpointaliases().size()); // No config for tester application assertNull(getLbServicesConfig(Zone.defaultZone(), testModel) @@ -231,7 +188,7 @@ public class LbServicesProducerTest { TenantName baz = TenantName.from("baz"); tMap.put(foo, createTestApplications(foo, deployStateBuilder)); tMap.put(bar, createTestApplications(bar, deployStateBuilder)); - tMap.put(bar, createTestApplications(baz, deployStateBuilder)); + tMap.put(baz, createTestApplications(baz, deployStateBuilder)); return tMap; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java index 854abee91f5..30d3dcffc30 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java @@ -116,7 +116,7 @@ public class MultiTenantRpcAuthorizerTest { new ConfigKey<>(LbServicesConfig.CONFIG_DEF_NAME, "*", LbServicesConfig.CONFIG_DEF_NAMESPACE), HOSTNAME); - exceptionRule.expectMessage("Node with type 'tenant' is not allowed to access global config [name=lb-services,namespace=cloud.config,configId=*]"); + exceptionRule.expectMessage("Node with type 'tenant' is not allowed to access global config [name=cloud.config.lb-services,configId=*]"); exceptionRule.expectCause(instanceOf(AuthorizationException.class)); authorizer.authorizeConfigRequest(configRequest) diff --git a/configutil/src/apps/configstatus/main.cpp b/configutil/src/apps/configstatus/main.cpp index 717c120f8e2..cd0424633d9 100644 --- a/configutil/src/apps/configstatus/main.cpp +++ b/configutil/src/apps/configstatus/main.cpp @@ -2,10 +2,10 @@ #include <vespa/defaults.h> #include <vespa/vespalib/text/stringtokenizer.h> -#include <iostream> -#include <cstdlib> #include "lib/configstatus.h" +#include <vespa/config/subscription/sourcespec.h> #include <vespa/fastos/app.h> +#include <iostream> #include <vespa/log/log.h> LOG_SETUP("vespa-config-status"); diff --git a/configutil/src/apps/modelinspect/main.cpp b/configutil/src/apps/modelinspect/main.cpp index c9fc51febb5..7053adf17fa 100644 --- a/configutil/src/apps/modelinspect/main.cpp +++ b/configutil/src/apps/modelinspect/main.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/defaults.h> -#include <iostream> -#include <cstdlib> #include "lib/modelinspect.h" #include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/config/subscription/sourcespec.h> #include <vespa/fastos/app.h> +#include <iostream> #include <vespa/log/log.h> LOG_SETUP("vespa-model-inspect"); diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index 529765dccd5..841d1604866 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -7,8 +7,8 @@ #include <vbench/http/server_spec.h> #include <vbench/http/http_client.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <iostream> -#include <cstdlib> using configdefinitions::tagsContain; @@ -69,7 +69,7 @@ struct ComponentTraverser : public vespalib::slime::ObjectTraverser } }; -ComponentTraverser::~ComponentTraverser() {} +ComponentTraverser::~ComponentTraverser() = default; class MyHttpHandler : public vbench::HttpResultHandler { private: diff --git a/configutil/src/lib/configstatus.h b/configutil/src/lib/configstatus.h index 9e1627af4a9..4d792748419 100644 --- a/configutil/src/lib/configstatus.h +++ b/configutil/src/lib/configstatus.h @@ -4,7 +4,7 @@ #include "hostfilter.h" #include <vespa/config-model.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configuri.h> class ConfigStatus { diff --git a/configutil/src/lib/modelinspect.cpp b/configutil/src/lib/modelinspect.cpp index 611a41d1a84..d840a7d45ca 100644 --- a/configutil/src/lib/modelinspect.cpp +++ b/configutil/src/lib/modelinspect.cpp @@ -7,6 +7,7 @@ #include <iostream> #include <algorithm> #include <cstdlib> +#include <set> using configdefinitions::tagsContain; using configdefinitions::upcase; @@ -32,7 +33,7 @@ ModelInspect::ModelInspect(Flags flags, const config::ConfigUri uri, std::ostrea } catch (config::ConfigRuntimeException &e) { std::cerr << e.getMessage() << "\n"; } - if (_cfg.get() != NULL) { + if (_cfg) { if (_flags.verbose) std::cerr << "success!\n"; } else { std::cerr << "FATAL ERROR: failed to get model configuration.\n"; @@ -40,9 +41,7 @@ ModelInspect::ModelInspect(Flags flags, const config::ConfigUri uri, std::ostrea } } -ModelInspect::~ModelInspect() -{ -} +ModelInspect::~ModelInspect() = default; void ModelInspect::printPort(const vespalib::string &host, int port, diff --git a/configutil/src/lib/modelinspect.h b/configutil/src/lib/modelinspect.h index dae5f3f4e92..33ef94b259b 100644 --- a/configutil/src/lib/modelinspect.h +++ b/configutil/src/lib/modelinspect.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/config-model.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configuri.h> class ModelInspect diff --git a/configutil/src/tests/config_status/config_status_test.cpp b/configutil/src/tests/config_status/config_status_test.cpp index 9b8b96acdb8..b6f1fd44d3f 100644 --- a/configutil/src/tests/config_status/config_status_test.cpp +++ b/configutil/src/tests/config_status/config_status_test.cpp @@ -3,7 +3,6 @@ #include <lib/configstatus.h> #include <vespa/vespalib/portal/portal.h> #include <vespa/config-model.h> -#include <vespa/config/config.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/config/common/configcontext.h> diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 173695c5299..decb0513e26 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -71,7 +71,8 @@ public class HandlersConfigurerDi { this.vespaContainer = vespaContainer; container = new Container(subscriberFactory, configId, deconstructor, osgiWrapper); - getNewComponentGraph(discInjector, true); + Runnable cleanupTask = waitForNextGraphGeneration(discInjector, true); + cleanupTask.run(); } private static class ContainerAndDiOsgi extends OsgiImpl implements OsgiWrapper { @@ -106,11 +107,15 @@ public class HandlersConfigurerDi { /** * Wait for new config to arrive and produce the new graph + * @return Task for deconstructing previous component graph and bundles */ - public void getNewComponentGraph(Injector discInjector, boolean isInitializing) { - currentGraph = container.getNewComponentGraph(currentGraph, - createFallbackInjector(vespaContainer, discInjector), - isInitializing); + public Runnable waitForNextGraphGeneration(Injector discInjector, boolean isInitializing) { + Container.ComponentGraphResult result = container.waitForNextGraphGeneration( + this.currentGraph, + createFallbackInjector(vespaContainer, discInjector), + isInitializing); + this.currentGraph = result.newGraph(); + return result.oldComponentsCleanupTask(); } @SuppressWarnings("deprecation") @@ -138,9 +143,9 @@ public class HandlersConfigurerDi { return currentGraph.getInstance(componentClass); } - public void shutdown(ComponentDeconstructor deconstructor) { - container.shutdown(currentGraph, deconstructor); - } + public void shutdown() { container.shutdown(currentGraph); } + + public void shutdownConfigRetriever() { container.shutdownConfigRetriever(); } /** Returns the currently active application configuration generation */ public long generation() { return currentGraph.generation(); } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 3ae28c2816d..729e1ca1f2e 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -39,6 +39,7 @@ import java.util.concurrent.Executors; */ public class HandlersConfigurerTestWrapper { + @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSourceSet configSources = new ConfigSourceSet(this.getClass().getSimpleName() + ": " + new Random().nextLong()); private final HandlersConfigurerDi configurer; @@ -109,7 +110,7 @@ public class HandlersConfigurerTestWrapper { } private ComponentDeconstructor getTestDeconstructor() { - return (components, bundles) -> components.forEach(component -> { + return (generation, components, bundles) -> components.forEach(component -> { if (component instanceof AbstractComponent) { AbstractComponent abstractComponent = (AbstractComponent) component; if (abstractComponent.isDeconstructable()) abstractComponent.deconstruct(); @@ -120,11 +121,12 @@ public class HandlersConfigurerTestWrapper { public void reloadConfig() { configurer.reloadConfig(++lastGeneration); - configurer.getNewComponentGraph(guiceInjector(), false); + Runnable cleanupTask = configurer.waitForNextGraphGeneration(guiceInjector(), false); + cleanupTask.run(); } public void shutdown() { - configurer.shutdown(getTestDeconstructor()); + configurer.shutdown(); // TODO: Remove once tests use ConfigSet rather than dir: for (File f : createdFiles) { f.delete(); diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index d1f0bd76ad9..44a70ea2f3b 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -2,12 +2,10 @@ package com.yahoo.container.core.documentapi; import com.google.inject.Inject; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -16,7 +14,7 @@ import com.yahoo.vespa.config.content.LoadTypeConfig; * * @author jonmv */ -public class DocumentAccessProvider extends AbstractComponent implements Provider<VespaDocumentAccess> { +public class DocumentAccessProvider implements Provider<VespaDocumentAccess> { private final VespaDocumentAccess access; @@ -35,7 +33,7 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide @Override public void deconstruct() { - access.shutdown(); + access.protectedShutdown(); } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index 87a5a90dde3..6976299cc7d 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.documentapi; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AsyncParameters; @@ -19,12 +18,14 @@ import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Wraps a lazily initialised {@link DocumentAccess}. Lazy to allow it to always be set up. @@ -34,6 +35,8 @@ import java.util.concurrent.atomic.AtomicReference; */ public class VespaDocumentAccess extends DocumentAccess { + private static final Logger log = Logger.getLogger(VespaDocumentAccess.class.getName()); + private final MessageBusParams parameters; private final AtomicReference<DocumentAccess> delegate = new AtomicReference<>(); @@ -68,6 +71,10 @@ public class VespaDocumentAccess extends DocumentAccess { @Override public void shutdown() { + log.log(Level.WARNING, "This injected document access should only be shut down by the container", new IllegalStateException()); + } + + void protectedShutdown() { delegate.updateAndGet(access -> { super.shutdown(); shutDown = true; diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java index 0247cda8bbd..2acbec2e388 100644 --- a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java @@ -21,6 +21,7 @@ import static java.util.logging.Level.FINE; * @author Tony Vaagenes * @author ollivir */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class CloudSubscriber implements Subscriber { private static final Logger log = Logger.getLogger(CloudSubscriber.class.getName()); diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java index a6327b01e21..a58baa5b2bd 100644 --- a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java @@ -20,6 +20,7 @@ import java.util.WeakHashMap; * @author Tony Vaagenes * @author ollivir */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class CloudSubscriberFactory implements SubscriberFactory { private final ConfigSource configSource; diff --git a/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java b/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java index bc5ddb3fa7a..0df91103985 100644 --- a/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java +++ b/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java @@ -13,6 +13,9 @@ import java.util.List; public interface ComponentDeconstructor { /** Deconstructs the given components in order, then the given bundles. */ - void deconstruct(List<Object> components, Collection<Bundle> bundles); + void deconstruct(long generation, List<Object> components, Collection<Bundle> bundles); + + /** Wait for all previous destruction tasks to complete */ + default void shutdown() {} } diff --git a/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java index c90f05e2227..0817a87c506 100644 --- a/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -70,12 +70,6 @@ public final class ConfigRetriever { allKeys.addAll(bootstrapKeys); setupComponentSubscriber(allKeys); - var maybeSnapshot = getConfigsOptional(leastGeneration, isInitializing); - log.log(FINE, () -> "getConfigsOnce returning " + maybeSnapshot); - return maybeSnapshot; - } - - private Optional<ConfigSnapshot> getConfigsOptional(long leastGeneration, boolean isInitializing) { if (componentSubscriber.generation() < bootstrapSubscriber.generation()) { return getComponentsSnapshot(leastGeneration, isInitializing); } @@ -87,7 +81,21 @@ public final class ConfigRetriever { if (newestBootstrapGeneration < leastGeneration) { return Optional.empty(); } - return bootstrapConfigIfChanged(); + return bootstrapConfigIfChanged() + // At this point, we normally assume that the bootstrap subscriber is one generation ahead + // of the component subscriber, but this is not always the case in practice. + .or(this::componentsSnapshotReceivedBeforeBootstrap); + } + + private Optional<ConfigSnapshot> componentsSnapshotReceivedBeforeBootstrap() { + if (componentSubscriber.generation() == bootstrapSubscriber.generation()) { + // The component subscriber originally had a newer config generation than the bootstrap subscriber. + // Ensure that this generation is applied if it contains changed configs. + // The root cause is probably that the component subscriber skipped a generation and got + // one ahead of the bootstrap subscriber. + return componentsConfigIfChanged(); + } + return Optional.empty(); } private Optional<ConfigSnapshot> getComponentsSnapshot(long leastGeneration, boolean isInitializing) { diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java index e437f440c41..c30b0bfaa80 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Container.java +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -5,6 +5,7 @@ import com.google.inject.Injector; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.SubscriberClosedException; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.ConfigRetriever.BootstrapConfigs; @@ -17,6 +18,7 @@ import com.yahoo.container.di.config.ApplicationBundlesConfig; import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.yolean.UncheckedInterruptedException; import org.osgi.framework.Bundle; import java.util.ArrayList; @@ -45,7 +47,7 @@ public class Container { private final ConfigKey<ApplicationBundlesConfig> applicationBundlesConfigKey; private final ConfigKey<PlatformBundlesConfig> platformBundlesConfigKey; private final ConfigKey<ComponentsConfig> componentsConfigKey; - private final ComponentDeconstructor componentDeconstructor; + private final ComponentDeconstructor destructor; private final Osgi osgi; private final ConfigRetriever retriever; @@ -53,9 +55,9 @@ public class Container { private long previousConfigGeneration = -1L; private long leastGeneration = -1L; - public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor componentDeconstructor, Osgi osgi) { + public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor destructor, Osgi osgi) { this.subscriberFactory = subscriberFactory; - this.componentDeconstructor = componentDeconstructor; + this.destructor = destructor; this.osgi = osgi; applicationBundlesConfigKey = new ConfigKey<>(ApplicationBundlesConfig.class, configId); @@ -65,29 +67,35 @@ public class Container { this.retriever = new ConfigRetriever(bootstrapKeys, subscriberFactory); } - public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor componentDeconstructor) { - this(subscriberFactory, configId, componentDeconstructor, new Osgi() { + public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor destructor) { + this(subscriberFactory, configId, destructor, new Osgi() { }); } - public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph, Injector fallbackInjector, boolean isInitializing) { + public ComponentGraphResult waitForNextGraphGeneration(ComponentGraph oldGraph, Injector fallbackInjector, boolean isInitializing) { try { Collection<Bundle> obsoleteBundles = new HashSet<>(); - ComponentGraph newGraph = getConfigAndCreateGraph(oldGraph, fallbackInjector, isInitializing, obsoleteBundles); + ComponentGraph newGraph = waitForNewConfigGenAndCreateGraph(oldGraph, fallbackInjector, isInitializing, obsoleteBundles); newGraph.reuseNodes(oldGraph); - constructComponents(newGraph); - deconstructObsoleteComponents(oldGraph, newGraph, obsoleteBundles); - return newGraph; + try { + constructComponents(newGraph); + } catch (Exception e) { + log.log(Level.WARNING, String.format( + "Failed to construct graph for generation '%d' - scheduling partial graph for deconstruction", + newGraph.generation()), e); + deconstructFailedGraph(oldGraph, newGraph); + throw e; + } + Runnable cleanupTask = createPreviousGraphDeconstructionTask(oldGraph, newGraph, obsoleteBundles); + return new ComponentGraphResult(newGraph, cleanupTask); } catch (Throwable t) { invalidateGeneration(oldGraph.generation(), t); throw t; } } - private ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, - Injector fallbackInjector, - boolean isInitializing, - Collection<Bundle> obsoleteBundles) // NOTE: Return value + private ComponentGraph waitForNewConfigGenAndCreateGraph( + ComponentGraph graph, Injector fallbackInjector, boolean isInitializing, Collection<Bundle> obsoleteBundles) // NOTE: Return value { ConfigSnapshot snapshot; while (true) { @@ -153,12 +161,27 @@ public class Container { } private void constructComponents(ComponentGraph graph) { - graph.nodes().forEach(Node::constructInstance); + graph.nodes().forEach(n -> { + if (Thread.interrupted()) + throw new UncheckedInterruptedException("Interrupted while constructing component graph", true); + n.constructInstance(); + }); + } + + private void deconstructFailedGraph(ComponentGraph currentGraph, ComponentGraph failedGraph) { + Set<Object> currentComponents = Collections.newSetFromMap(new IdentityHashMap<>(currentGraph.size())); + currentComponents.addAll(currentGraph.allConstructedComponentsAndProviders()); + + List<Object> unusedComponents = new ArrayList<>(); + for (Object component : failedGraph.allConstructedComponentsAndProviders()) { + if (!currentComponents.contains(component)) unusedComponents.add(component); + } + destructor.deconstruct(failedGraph.generation(), unusedComponents, List.of()); } - private void deconstructObsoleteComponents(ComponentGraph oldGraph, - ComponentGraph newGraph, - Collection<Bundle> obsoleteBundles) { + private Runnable createPreviousGraphDeconstructionTask(ComponentGraph oldGraph, + ComponentGraph newGraph, + Collection<Bundle> obsoleteBundles) { Map<Object, ?> newComponents = new IdentityHashMap<>(newGraph.size()); for (Object component : newGraph.allConstructedComponentsAndProviders()) newComponents.put(component, null); @@ -168,7 +191,7 @@ public class Container { if ( ! newComponents.containsKey(component)) obsoleteComponents.add(component); - componentDeconstructor.deconstruct(obsoleteComponents, obsoleteBundles); + return () -> destructor.deconstruct(oldGraph.generation(), obsoleteComponents, obsoleteBundles); } private Set<Bundle> installApplicationBundles(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs) { @@ -214,9 +237,10 @@ public class Container { } } + @SuppressWarnings("removal") // TODO Vespa 8: remove private void invalidateGeneration(long generation, Throwable cause) { leastGeneration = Math.max(retriever.getComponentsGeneration(), retriever.getBootstrapGeneration()) + 1; - if (!(cause instanceof InterruptedException) && !(cause instanceof ConfigInterruptedException)) { + if (!(cause instanceof InterruptedException) && !(cause instanceof ConfigInterruptedException) && !(cause instanceof SubscriberClosedException)) { log.log(Level.WARNING, newGraphErrorMessage(generation, cause), cause); } } @@ -242,14 +266,15 @@ public class Container { } } - public void shutdown(ComponentGraph graph, ComponentDeconstructor deconstructor) { - shutdownConfigurer(); + public void shutdown(ComponentGraph graph) { + shutdownConfigRetriever(); if (graph != null) { - deconstructAllComponents(graph, deconstructor); + scheduleGraphForDeconstruction(graph); + destructor.shutdown(); } } - void shutdownConfigurer() { + public void shutdownConfigRetriever() { retriever.shutdown(); } @@ -258,9 +283,9 @@ public class Container { subscriberFactory.reloadActiveSubscribers(generation); } - private void deconstructAllComponents(ComponentGraph graph, ComponentDeconstructor deconstructor) { - // This is only used for shutdown, so no need to uninstall any bundles. - deconstructor.deconstruct(graph.allConstructedComponentsAndProviders(), Collections.emptyList()); + private void scheduleGraphForDeconstruction(ComponentGraph graph) { + // This is only used for shutdown and cleanup of failed graph, so no need to uninstall any bundles. + destructor.deconstruct(graph.generation(), graph.allConstructedComponentsAndProviders(), List.of()); } public static <T extends ConfigInstance> T getConfig(ConfigKey<T> key, @@ -278,4 +303,17 @@ public class Container { return BundleInstantiationSpecification.getFromStrings(config.id(), config.classId(), config.bundle()); } + public static class ComponentGraphResult { + private final ComponentGraph newGraph; + private final Runnable oldComponentsCleanupTask; + + public ComponentGraphResult(ComponentGraph newGraph, Runnable oldComponentsCleanupTask) { + this.newGraph = newGraph; + this.oldComponentsCleanupTask = oldComponentsCleanupTask; + } + + public ComponentGraph newGraph() { return newGraph; } + public Runnable oldComponentsCleanupTask() { return oldComponentsCleanupTask; } + } + } diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index 86e6bc4fa4a..8a07ef0ae14 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -169,7 +169,10 @@ public class ComponentGraph { public List<Object> allConstructedComponentsAndProviders() { List<Node> orderedNodes = topologicalSort(nodes()); Collections.reverse(orderedNodes); - return orderedNodes.stream().map(node -> node.constructedInstance().get()).collect(Collectors.toList()); + return orderedNodes.stream() + .filter(node -> node.constructedInstance().isPresent()) + .map(node -> node.constructedInstance().orElseThrow()) + .collect(Collectors.toList()); } private void completeComponentRegistryNode(ComponentRegistryNode registry) { diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java index 3f37ffb7a83..4926f0e7dd5 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java @@ -22,7 +22,7 @@ import java.util.Set; public abstract class Node { private final ComponentId componentId; - protected Optional<Object> instance = Optional.empty(); + protected volatile Optional<Object> instance = Optional.empty(); List<Node> componentsToInject = new ArrayList<>(); public Node(ComponentId componentId) { diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java index 54a4d05853c..71e5e8db3e5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java @@ -11,6 +11,7 @@ import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.Path; import java.net.URI; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; @@ -28,14 +29,25 @@ import static java.util.logging.Level.WARNING; public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler { private static final ObjectMapper jsonMapper = new ObjectMapper(); + private final Duration defaultTimeout; protected HttpHandlerBase(Executor executor) { + this(executor, Duration.ofSeconds(25)); + } + + protected HttpHandlerBase(Executor executor, Duration defaultTimeout) { super(executor); + this.defaultTimeout = defaultTimeout; } protected abstract Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer); @Override + public Duration getTimeout() { + return defaultTimeout; + } + + @Override public final HttpResponse handle(HttpRequest request) { if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java index b8175802ff7..805da3a0a0f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java @@ -4,7 +4,7 @@ package com.yahoo.container.handler.test; import com.yahoo.api.annotations.Beta; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.jdisc.Metric; @@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit; * @author Ulf Lilleengen */ @Beta -public class MockService extends LoggingRequestHandler { +public class MockService extends ThreadedHttpRequestHandler { private final MockServiceHandler handler; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java index 6f5cc5722c8..927284f2c66 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java @@ -552,11 +552,6 @@ public class HttpRequest { } @Override - public <T> T getInstance(Key<T> tKey) { - return null; - } - - @Override public <T> T getInstance(Class<T> tClass) { return null; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index b6e370dd911..c6735559429 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -25,8 +25,11 @@ import java.util.logging.Level; * ThreadedHttpRequestHandler with access logging. * * @author Steinar Knutsen + * + * @deprecated Use {@link ThreadedHttpRequestHandler}, which provides the same level of functionality. */ // TODO Vespa 8: Remove deprecated constructors +@Deprecated public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { // TODO: Deprecate @@ -58,7 +61,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } - // TODO: Deprecate public static Context testOnlyContext() { return new Context(new Executor() { @Override diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 722f1850013..8986fa596b8 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -3,6 +3,7 @@ package com.yahoo.container.jdisc; import com.google.inject.Inject; import com.yahoo.container.logging.AccessLog; +import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.BufferedContentChannel; @@ -10,7 +11,10 @@ import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.UnsafeContentInputStream; import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; +import com.yahoo.yolean.Exceptions; +import java.util.Optional; import java.util.logging.Level; import java.io.IOException; @@ -244,7 +248,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler HttpResponse response, HttpRequest httpRequest, ContentChannelOutputStream rendererWiring) { - return null; + return new ResponseLogger(httpRequest, response); } protected com.yahoo.jdisc.http.HttpRequest asHttpRequest(Request request) { @@ -287,4 +291,39 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler } + + private class ResponseLogger implements LoggingCompletionHandler { + + private final com.yahoo.jdisc.http.HttpRequest jdiscRequest; + private final HttpResponse httpResponse; + + ResponseLogger(HttpRequest httpRequest, HttpResponse httpResponse) { + this.jdiscRequest = httpRequest.getJDiscRequest(); + this.httpResponse = httpResponse; + } + + @Override + public void markCommitStart() { } + + @Override + public void completed() { + writeToAccessLog(); + } + + @Override + public void failed(Throwable throwable) { + writeToAccessLog(); + log.log(Level.FINE, () -> "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); + } + + private void writeToAccessLog() { + Optional<AccessLogEntry> jdiscReqAccessLogEntry = AccessLoggingRequestHandler.getAccessLogEntry(jdiscRequest); + AccessLogEntry entry; + if (jdiscReqAccessLogEntry.isPresent()) { + entry = jdiscReqAccessLogEntry.get(); + httpResponse.populateAccessLogEntry(entry); + } + } + } + } diff --git a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java index 26cdd1597c5..e2eeb5d3517 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java +++ b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java @@ -31,6 +31,8 @@ public class ConnectionLogEntry { private final String sslPeerSubject; private final Instant sslPeerNotBefore; private final Instant sslPeerNotAfter; + private final String sslPeerIssuerSubject; + private final String sslPeerFingerprint; private final String sslSniServerName; private final SslHandshakeFailure sslHandshakeFailure; private final List<String> sslSubjectAlternativeNames; @@ -58,6 +60,8 @@ public class ConnectionLogEntry { this.sslPeerSubject = builder.sslPeerSubject; this.sslPeerNotBefore = builder.sslPeerNotBefore; this.sslPeerNotAfter = builder.sslPeerNotAfter; + this.sslPeerIssuerSubject = builder.sslPeerIssuerSubject; + this.sslPeerFingerprint = builder.sslPeerFingerprint; this.sslSniServerName = builder.sslSniServerName; this.sslHandshakeFailure = builder.sslHandshakeFailure; this.sslSubjectAlternativeNames = builder.sslSubjectAlternativeNames; @@ -88,6 +92,8 @@ public class ConnectionLogEntry { public Optional<String> sslPeerSubject() { return Optional.ofNullable(sslPeerSubject); } public Optional<Instant> sslPeerNotBefore() { return Optional.ofNullable(sslPeerNotBefore); } public Optional<Instant> sslPeerNotAfter() { return Optional.ofNullable(sslPeerNotAfter); } + public Optional<String> sslPeerIssuerSubject() { return Optional.ofNullable(sslPeerIssuerSubject); } + public Optional<String> sslPeerFingerprint() { return Optional.ofNullable(sslPeerFingerprint); } public Optional<String> sslSniServerName() { return Optional.ofNullable(sslSniServerName); } public Optional<SslHandshakeFailure> sslHandshakeFailure() { return Optional.ofNullable(sslHandshakeFailure); } public List<String> sslSubjectAlternativeNames() { return sslSubjectAlternativeNames == null ? List.of() : sslSubjectAlternativeNames; } @@ -140,6 +146,8 @@ public class ConnectionLogEntry { private String sslPeerSubject; private Instant sslPeerNotBefore; private Instant sslPeerNotAfter; + private String sslPeerIssuerSubject; + private String sslPeerFingerprint; private String sslSniServerName; private SslHandshakeFailure sslHandshakeFailure; private List<String> sslSubjectAlternativeNames; @@ -221,6 +229,14 @@ public class ConnectionLogEntry { this.sslPeerNotAfter = sslPeerNotAfter; return this; } + public Builder withSslPeerIssuerSubject(String value) { + this.sslPeerIssuerSubject = value; + return this; + } + public Builder withSslPeerFingerprint(String value) { + this.sslPeerFingerprint = value; + return this; + } public Builder withSslSniServerName(String sslSniServerName) { this.sslSniServerName = sslSniServerName; return this; diff --git a/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java b/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java index d686c97249f..6d98c247ca0 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java +++ b/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java @@ -68,20 +68,24 @@ class JsonConnectionLogWriter implements LogWriter<ConnectionLogEntry> { Instant sslPeerNotBefore = unwrap(record.sslPeerNotBefore()); Instant sslPeerNotAfter = unwrap(record.sslPeerNotAfter()); String sslSniServerName = unwrap(record.sslSniServerName()); + String sslPeerIssuerSubject = unwrap(record.sslPeerIssuerSubject()); + String sslPeerFingerprint = unwrap(record.sslPeerFingerprint()); ConnectionLogEntry.SslHandshakeFailure sslHandshakeFailure = unwrap(record.sslHandshakeFailure()); List<String> sslSubjectAlternativeNames = record.sslSubjectAlternativeNames(); if (isAnyValuePresent( sslProtocol, sslSessionId, sslCipherSuite, sslPeerSubject, sslPeerNotBefore, sslPeerNotAfter, - sslSniServerName, sslHandshakeFailure)) { + sslSniServerName, sslHandshakeFailure, sslPeerIssuerSubject, sslPeerFingerprint)) { generator.writeObjectFieldStart("ssl"); writeOptionalString(generator, "protocol", sslProtocol); writeOptionalString(generator, "sessionId", sslSessionId); writeOptionalString(generator, "cipherSuite", sslCipherSuite); writeOptionalString(generator, "peerSubject", sslPeerSubject); + writeOptionalString(generator, "peerIssuerSubject", sslPeerIssuerSubject); writeOptionalTimestamp(generator, "peerNotBefore", sslPeerNotBefore); writeOptionalTimestamp(generator, "peerNotAfter", sslPeerNotAfter); + writeOptionalString(generator, "peerFingerprint", sslPeerFingerprint); writeOptionalString(generator, "sniServerName", sslSniServerName); if (sslHandshakeFailure != null) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index c54fa1cf1b9..8edc2eb84d0 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -13,6 +13,7 @@ import java.security.cert.X509Certificate; import java.util.Enumeration; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnection; import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; @@ -27,7 +28,7 @@ class HttpRequestFactory { HttpRequest httpRequest = HttpRequest.newServerRequest( container, getUri(servletRequest), - HttpRequest.Method.valueOf(servletRequest.getMethod()), + getMethod(servletRequest), HttpRequest.Version.fromString(servletRequest.getProtocol()), new InetSocketAddress(servletRequest.getRemoteAddr(), servletRequest.getRemotePort()), getConnection((Request) servletRequest).getCreatedTimeStamp()); @@ -39,6 +40,15 @@ class HttpRequestFactory { } } + private static HttpRequest.Method getMethod(HttpServletRequest servletRequest) { + String method = servletRequest.getMethod(); + try { + return HttpRequest.Method.valueOf(method); + } catch (IllegalArgumentException e) { + throw new RequestException(METHOD_NOT_ALLOWED, "Invalid method '" + method + "'"); + } + } + // Implementation based on org.eclipse.jetty.server.Request.getRequestURL(), but with the connector's local port instead public static URI getUri(HttpServletRequest servletRequest) { try { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java index 451a7dbf10d..4e3fd3f29b3 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java @@ -30,6 +30,9 @@ import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; import javax.net.ssl.StandardConstants; import java.net.InetSocketAddress; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateEncodingException; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.ArrayList; @@ -227,7 +230,6 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List throw new IllegalArgumentException("Unknown connection endpoint type: " + endpoint.getClass().getName()); } } - @FunctionalInterface private interface ListenerHandler { void run() throws Exception; } private static class ConnectionInfo { @@ -249,6 +251,8 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List private Date sslPeerNotBefore; private Date sslPeerNotAfter; private List<SNIServerName> sslSniServerNames; + private String sslPeerIssuerSubject; + private byte[] sslPeerEncodedCertificate; private SSLHandshakeException sslHandshakeException; private List<String> sslSubjectAlternativeNames; private String proxyProtocolVersion; @@ -307,8 +311,9 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List this.sslSubjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(peerCertificate).stream() .map(SubjectAlternativeName::getValue) .collect(Collectors.toList()); - - } catch (SSLPeerUnverifiedException e) { + this.sslPeerIssuerSubject = peerCertificate.getIssuerDN().getName(); + this.sslPeerEncodedCertificate = peerCertificate.getEncoded(); + } catch (SSLPeerUnverifiedException | CertificateEncodingException e) { // Throw if peer is not authenticated (e.g when client auth is disabled) // JSSE provides no means of checking for client authentication without catching this exception } @@ -365,10 +370,13 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List .findAny() .ifPresent(builder::withSslSniServerName); } - if (sslPeerSubject != null && sslPeerNotAfter != null && sslPeerNotBefore != null) { + if (sslPeerSubject != null && sslPeerNotAfter != null && sslPeerNotBefore != null + && sslPeerIssuerSubject != null && sslPeerEncodedCertificate != null) { builder.withSslPeerSubject(sslPeerSubject) + .withSslPeerIssuerSubject(sslPeerIssuerSubject) .withSslPeerNotAfter(sslPeerNotAfter.toInstant()) - .withSslPeerNotBefore(sslPeerNotBefore.toInstant()); + .withSslPeerNotBefore(sslPeerNotBefore.toInstant()) + .withSslPeerFingerprint(certificateFingerprint(sslPeerEncodedCertificate)); } if (sslSubjectAlternativeNames != null && !sslSubjectAlternativeNames.isEmpty()) { builder.withSslSubjectAlternativeNames(sslSubjectAlternativeNames); @@ -394,6 +402,14 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List return builder.build(); } + private static String certificateFingerprint(byte[] derEncoded) { + try { + return HexDump.toHexString(MessageDigest.getInstance("SHA-1").digest(derEncoded)); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 006d6cc84da..335f5d20ad5 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -214,11 +214,14 @@ public class JettyHttpServer extends AbstractServerProvider { @Override public void close() { try { - log.log(Level.INFO, String.format("Shutting down server (graceful=%b, timeout=%.1fs)", isGracefulShutdownEnabled(), server.getStopTimeout()/1000d)); + log.log(Level.INFO, String.format("Shutting down Jetty server (graceful=%b, timeout=%.1fs)", + isGracefulShutdownEnabled(), server.getStopTimeout()/1000d)); + long start = System.currentTimeMillis(); server.stop(); - log.log(Level.INFO, "Server shutdown completed"); + log.log(Level.INFO, String.format("Jetty server shutdown completed in %.3f seconds", + (System.currentTimeMillis()-start)/1000D)); } catch (final Exception e) { - log.log(Level.SEVERE, "Server shutdown threw an unexpected exception.", e); + log.log(Level.SEVERE, "Jetty server shutdown threw an unexpected exception.", e); } metricsReporter.shutdown(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java index 91230504c1c..b5a72b60f1c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java @@ -40,7 +40,7 @@ class ReferenceCountingRequestHandler implements DelegatedRequestHandler { @Override public ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { - try (final ResourceReference requestReference = request.refer()) { + try (final ResourceReference requestReference = request.refer(this)) { ContentChannel contentChannel; final ReferenceCountingResponseHandler referenceCountingResponseHandler = new ReferenceCountingResponseHandler(request, new NullContentResponseHandler(responseHandler)); diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index 9b9224e70ef..7b5187bdae2 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -43,6 +43,7 @@ import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; * @author Tony Vaagenes * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") // super class is deprecated public abstract class AbstractProcessingHandler<COMPONENT extends Processor> extends LoggingRequestHandler { private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener"); diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java index b126fbd16e4..001d00c2b37 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java @@ -3,7 +3,7 @@ package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.jdisc.Metric; @@ -12,7 +12,7 @@ import java.util.concurrent.Executor; /** * @author bjorncs */ -public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> extends LoggingRequestHandler { +public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> extends ThreadedHttpRequestHandler { private final RestApi restApi; @@ -24,7 +24,7 @@ public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> * Caller must ensure that provider instance does not try to access any uninitialized fields. */ @SuppressWarnings("unchecked") - protected RestApiRequestHandler(LoggingRequestHandler.Context context, RestApiProvider<T> provider) { + protected RestApiRequestHandler(ThreadedHttpRequestHandler.Context context, RestApiProvider<T> provider) { super(context); this.restApi = provider.createRestApi((T)this); } @@ -38,7 +38,7 @@ public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> this.restApi = provider.createRestApi((T)this); } - protected RestApiRequestHandler(LoggingRequestHandler.Context context, RestApi restApi) { + protected RestApiRequestHandler(ThreadedHttpRequestHandler.Context context, RestApi restApi) { super(context); this.restApi = restApi; } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java b/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java index ad943407743..bd693e47eda 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.http.server.jetty.testutils.TestDriver; import com.yahoo.jdisc.test.MockMetric; @@ -34,11 +34,11 @@ public class RestApiTestDriver implements AutoCloseable { public static Builder newBuilder(RestApiRequestHandler<?> handler) { return new Builder(handler); } - @FunctionalInterface public interface RestApiRequestHandlerFactory { RestApiRequestHandler<?> create(LoggingRequestHandler.Context context); } + @FunctionalInterface public interface RestApiRequestHandlerFactory { RestApiRequestHandler<?> create(ThreadedHttpRequestHandler.Context context); } public static Builder newBuilder(RestApiRequestHandlerFactory factory) { return new Builder(factory); } - public static LoggingRequestHandler.Context createHandlerTestContext() { - return new LoggingRequestHandler.Context(Executors.newSingleThreadExecutor(), new MockMetric()); + public static ThreadedHttpRequestHandler.Context createHandlerTestContext() { + return new ThreadedHttpRequestHandler.Context(Executors.newSingleThreadExecutor(), new MockMetric()); } public OptionalInt listenPort() { diff --git a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java index 2772ee593bb..74c08b1044b 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java @@ -44,7 +44,7 @@ public class ContainerTest extends ContainerTestBase { ComponentTakingConfig component = createComponentTakingConfig(getNewComponentGraph(container)); assertEquals("myString", component.config.stringVal()); - container.shutdownConfigurer(); + container.shutdownConfigRetriever(); } @Test @@ -66,7 +66,7 @@ public class ContainerTest extends ContainerTestBase { ComponentTakingConfig component2 = createComponentTakingConfig(newComponentGraph); assertEquals("reconfigured", component2.config.stringVal()); - container.shutdownConfigurer(); + container.shutdownConfigRetriever(); } @Test @@ -90,7 +90,7 @@ public class ContainerTest extends ContainerTestBase { assertNotNull(ComponentGraph.getNode(newGraph, "id1")); assertNotNull(ComponentGraph.getNode(newGraph, "id2")); - container.shutdownConfigurer(); + container.shutdownConfigRetriever(); } //@Test TODO @@ -219,7 +219,7 @@ public class ContainerTest extends ContainerTestBase { public void providers_are_destructed() { writeBootstrapConfigs("id1", DestructableProvider.class); - ComponentDeconstructor deconstructor = (components, bundles) -> { + ComponentDeconstructor deconstructor = (generation, components, bundles) -> { components.forEach(component -> { if (component instanceof AbstractComponent) { ((AbstractComponent) component).deconstruct(); @@ -312,7 +312,7 @@ public class ContainerTest extends ContainerTestBase { public static class TestDeconstructor implements ComponentDeconstructor { @Override - public void deconstruct(List<Object> components, Collection<Bundle> bundles) { + public void deconstruct(long generation, List<Object> components, Collection<Bundle> bundles) { components.forEach(component -> { if (component instanceof DestructableComponent) { DestructableComponent vespaComponent = (DestructableComponent) component; @@ -333,11 +333,13 @@ public class ContainerTest extends ContainerTestBase { } ComponentGraph getNewComponentGraph(Container container, ComponentGraph oldGraph) { - return container.getNewComponentGraph(oldGraph, Guice.createInjector(), true); + Container.ComponentGraphResult result = container.waitForNextGraphGeneration(oldGraph, Guice.createInjector(), true); + result.oldComponentsCleanupTask().run(); + return result.newGraph(); } ComponentGraph getNewComponentGraph(Container container) { - return container.getNewComponentGraph(new ComponentGraph(), Guice.createInjector(), true); + return container.waitForNextGraphGeneration(new ComponentGraph(), Guice.createInjector(), true).newGraph(); } private ComponentTakingConfig createComponentTakingConfig(ComponentGraph componentGraph) { diff --git a/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java b/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java index 673f79e515e..e7f12d3f228 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java @@ -65,7 +65,9 @@ public class ContainerTestBase { throw new UnsupportedOperationException("getBundle not supported."); } }); - componentGraph = container.getNewComponentGraph(componentGraph, Guice.createInjector(), true); + Container.ComponentGraphResult result = container.waitForNextGraphGeneration(this.componentGraph, Guice.createInjector(), true); + result.oldComponentsCleanupTask().run(); + this.componentGraph = result.newGraph(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java b/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java index d459070e2bd..34119163d2f 100644 --- a/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java +++ b/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java @@ -16,6 +16,7 @@ import java.util.Random; * @author gjoranv * @author ollivir */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class DirConfigSource { private final TemporaryFolder tempFolder = createTemporaryFolder(); public final ConfigSource configSource; diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java index e95dbcbcb22..e78810db897 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java @@ -187,11 +187,6 @@ public class VipStatusHandlerTestCase { } @Override - public <T> T getInstance(Key<T> tKey) { - return null; - } - - @Override public <T> T getInstance(Class<T> tClass) { return null; } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java index 4563000793d..ae0f679771e 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java @@ -97,7 +97,7 @@ public class LoggingRequestHandlerTestCase { } } - static final class AccessLogTestHandler extends LoggingRequestHandler { + static final class AccessLogTestHandler extends ThreadedHttpRequestHandler { public AccessLogTestHandler(Executor executor) { super(executor); @@ -150,11 +150,6 @@ public class LoggingRequestHandlerTestCase { } @Override - public <T> T getInstance(Key<T> tKey) { - return null; - } - - @Override public <T> T getInstance(Class<T> tClass) { return null; } diff --git a/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index 5dc9b72bafe..2a3877100f5 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -134,15 +134,16 @@ public class LogFileHandlerTestCase { firstHandler.shutdown(); assertThat(Files.size(Paths.get(firstHandler.getFileName()))).isEqualTo(5); - assertThat(root.toPath().resolve("symlink").toRealPath().toString()).isEqualTo(firstHandler.getFileName()); + assertThat(root.toPath().resolve("symlink").toRealPath().toString()).isEqualTo( + Paths.get(firstHandler.getFileName()).toRealPath().toString()); LogFileHandler<String> secondHandler = new LogFileHandler<>( Compression.ZSTD, BUFFER_SIZE, root.getAbsolutePath() + "/compressespreviouslogfile.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); secondHandler.publishAndWait("test"); secondHandler.rotateNow(); - assertThat(root.toPath().resolve("symlink").toRealPath().toString()).isEqualTo(secondHandler.getFileName()); - + assertThat(root.toPath().resolve("symlink").toRealPath().toString()).isEqualTo( + Paths.get(secondHandler.getFileName()).toRealPath().toString()); while (Files.exists(root.toPath().resolve(firstHandler.getFileName()))) Thread.sleep(1); assertThat(Files.exists(Paths.get(firstHandler.getFileName() + ".zst"))).isTrue(); diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java index e2ebdf7a8fc..d1ba4a4d190 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.inject.Key; import com.yahoo.jdisc.Container; import com.yahoo.jdisc.References; import com.yahoo.jdisc.ResourceReference; @@ -158,11 +157,6 @@ public class HttpRequestFactoryTest { } @Override - public <T> T getInstance(Key<T> tKey) { - return null; - } - - @Override public <T> T getInstance(Class<T> tClass) { return null; } diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 97b5ff90e28..3ce00e5d08a 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -35,111 +35,99 @@ <scope>test</scope> </dependency> </dependencies> - <profiles> - <profile> - <id>enforce-container-deps</id> - <activation> - <activeByDefault>false</activeByDefault> - <property> - <!-- Dependency resolution is broken for old maven used in our CentOS docker containers --> - <name>maven.version</name> - <value>!3.0.5</value> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <executions> - <execution> - <!-- To allow running 'mvn enforcer:enforce' from the command line --> - <id>default-cli</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <bannedDependencies> - <excludes> - <!-- Only allow explicitly listed deps in provided and compile scope --> - <exclude>*:*:*:jar:provided:*</exclude> - <exclude>*:*:*:jar:compile:*</exclude> - </excludes> - <includes> - <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <!-- To allow running 'mvn enforcer:enforce' from the command line --> + <id>default-cli</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <excludes> + <!-- Only allow explicitly listed deps in provided and compile scope --> + <exclude>*:*:*:jar:provided:*</exclude> + <exclude>*:*:*:jar:compile:*</exclude> + </excludes> + <includes> + <include>com.yahoo.vespa</include> + <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> - <!-- Use version range for jax deps, because jersey and junit affect the versions. --> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> - <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> - <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> - <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> - <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> - <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> - <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> - <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> - <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> - <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> - <include>org.json:json:[${org.json.version}]:jar:provided</include> - <include>org.jvnet.mimepull:mimepull:[${mimepull.version}]:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> - <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> - </includes> - </bannedDependencies> - </rules> - <fail>true</fail> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> + <!-- Use version range for jax deps, because jersey and junit affect the versions. --> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> + + <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> + <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> + <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> + <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> + <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> + <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> + <include>commons-logging:commons-logging:[1.2]:jar:provided</include> + <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> + <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> + <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> + <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> + <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> + <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> + <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> + <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> + <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> + <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> + <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> + <include>org.json:json:[${org.json.version}]:jar:provided</include> + <include>org.jvnet.mimepull:mimepull:[${mimepull.version}]:jar:provided</include> + <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> + <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> + </includes> + </bannedDependencies> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> <properties> <maven.javadoc.skip>true</maven.javadoc.skip> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index ff6786080ef..6e06791dc1c 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -411,7 +411,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.44.v20210927</jetty.version> + <jetty.version>9.4.45.v20220203</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20090211</org.json.version> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 0d602e9e730..1c3115900de 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -21,6 +21,16 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 099df79cd9b..a27b082f014 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -8,9 +8,9 @@ import com.google.inject.Injector; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.Vtag; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.SubscriberClosedException; import com.yahoo.container.Container; import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; @@ -26,13 +26,17 @@ import com.yahoo.jdisc.application.Application; import com.yahoo.jdisc.application.BindingRepository; import com.yahoo.jdisc.application.ContainerActivator; import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.application.DeactivatedContainer; import com.yahoo.jdisc.application.GuiceRepository; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; @@ -42,19 +46,17 @@ import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; +import com.yahoo.yolean.UncheckedInterruptedException; +import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.WeakHashMap; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; @@ -67,10 +69,9 @@ import static com.yahoo.collections.CollectionUtil.first; public final class ConfiguredApplication implements Application { private static final Logger log = Logger.getLogger(ConfiguredApplication.class.getName()); - private static final Set<ClientProvider> startedClients = Collections.newSetFromMap(new WeakHashMap<>()); - static final String SANITIZE_FILENAME = "[/,;]"; - - private static final Set<ServerProvider> startedServers = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Object monitor = new Object(); + private final Set<ClientProvider> startedClients = createIdentityHashSet(); + private final Set<ServerProvider> startedServers = createIdentityHashSet(); private final SubscriberFactory subscriberFactory; private final Metric metric; private final ContainerActivator activator; @@ -78,11 +79,12 @@ public final class ConfiguredApplication implements Application { private final OsgiFramework osgiFramework; private final com.yahoo.jdisc.Timer timerSingleton; private final AtomicBoolean dumpHeapOnShutdownTimeout = new AtomicBoolean(false); - private final AtomicDouble shudownTimeoutS = new AtomicDouble(50.0); + private final AtomicDouble shutdownTimeoutS = new AtomicDouble(50.0); // Subscriber that is used when this is not a standalone-container. Subscribes // to config to make sure that container will be registered in slobrok (by {@link com.yahoo.jrt.slobrok.api.Register}) // if slobrok config changes (typically slobroks moving to other nodes) private final Optional<SlobrokConfigSubscriber> slobrokConfigSubscriber; + private final ShutdownDeadline shutdownDeadline; //TODO: FilterChainRepository should instead always be set up in the model. private final FilterChainRepository defaultFilterChainRepository = @@ -92,15 +94,16 @@ public final class ConfiguredApplication implements Application { new ComponentRegistry<>(), new ComponentRegistry<>()); private final OsgiFramework restrictedOsgiFramework; + private final Phaser nonTerminatedContainerTracker = new Phaser(1); + private final Thread reconfigurerThread; + private final Thread portWatcher; private HandlersConfigurerDi configurer; - private ScheduledThreadPoolExecutor shutdownDeadlineExecutor; - private Thread reconfigurerThread; - private Thread portWatcher; private QrConfig qrConfig; private Register slobrokRegistrator = null; private Supervisor supervisor = null; private Acceptor acceptor = null; + private volatile boolean shutdownReconfiguration = false; static { LogSetup.initVespaLogging("Container"); @@ -133,6 +136,9 @@ public final class ConfiguredApplication implements Application { ? Optional.of(new SlobrokConfigSubscriber(configId)) : Optional.empty(); this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext())); + this.shutdownDeadline = new ShutdownDeadline(configId); + this.reconfigurerThread = new Thread(this::doReconfigurationLoop, "configured-application-reconfigurer"); + this.portWatcher = new Thread(this::watchPortChange, "configured-application-port-watcher"); } @Override @@ -143,35 +149,34 @@ public final class ConfiguredApplication implements Application { ContainerBuilder builder = createBuilderWithGuiceBindings(); configurer = createConfigurer(builder.guiceModules().activate()); - initializeAndActivateContainer(builder); - startReconfigurerThread(); - portWatcher = new Thread(this::watchPortChange); + initializeAndActivateContainer(builder, () -> {}); + reconfigurerThread.setDaemon(true); + reconfigurerThread.start(); + portWatcher.setDaemon(true); portWatcher.start(); - slobrokRegistrator = registerInSlobrok(qrConfig); // marks this as up + if (setupRpc()) { + slobrokRegistrator = registerInSlobrok(qrConfig); // marks this as up + } } - /** - * The container has no RPC methods, but we still need an RPC server - * to register in Slobrok to enable orchestration. - */ - private Register registerInSlobrok(QrConfig qrConfig) { - if ( ! qrConfig.rpc().enabled()) return null; - - // 1. Set up RPC server - supervisor = new Supervisor(new Transport("slobrok")).setDropEmptyBuffers(true); + private boolean setupRpc() { + if ( ! qrConfig.rpc().enabled()) return false; + supervisor = new Supervisor(new Transport("configured-application")).setDropEmptyBuffers(true); + supervisor.addMethod(new Method("prepareStop", "d", "", this::prepareStop)); Spec listenSpec = new Spec(qrConfig.rpc().port()); try { acceptor = supervisor.listen(listenSpec); - } - catch (ListenFailedException e) { + return true; + } catch (ListenFailedException e) { throw new RuntimeException("Could not create rpc server listening on " + listenSpec, e); } + } - // 2. Register it in slobrok + private Register registerInSlobrok(QrConfig qrConfig) { SlobrokList slobrokList = getSlobrokList(); Spec mySpec = new Spec(HostName.getLocalhost(), acceptor.port()); - slobrokRegistrator = new Register(supervisor, slobrokList, mySpec); + Register slobrokRegistrator = new Register(supervisor, slobrokList, mySpec); slobrokRegistrator.registerName(qrConfig.rpc().slobrokId()); log.log(Level.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + "' at " + mySpec + " with: " + slobrokList); @@ -245,54 +250,68 @@ public final class ConfiguredApplication implements Application { void reconfigure(QrConfig qrConfig) { dumpHeapOnShutdownTimeout.set(qrConfig.shutdown().dumpHeapOnTimeout()); - shudownTimeoutS.set(qrConfig.shutdown().timeout()); + shutdownTimeoutS.set(qrConfig.shutdown().timeout()); } - private void initializeAndActivateContainer(ContainerBuilder builder) { + private void initializeAndActivateContainer(ContainerBuilder builder, Runnable cleanupTask) { addHandlerBindings(builder, Container.get().getRequestHandlerRegistry(), configurer.getComponent(ApplicationContext.class).discBindingsConfig); - installServerProviders(builder); - - activator.activateContainer(builder); // TODO: .notifyTermination(.. decompose previous component graph ..) + List<ServerProvider> currentServers = Container.get().getServerProviderRegistry().allComponents(); + for (ServerProvider server : currentServers) { + builder.serverProviders().install(server); + } + activateContainer(builder, cleanupTask); + startAndStopServers(currentServers); - startClients(); - startAndStopServers(); + startAndRemoveClients(Container.get().getClientProviderRegistry().allComponents()); log.info("Switching to the latest deployed set of configurations and components. " + "Application config generation: " + configurer.generation()); metric.set("application_generation", configurer.generation(), metric.createContext(Map.of())); } + private void activateContainer(ContainerBuilder builder, Runnable onPreviousContainerTermination) { + DeactivatedContainer deactivated = activator.activateContainer(builder); + if (deactivated != null) { + nonTerminatedContainerTracker.register(); + deactivated.notifyTermination(() -> { + try { + onPreviousContainerTermination.run(); + } finally { + nonTerminatedContainerTracker.arriveAndDeregister(); + } + }); + } + } + private ContainerBuilder createBuilderWithGuiceBindings() { ContainerBuilder builder = activator.newContainerBuilder(); setupGuiceBindings(builder.guiceModules()); return builder; } - private void startReconfigurerThread() { - reconfigurerThread = new Thread(() -> { - while ( ! Thread.interrupted()) { - try { - ContainerBuilder builder = createBuilderWithGuiceBindings(); - - // Block until new config arrives, and it should be applied - configurer.getNewComponentGraph(builder.guiceModules().activate(), false); - initializeAndActivateContainer(builder); - } catch (ConfigInterruptedException e) { - break; - } catch (Exception | LinkageError e) { // LinkageError: OSGi problems - tryReportFailedComponentGraphConstructionMetric(configurer, e); - log.log(Level.SEVERE, - "Reconfiguration failed, your application package must be fixed, unless this is a " + - "JNI reload issue: " + Exceptions.toMessageString(e), e); - } catch (Error e) { - com.yahoo.protect.Process.logAndDie("java.lang.Error on reconfiguration: We are probably in " + - "a bad state and will terminate", e); - } + @SuppressWarnings("removal") // TODO Vespa 8: remove + private void doReconfigurationLoop() { + while (!shutdownReconfiguration) { + try { + ContainerBuilder builder = createBuilderWithGuiceBindings(); + + // Block until new config arrives, and it should be applied + Runnable cleanupTask = configurer.waitForNextGraphGeneration(builder.guiceModules().activate(), false); + initializeAndActivateContainer(builder, cleanupTask); + } catch (UncheckedInterruptedException | SubscriberClosedException | ConfigInterruptedException e) { + break; + } catch (Exception | LinkageError e) { // LinkageError: OSGi problems + tryReportFailedComponentGraphConstructionMetric(configurer, e); + log.log(Level.SEVERE, + "Reconfiguration failed, your application package must be fixed, unless this is a " + + "JNI reload issue: " + Exceptions.toMessageString(e), e); + } catch (Error e) { + com.yahoo.protect.Process.logAndDie("java.lang.Error on reconfiguration: We are probably in " + + "a bad state and will terminate", e); } - log.fine("Shutting down HandlersConfigurerDi"); - }); - reconfigurerThread.start(); + } + log.fine("Reconfiguration loop exited"); } private static void tryReportFailedComponentGraphConstructionMetric(HandlersConfigurerDi configurer, Throwable error) { @@ -307,47 +326,47 @@ public final class ConfiguredApplication implements Application { } } - private static void installServerProviders(ContainerBuilder builder) { - List<ServerProvider> serverProviders = Container.get().getServerProviderRegistry().allComponents(); - for (ServerProvider server : serverProviders) { - builder.serverProviders().install(server); - } - } - - private static void startClients() { - for (ClientProvider client : Container.get().getClientProviderRegistry().allComponents()) { - if (!startedClients.contains(client)) { - client.start(); - startedClients.add(client); + private void startAndStopServers(List<ServerProvider> currentServers) { + synchronized (monitor) { + Set<ServerProvider> serversToClose = createIdentityHashSet(startedServers); + serversToClose.removeAll(currentServers); + if (serversToClose.size() > 0) { + log.info(String.format("Closing %d server instances", serversToClose.size())); + for (ServerProvider server : serversToClose) { + server.close(); + startedServers.remove(server); + } + } + for (ServerProvider server : currentServers) { + if (!startedServers.contains(server)) { + server.start(); + startedServers.add(server); + } } } } - private static void startAndStopServers() { - List<ServerProvider> currentServers = Container.get().getServerProviderRegistry().allComponents(); - HashSet<ServerProvider> serversToClose = new HashSet<>(startedServers); - serversToClose.removeAll(currentServers); - for (ServerProvider server : serversToClose) { - closeServer(server); - } - for (ServerProvider server : currentServers) { - if (!startedServers.contains(server)) { - server.start(); - startedServers.add(server); + private void startAndRemoveClients(List<ClientProvider> currentClients) { + synchronized (monitor) { + Set<ClientProvider> clientToRemove = createIdentityHashSet(startedClients); + clientToRemove.removeAll(currentClients); + for (ClientProvider client : clientToRemove) { + startedClients.remove(client); + } + for (ClientProvider client : currentClients) { + if (!startedClients.contains(client)) { + client.start(); + startedClients.add(client); + } } } } - private static void closeServer(ServerProvider server) { - server.close(); - startedServers.remove(server); - } - private HandlersConfigurerDi createConfigurer(Injector discInjector) { return new HandlersConfigurerDi(subscriberFactory, Container.get(), configId, - new Deconstructor(Deconstructor.Mode.RECONFIG), + new Deconstructor(), discInjector, osgiFramework); } @@ -367,69 +386,63 @@ public final class ConfiguredApplication implements Application { @Override public void stop() { - startShutdownDeadlineExecutor(); - shutdownReconfigurerThread(); + log.info("Stop: Initiated"); + shutdownDeadline.schedule((long)(shutdownTimeoutS.get() * 1000), dumpHeapOnShutdownTimeout.get()); + stopServersAndAwaitTermination(); + log.info("Stop: Finished"); + } - log.info("Stop: Closing servers"); - for (ServerProvider server : Container.get().getServerProviderRegistry().allComponents()) { - if (startedServers.contains(server)) { - closeServer(server); - } + private void prepareStop(Request request) { + log.info("PrepareStop: Initiated"); + long timeoutMillis = (long) (request.parameters().get(0).asDouble() * 1000); + try (ShutdownDeadline ignored = + new ShutdownDeadline(configId).schedule(timeoutMillis, dumpHeapOnShutdownTimeout.get())) { + stopServersAndAwaitTermination(); + log.info("PrepareStop: Finished"); + } catch (Exception e) { + request.setError(ErrorCode.METHOD_FAILED, e.getMessage()); + throw e; } + } - log.info("Stop: Shutting container down"); - configurer.shutdown(new Deconstructor(Deconstructor.Mode.SHUTDOWN)); - slobrokConfigSubscriber.ifPresent(SlobrokConfigSubscriber::shutdown); - Container.get().shutdown(); - - unregisterInSlobrok(); - LogSetup.cleanup(); - log.info("Stop: Finished"); + private void stopServersAndAwaitTermination() { + shutdownReconfigurer(); + startAndStopServers(List.of()); + startAndRemoveClients(List.of()); + activateContainer(null, () -> log.info("Last active container generation has terminated")); + nonTerminatedContainerTracker.arriveAndAwaitAdvance(); } - private void shutdownReconfigurerThread() { - if (reconfigurerThread == null) return; - reconfigurerThread.interrupt(); + private void shutdownReconfigurer() { + if (!reconfigurerThread.isAlive()) return; + log.info("Shutting down reconfiguration thread"); + long start = System.currentTimeMillis(); + shutdownReconfiguration = true; + configurer.shutdownConfigRetriever(); try { - //Workaround for component constructors masking InterruptedException. - while (reconfigurerThread.isAlive()) { - reconfigurerThread.interrupt(); - long millis = 200; - reconfigurerThread.join(millis); - } + reconfigurerThread.join(); + log.info(String.format( + "Reconfiguration thread shutdown completed in %.3f seconds", (System.currentTimeMillis() - start) / 1000D)); } catch (InterruptedException e) { - log.info("Interrupted while joining on HandlersConfigurer reconfigure thread."); - Thread.currentThread().interrupt(); + String message = "Interrupted while waiting for reconfiguration shutdown"; + log.warning(message); + log.log(Level.FINE, e.getMessage(), e); + throw new UncheckedInterruptedException(message, true); } } @Override public void destroy() { - if (shutdownDeadlineExecutor != null) { //stop() is not called when exception happens during start - shutdownDeadlineExecutor.shutdownNow(); + log.info("Destroy: Shutting down container now"); + if (configurer != null) { + configurer.shutdown(); } - } - - static String santizeFileName(String s) { - return s.trim() - .replace('\\', '.') - .replaceAll(SANITIZE_FILENAME, "."); - } - - // Workaround for ApplicationLoader.stop not being able to shutdown - private void startShutdownDeadlineExecutor() { - shutdownDeadlineExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer")); - shutdownDeadlineExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - long delayMillis = (long)(shudownTimeoutS.get() * 1000.0); - shutdownDeadlineExecutor.schedule(() -> { - if (dumpHeapOnShutdownTimeout.get()) { - String heapDumpName = Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + santizeFileName(configId) + "." + ProcessHandle.current().pid() + ".hprof"; - com.yahoo.protect.Process.dumpHeap(heapDumpName, true); - } - com.yahoo.protect.Process.logAndDie( - "Timed out waiting for application shutdown. Please check that all your request handlers " + - "drain their request content channels.", true); - }, delayMillis, TimeUnit.MILLISECONDS); + slobrokConfigSubscriber.ifPresent(SlobrokConfigSubscriber::shutdown); + Container.get().shutdown(); + unregisterInSlobrok(); + LogSetup.cleanup(); + shutdownDeadline.cancel(); + log.info("Destroy: Finished"); } private static void addHandlerBindings(ContainerBuilder builder, @@ -455,6 +468,16 @@ public final class ConfiguredApplication implements Application { } } + private static <E> Set<E> createIdentityHashSet() { + return Collections.newSetFromMap(new IdentityHashMap<>()); + } + + private static <E> Set<E> createIdentityHashSet(Collection<E> items) { + Set<E> set = createIdentityHashSet(); + set.addAll(items); + return set; + } + public static final class ApplicationContext { final JdiscBindingsConfig discBindingsConfig; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ShutdownDeadline.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ShutdownDeadline.java new file mode 100644 index 00000000000..15af216c210 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ShutdownDeadline.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static com.yahoo.protect.Process.dumpHeap; +import static com.yahoo.protect.Process.logAndDie; + +/** + * Kills the JVM if the application is unable to shutdown before deadline. + * + * @author bjorncs + */ +class ShutdownDeadline implements AutoCloseable { + + private final String configId; + private final ScheduledThreadPoolExecutor executor; + + ShutdownDeadline(String configId) { + this.configId = configId; + executor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer")); + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + } + + ShutdownDeadline schedule(long millis, boolean heapdumpOnShutdown) { + executor.schedule(() -> onDeadline(heapdumpOnShutdown), millis, TimeUnit.MILLISECONDS); + return this; + } + + void cancel() { executor.shutdownNow(); } + @Override public void close() { cancel(); } + + private void onDeadline(boolean heapdumpOnShutdown) { + if (heapdumpOnShutdown) dumpHeap(heapdumpFilename(), true); + logAndDie("Timed out waiting for application shutdown. Please check that all your request handlers " + + "drain their request content channels.", true); + } + + private String heapdumpFilename() { + return Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + sanitizeFileName(configId) + "." + + ProcessHandle.current().pid() + ".hprof"; + } + + static String sanitizeFileName(String s) { + return s.trim().replace('\\', '.').replaceAll("[/,;]", "."); + } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index acf4a494a96..72ba7240361 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -7,6 +7,7 @@ import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.SharedResource; +import com.yahoo.yolean.UncheckedInterruptedException; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; @@ -15,12 +16,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Random; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,35 +35,20 @@ public class Deconstructor implements ComponentDeconstructor { private static final Logger log = Logger.getLogger(Deconstructor.class.getName()); - private static final Duration RECONFIG_DECONSTRUCT_DELAY = Duration.ofSeconds(60); + private final ExecutorService executor = + Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory("component-deconstructor")); // This must be smaller than the shutdownDeadlineExecutor delay in ConfiguredApplication - private static final Duration SHUTDOWN_DECONSTRUCT_TIMEOUT = Duration.ofSeconds(45); + private final Duration shutdownTimeout; - public enum Mode { - RECONFIG, // Delay deconstruction to allow old components to finish processing in-flight requests. - SHUTDOWN // The container is shutting down. Start deconstructing immediately, and wait until all components - // are deconstructed, to prevent shutting down while deconstruct is in progress. + public Deconstructor(Duration shutdownTimeout) { + this.shutdownTimeout = shutdownTimeout; } - private final ScheduledExecutorService executor = - Executors.newScheduledThreadPool(2, ThreadFactoryFactory.getThreadFactory("component-deconstructor")); - - private final Mode mode; - private final Duration delay; - - public Deconstructor(Mode mode) { - this(mode, (mode == Mode.RECONFIG) ? RECONFIG_DECONSTRUCT_DELAY : Duration.ZERO); - } - - // For testing only - Deconstructor(Mode mode, Duration reconfigDeconstructDelay) { - this.mode = mode; - this.delay = reconfigDeconstructDelay; - } + public Deconstructor() { this(Duration.ofSeconds(45)); } @Override - public void deconstruct(List<Object> components, Collection<Bundle> bundles) { + public void deconstruct(long generation, List<Object> components, Collection<Bundle> bundles) { Collection<Deconstructable> destructibleComponents = new ArrayList<>(); for (var component : components) { if (component instanceof AbstractComponent) { @@ -76,41 +59,46 @@ public class Deconstructor implements ComponentDeconstructor { } else if (component instanceof Provider) { destructibleComponents.add((Deconstructable) component); } else if (component instanceof SharedResource) { - log.log(FINE, () -> "Releasing container reference to resource " + component); - // No need to delay release, as jdisc does ref-counting - ((SharedResource) component).release(); + // Release shared resources in same order as other components in case of usage without reference counting + destructibleComponents.add(new SharedResourceReleaser(component)); } } if (!destructibleComponents.isEmpty() || !bundles.isEmpty()) { - var task = executor.schedule(new DestructComponentTask(destructibleComponents, bundles), - delay.getSeconds(), TimeUnit.SECONDS); - if (mode.equals(Mode.SHUTDOWN)) { - waitFor(task, SHUTDOWN_DECONSTRUCT_TIMEOUT); - } + executor.execute(new DestructComponentTask(generation, destructibleComponents, bundles)); } } - private void waitFor(ScheduledFuture<?> task, Duration timeout) { + @Override + public void shutdown() { + executor.shutdown(); try { - log.info("Waiting up to " + timeout.toSeconds() + " seconds for all components to deconstruct."); - task.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + log.info("Waiting up to " + shutdownTimeout.toSeconds() + " seconds for all previous components graphs to deconstruct."); + if (!executor.awaitTermination(shutdownTimeout.getSeconds(), TimeUnit.SECONDS)) { + log.warning("Waiting for deconstruction timed out."); + } } catch (InterruptedException e) { log.info("Interrupted while waiting for component deconstruction to finish."); - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - log.warning("Component deconstruction threw an exception: " + e.getMessage()); - } catch (TimeoutException e) { - log.warning("Component deconstruction timed out."); + throw new UncheckedInterruptedException(e, true); } } + private static class SharedResourceReleaser implements Deconstructable { + final SharedResource resource; + + private SharedResourceReleaser(Object resource) { this.resource = (SharedResource) resource; } + + @Override public void deconstruct() { resource.release(); } + } + private static class DestructComponentTask implements Runnable { private final Random random = new Random(System.nanoTime()); + private final long generation; private final Collection<Deconstructable> components; private final Collection<Bundle> bundles; - DestructComponentTask(Collection<Deconstructable> components, Collection<Bundle> bundles) { + DestructComponentTask(long generation, Collection<Deconstructable> components, Collection<Bundle> bundles) { + this.generation = generation; this.components = components; this.bundles = bundles; } @@ -120,12 +108,15 @@ public class Deconstructor implements ComponentDeconstructor { * Used to randomize restart to avoid simultaneous cluster restarts. */ private Duration getRandomizedShutdownDelay() { - long seconds = (long) random.nextDouble() * 60 * 10; + long seconds = (long) (random.nextDouble() * 60 * 10); return Duration.ofSeconds(seconds); } @Override public void run() { + long start = System.currentTimeMillis(); + log.info(String.format("Starting deconstruction of %d components and %d bundles from generation %d", + components.size(), bundles.size(), generation)); for (var component : components) { log.log(FINE, () -> "Starting deconstruction of " + component); try { @@ -156,6 +147,7 @@ public class Deconstructor implements ComponentDeconstructor { log.log(SEVERE, "Could not uninstall bundle " + bundle); } } + log.info(String.format("Completed deconstruction in %.3f seconds", (System.currentTimeMillis() - start) / 1000D)); // NOTE: It could be tempting to refresh packages here, but if active bundles were using any of // the removed ones, they would switch wiring in the middle of a generation's lifespan. // This would happen if the dependent active bundle has not been rebuilt with a new version diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 1aad9f18616..a6c2c5999a8 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -46,6 +46,7 @@ printenv > $cfpfile || exit 1 getconfig() { + set -e qrstartcfg="" case "${VESPA_CONFIG_ID}" in dir:*) @@ -58,6 +59,7 @@ getconfig() { esac cmds=`echo "$qrstartcfg" | perl -ne 's/^(\w+)\.(\w+) (.*)/$1_$2=$3/ && print'` eval "$cmds" + set +e } # Print the value of the cgroups v2 interface filename $1 for current process, diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java deleted file mode 100644 index c8cf5c0ce63..00000000000 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class ConfiguredApplicationTest { - @Test - public void testConfigId2FileName() { - assertEquals("admin.metrics.2088223-v6-1.ostk.bm2.prod.ne1.yahoo.com", ConfiguredApplication.santizeFileName("admin/metrics/2088223-v6-1.ostk.bm2.prod.ne1.yahoo.com")); - assertEquals("admin.standalone.cluster-controllers.1", ConfiguredApplication.santizeFileName("admin/standalone/cluster-controllers/1 ")); - } -} diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java new file mode 100644 index 00000000000..f2733d61f97 --- /dev/null +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java @@ -0,0 +1,18 @@ +package com.yahoo.container.jdisc;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +import org.junit.jupiter.api.Test; + +import static com.yahoo.container.jdisc.ShutdownDeadline.sanitizeFileName; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bjorncs + */ +class ShutdownDeadlineTest { + @Test + void testConfigId2FileName() { + assertEquals("admin.metrics.2088223-v6-1.ostk.bm2.prod.ne1.yahoo.com", sanitizeFileName("admin/metrics/2088223-v6-1.ostk.bm2.prod.ne1.yahoo.com")); + assertEquals("admin.standalone.cluster-controllers.1", sanitizeFileName("admin/standalone/cluster-controllers/1 ")); + } +} diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java index e9a9d310380..715c2759aa6 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java @@ -9,7 +9,6 @@ import com.yahoo.jdisc.SharedResource; import org.junit.Before; import org.junit.Test; -import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.function.Supplier; @@ -26,22 +25,23 @@ public class DeconstructorTest { @Before public void init() { - deconstructor = new Deconstructor(Deconstructor.Mode.RECONFIG, Duration.ZERO); + deconstructor = new Deconstructor(); } @Test - public void deconstruct_is_synchronous_in_shutdown_mode() { - deconstructor = new Deconstructor(Deconstructor.Mode.SHUTDOWN); + public void deconstructor_waits_for_completion_on_shutdown() { + deconstructor = new Deconstructor(); var slowDeconstructComponent = new SlowDeconstructComponent(); - deconstructor.deconstruct(List.of(slowDeconstructComponent), emptyList()); + deconstructor.deconstruct(0, List.of(slowDeconstructComponent), emptyList()); + deconstructor.shutdown(); assertTrue(slowDeconstructComponent.destructed); } @Test public void require_abstract_component_destructed() throws InterruptedException { TestAbstractComponent abstractComponent = new TestAbstractComponent(); - deconstructor.deconstruct(List.of(abstractComponent), emptyList()); + deconstructor.deconstruct(0, List.of(abstractComponent), emptyList()); waitForDeconstructToComplete(() -> abstractComponent.destructed); assertTrue(abstractComponent.destructed); @@ -50,16 +50,17 @@ public class DeconstructorTest { @Test public void require_provider_destructed() throws InterruptedException { TestProvider provider = new TestProvider(); - deconstructor.deconstruct(List.of(provider), emptyList()); + deconstructor.deconstruct(0, List.of(provider), emptyList()); waitForDeconstructToComplete(() -> provider.destructed); assertTrue(provider.destructed); } @Test - public void require_shared_resource_released() { + public void require_shared_resource_released() throws InterruptedException { TestSharedResource sharedResource = new TestSharedResource(); - deconstructor.deconstruct(List.of(sharedResource), emptyList()); + deconstructor.deconstruct(0, List.of(sharedResource), emptyList()); + waitForDeconstructToComplete(() -> sharedResource.released); assertTrue(sharedResource.released); } @@ -67,7 +68,7 @@ public class DeconstructorTest { public void bundles_are_uninstalled() throws InterruptedException { var bundle = new UninstallableMockBundle(); // Done by executor, so it takes some time even with a 0 delay. - deconstructor.deconstruct(emptyList(), singleton(bundle)); + deconstructor.deconstruct(0, emptyList(), singleton(bundle)); waitForDeconstructToComplete(() -> bundle.uninstalled); assertTrue(bundle.uninstalled); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index 9a8dda246a0..8f21cb227d8 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -10,10 +10,6 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.service.ServerProvider; - -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Level; - import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -23,6 +19,8 @@ import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.shared.ServerSession; import java.net.URI; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -57,6 +55,7 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override public void close() { log.log(Level.FINE, "Closing message bus server."); + session.disconnect(); runState.set(State.STOPPED); } diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java index e0ab9dbff4f..49057330774 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java @@ -3,7 +3,6 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.EmptyReply; @@ -14,6 +13,7 @@ import com.yahoo.messagebus.MessageHandler; import com.yahoo.messagebus.Reply; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -81,6 +81,8 @@ public class SharedDestinationSession extends AbstractResource implements Messag session.connect(); } + @Override public void disconnect() { session.disconnect(); } + @Override protected void destroy() { log.log(Level.FINE, "Destroying shared destination session."); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index 06c6192cb7a..49bf6ba8fb7 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -3,7 +3,6 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -16,6 +15,7 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -101,6 +101,9 @@ public class SharedIntermediateSession extends AbstractResource } @Override + public void disconnect() { session.disconnect(); } + + @Override protected void destroy() { log.log(Level.FINE, "Destroying shared intermediate session."); session.destroy(); diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java index 84a722a753f..d8481138a99 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java @@ -354,6 +354,8 @@ public class MbusServerTestCase { @Override public void connect() { } + @Override public void disconnect() {} + @Override public ResourceReference refer() { ++refCount; diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java index cc473a030c4..b2b9dd47514 100644 --- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java +++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java @@ -8,7 +8,7 @@ import com.google.inject.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.querytransform.RecallSearcher; import com.yahoo.restapi.Path; @@ -36,7 +36,7 @@ import java.util.logging.Level; * * @author Henrik Høiness */ -public class GUIHandler extends LoggingRequestHandler { +public class GUIHandler extends ThreadedHttpRequestHandler { private static final ObjectMapper jsonMapper = new ObjectMapper(); diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b320a1090ae..8cd32985057 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -503,7 +503,8 @@ "public void <init>(com.yahoo.prelude.query.Item, java.util.Collection)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public java.lang.String getName()", - "protected void adding(com.yahoo.prelude.query.Item)" + "protected void adding(com.yahoo.prelude.query.Item)", + "public static boolean acceptsChildrenOfType(com.yahoo.prelude.query.Item$ItemType)" ], "fields": [] }, @@ -719,7 +720,6 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType WORD", "public static final enum com.yahoo.prelude.query.Item$ItemType INT", "public static final enum com.yahoo.prelude.query.Item$ItemType PHRASE", - "public static final enum com.yahoo.prelude.query.Item$ItemType PAREN", "public static final enum com.yahoo.prelude.query.Item$ItemType PREFIX", "public static final enum com.yahoo.prelude.query.Item$ItemType SUBSTRING", "public static final enum com.yahoo.prelude.query.Item$ItemType NEAR", @@ -1594,6 +1594,7 @@ "public boolean hasItemClass(java.lang.Class)", "public com.yahoo.prelude.query.Item createItemClass()", "public java.lang.String toSign()", + "public com.yahoo.prelude.query.Item$ItemType toItemType()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -2118,6 +2119,7 @@ "public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)", "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)", "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", + "public java.lang.String name()", "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)", "protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)", @@ -2211,6 +2213,7 @@ "abstract" ], "methods": [ + "public java.lang.String name()", "public abstract void working(java.lang.Object)", "public abstract void failed(java.lang.Object)", "public void ping(java.lang.Object, java.util.concurrent.Executor)", @@ -5234,13 +5237,16 @@ "public java.lang.String getFormat()", "public void setFormat(java.lang.String)", "public java.lang.Object clone()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", "public boolean getTiming()", "public void setTiming(boolean)", "public java.util.Set getSummaryFields()", + "public void setSummaryFields(java.lang.String)", + "public boolean getTensorShortForm()", + "public void setTensorShortForm(java.lang.String)", + "public void setTensorShortForm(boolean)", "public void prepare()", - "public void setSummaryFields(java.lang.String)" + "public boolean equals(java.lang.Object)", + "public int hashCode()" ], "fields": [ "public static final java.lang.String PRESENTATION", @@ -5248,6 +5254,7 @@ "public static final java.lang.String TIMING", "public static final java.lang.String SUMMARY", "public static final java.lang.String SUMMARY_FIELDS", + "public static final java.lang.String TENSORS", "public static final java.lang.String FORMAT" ] }, @@ -5292,6 +5299,7 @@ "public boolean isEmpty()", "public com.yahoo.prelude.query.Item and(com.yahoo.prelude.query.Item)", "public static java.util.List getPositiveTerms(com.yahoo.prelude.query.Item)", + "public int treeSize()", "public bridge synthetic com.yahoo.prelude.query.CompositeItem clone()", "public bridge synthetic com.yahoo.prelude.query.Item clone()", "public bridge synthetic java.lang.Object clone()" @@ -6241,7 +6249,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig)", + "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig, java.util.concurrent.Executor)", "public void <init>()", "public void <init>(com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", "public final void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", @@ -7154,6 +7162,7 @@ "methods": [ "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean)", "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean)", + "protected void <init>(boolean, boolean, boolean)", "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean, boolean)", "public void accept(java.lang.String, java.lang.Object)", "public void accept(java.lang.String, byte[], int, int)", @@ -7195,6 +7204,7 @@ "public void endResponse()", "public java.lang.String getEncoding()", "public java.lang.String getMimeType()", + "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(boolean)", "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(com.fasterxml.jackson.core.JsonGenerator, boolean)" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 51c82eac264..ae953426e63 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -348,7 +348,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, Execution.ATTRIBUTEPREFETCH, execution); + searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS, execution); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index df87de2a12b..1e0cfa3be9e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -51,12 +51,17 @@ public final class DocsumDefinitionSet { } if (ds == null) { throw new ConfigurationException("Fetched hit with summary class " + summaryClass + - ", but this summary class is not in current summary config (" + toString() + ")" + + ", but this summary class is not in current summary config (" + this + ")" + " (that is, you asked for something unknown, and no default was found)"); } return ds; } + /** Do we have a summary definition with the given name */ + public boolean hasDocsum(String summaryClass) { + return definitionsByName.containsKey(summaryClass); + } + /** * Makes data available for decoding for the given hit. * @@ -66,7 +71,7 @@ public final class DocsumDefinitionSet { * @return Error message or null on success. * @throws ConfigurationException if the summary class of this hit is missing */ - public final String lazyDecode(String summaryClass, byte[] data, FastHit hit) { + public String lazyDecode(String summaryClass, byte[] data, FastHit hit) { ByteBuffer buffer = ByteBuffer.wrap(data); buffer.order(ByteOrder.LITTLE_ENDIAN); long docsumClassId = buffer.getInt(); @@ -83,6 +88,7 @@ public final class DocsumDefinitionSet { return null; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); for (Map.Entry<String, DocsumDefinition> e : definitionsByName.entrySet() ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 7c7f01b414a..094367dc140 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -377,7 +377,7 @@ public class FastHit extends Hit { } } - private static void appendAsHex(byte [] gid, StringBuilder sb) { + private static void appendAsHex(byte[] gid, StringBuilder sb) { for (byte b : gid) { String hex = Integer.toHexString(0xFF & b); if (hex.length() == 1) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 1a92e388f47..7abaf99c93d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -37,6 +37,9 @@ import java.util.logging.Logger; */ public abstract class VespaBackEndSearcher extends PingableSearcher { + /** for vespa-internal use only; consider renaming the summary class */ + public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; + private static final CompoundName TRACE_DISABLE = new CompoundName("trace.disable"); private String serverId; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java index 55adc3cba8d..0c4dff15744 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java @@ -22,9 +22,7 @@ public abstract class CompositeIndexedItem extends CompositeTaggableItem impleme private String index = ""; - /** - * The name of the index this belongs to, or "" (never null) if not specified - **/ + /** The name of the index this belongs to, or "" (never null) if not specified */ public String getIndexName() { return index; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java index 97cbfac0b2d..500c68cf4bd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java @@ -83,11 +83,16 @@ public class EquivItem extends CompositeTaggableItem { super.adding(item); Validator.ensure("Could not add an item of type " + item.getItemType() + ": Equiv can only have word, wordAlternatives, int, exact, or phrase as children", - item.getItemType() == ItemType.WORD || - item.getItemType() == ItemType.WORD_ALTERNATIVES || - item.getItemType() == ItemType.INT || - item.getItemType() == ItemType.EXACT || - item.getItemType() == ItemType.PHRASE); + acceptsChildrenOfType(item.getItemType())); + } + + /** Returns true if this accepts child items of the given type */ + public static boolean acceptsChildrenOfType(ItemType itemType) { + return itemType == ItemType.WORD || + itemType == ItemType.WORD_ALTERNATIVES || + itemType == ItemType.INT || + itemType == ItemType.EXACT || + itemType == ItemType.PHRASE; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 670983f0cd7..41d897b8d83 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -39,7 +39,7 @@ public abstract class Item implements Cloneable { WORD(4), INT(5), PHRASE(6), - PAREN(7), // TODO not used - remove on Vespa 8 + // 7 was PAREN, unused in Vespa 7 PREFIX(8), SUBSTRING(9), NEAR(11), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 8c4c5c84a28..a93dd1b9de4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -19,6 +20,8 @@ public class QueryCanonicalizer { /** The name of the operation performed by this, for use in search chain ordering */ public static final String queryCanonicalization = "queryCanonicalization"; + private static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems"); + /** * Validates this query and carries out possible operations on this query * which simplifies it without changing its semantics. @@ -26,7 +29,17 @@ public class QueryCanonicalizer { * @return null if the query is valid, an error message if it is invalid */ public static String canonicalize(Query query) { - return canonicalize(query.getModel().getQueryTree()); + Integer maxQueryItems = query.properties().getInteger(MAX_QUERY_ITEMS, Integer.MAX_VALUE); + return canonicalize(query.getModel().getQueryTree(), maxQueryItems); + } + + /** + * Canonicalizes this query, allowing any query tree size + * + * @return null if the query is valid, an error message if it is invalid + */ + public static String canonicalize(QueryTree queryTree) { + return canonicalize(queryTree, Integer.MAX_VALUE); } /** @@ -34,10 +47,12 @@ public class QueryCanonicalizer { * * @return null if the query is valid, an error message if it is invalid */ - public static String canonicalize(QueryTree query) { + private static String canonicalize(QueryTree query, Integer maxQueryItems) { ListIterator<Item> rootItemIterator = query.getItemIterator(); CanonicalizationResult result = recursivelyCanonicalize(rootItemIterator.next(), rootItemIterator); if (query.isEmpty() && ! result.isError()) result = CanonicalizationResult.error("No query"); + int itemCount = query.treeSize(); + if (itemCount > maxQueryItems) result = CanonicalizationResult.error(String.format("Query tree exceeds allowed item count. Configured limit: %d - Item count: %d", maxQueryItems, itemCount)); return result.error().orElse(null); // preserve old API, unfortunately } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 309befd80f5..fbbc746b130 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -3,35 +3,41 @@ package com.yahoo.prelude.query; /** - * A term type enumeration + * A term type enumeration. * * @author bratseth * @author Steinar Knutsen */ public class TermType { - public static final TermType RANK = new TermType("rank", RankItem.class, null, "$"); + public static final TermType RANK = new TermType("rank", Item.ItemType.RANK, RankItem.class, null, "$"); - public static final TermType AND = new TermType("and", AndItem.class, null, "+"); + public static final TermType AND = new TermType("and", Item.ItemType.AND, AndItem.class, null, "+"); - public static final TermType OR = new TermType("or", OrItem.class, null, "?"); + public static final TermType OR = new TermType("or", Item.ItemType.OR, OrItem.class, null, "?"); - public static final TermType NOT = new TermType("not", NotItem.class, null, "-"); + public static final TermType NOT = new TermType("not", Item.ItemType.NOT, NotItem.class, null, "-"); - public static final TermType PHRASE = new TermType("phrase", PhraseItem.class, null, "\""); + public static final TermType PHRASE = new TermType("phrase", Item.ItemType.PHRASE, PhraseItem.class, null, "\""); - public static final TermType EQUIV = new TermType("equiv", EquivItem.class, null, ""); + public static final TermType EQUIV = new TermType("equiv", Item.ItemType.EQUIV, EquivItem.class, null, ""); - public static final TermType DEFAULT = new TermType("", CompositeItem.class, AndItem.class, ""); + public static final TermType DEFAULT = new TermType("", Item.ItemType.AND, CompositeItem.class, AndItem.class, ""); public final String name; + private final Item.ItemType itemType; private final String sign; private final Class<? extends CompositeItem> instanceClass; private final Class<? extends CompositeItem> itemClass; - private TermType(String name, Class<? extends CompositeItem> itemClass, Class<? extends CompositeItem> instanceClass, String sign) { + private TermType(String name, + Item.ItemType itemType, + Class<? extends CompositeItem> itemClass, + Class<? extends CompositeItem> instanceClass, + String sign) { this.name = name; + this.itemType = itemType; this.itemClass = itemClass; if (instanceClass == null) { this.instanceClass = itemClass; @@ -73,6 +79,8 @@ public class TermType { return sign; } + public Item.ItemType toItemType() { return itemType; } + @Override public boolean equals(Object o) { if ( ! (o instanceof TermType)) return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 5fb90deaa0a..586d1d32d57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.query.parser; import com.yahoo.language.Language; import com.yahoo.language.process.Segmenter; -import com.yahoo.log.event.*; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 932f4ea7c38..5dcd591b1fc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,6 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -113,7 +114,8 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { - execution.fillAttributes(result); // Always correct as we can only sort on attributes + // TODO: remove or rename this internal summary class for Vespa 9 + execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 8e137d99951..4980b035876 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; -import com.yahoo.language.Language; import com.yahoo.language.Linguistics; -import com.yahoo.language.process.StemMode; -import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.rule.CompositeCondition; import com.yahoo.prelude.semantics.rule.Condition; import com.yahoo.prelude.semantics.rule.NamedCondition; @@ -76,12 +73,9 @@ public class RuleBase { */ private boolean usesAutomata = false; - private RuleBaseLinguistics linguistics; - /** Creates an empty rule base */ - public RuleBase(String name, Linguistics linguistics) { + public RuleBase(String name) { this.name = name; - this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics); } /** @@ -284,7 +278,7 @@ public class RuleBase { } /** - * Set to truew if this uses an automata, even if an automata is not present right now. + * Set to true if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } @@ -342,7 +336,7 @@ public class RuleBase { } // TODO: Values are not added right now - protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) { + protected void annotatePhrase(PhraseMatcher.Phrase phrase, Query query, int traceLevel) { for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) { String token = tokens.nextToken(); int semicolonIndex = token.indexOf(";"); @@ -357,12 +351,12 @@ public class RuleBase { phrase.getItem(0).addAnnotation(annotation, phrase); if (traceLevel >= 4) query.trace(" Annotating '" + phrase + "' as " + annotation + - (value.equals("") ? "" :"=" + value),false,1); + (value.equals("") ? "" :"=" + value),false,1); } } private void makeReferences() { - for (Iterator<ProductionRule> i=ruleIterator(); i.hasNext(); ) { + for (Iterator<ProductionRule> i = ruleIterator(); i.hasNext(); ) { ProductionRule rule = i.next(); rule.makeReferences(this); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index acbf9a7ffb6..6e7286cb8dc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -99,7 +99,7 @@ public class RuleImporter { reader = IOUtils.createReader(fileName, "utf-8"); File file = new File(fileName); String absoluteFileName = file.getAbsolutePath(); - var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics); + var ruleBase = new RuleBase(stripLastName(file.getName())); privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } @@ -205,7 +205,7 @@ public class RuleImporter { /** Imports an unitialized rule base */ public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); - RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics); + RuleBase ruleBase = new RuleBase(ruleBaseConfig.name()); return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), "semantic-rules.cfg", ruleBaseConfig.automata(),ruleBase); @@ -234,7 +234,7 @@ public class RuleImporter { public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { if (ruleBase == null) - ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics); + ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName); ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this); if (automataFile != null && !automataFile.isEmpty()) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index 85bcdf8bd20..d5ed89b0724 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.semantics.engine; import com.yahoo.prelude.query.*; +import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -32,11 +33,13 @@ public class Evaluation { /** The rule evaluation context, can be reset once the rule is evaluated */ private final RuleEvaluation ruleEvaluation; + private final RuleBase ruleBase; + /** * The amount of context information to collect about this evaluation. * 0 means no context information, higher numbers means more context information. */ - private int traceLevel = 0; + private final int traceLevel; private String traceIndentation = ""; @@ -49,8 +52,8 @@ public class Evaluation { /** Should we allow stemmed matches? */ private boolean stemming = true; - public Evaluation(Query query) { - this(query,0); + public Evaluation(Query query, RuleBase ruleBase) { + this(query, ruleBase, 0); } /** @@ -59,13 +62,17 @@ public class Evaluation { * @param query the query this evaluation is for * @param traceLevel the amount of tracing to do */ - public Evaluation(Query query, int traceLevel) { + public Evaluation(Query query, RuleBase ruleBase, int traceLevel) { this.query = query; + this.ruleBase = ruleBase; this.traceLevel = traceLevel; reset(); ruleEvaluation = new RuleEvaluation(this); } + /** Returns the rule base this evaluates over */ + public RuleBase ruleBase() { return ruleBase; } + /** Resets the item iterator to point to the first item */ public void reset() { if (flattenedItems != null) @@ -134,11 +141,11 @@ public class Evaluation { /** Adds an item to the query being evaluated in a way consistent with the query type */ // TODO: Add this functionality to Query? public void addItem(Item item, TermType termType) { - Item root= query.getModel().getQueryTree().getRoot(); - if (root==null) + Item root = query.getModel().getQueryTree().getRoot(); + if (root == null) query.getModel().getQueryTree().setRoot(item); else - query.getModel().getQueryTree().setRoot(combineItems(root,item,termType)); + query.getModel().getQueryTree().setRoot(combineItems(root, item, termType)); } /** Removes this item */ @@ -151,18 +158,18 @@ public class Evaluation { * equal items */ public void removeItemByIdentity(Item item) { - int position=findIndexByIdentity(item); - if (position>=0) + int position = findIndexByIdentity(item); + if (position >= 0) item.getParent().removeItem(position); else item.getParent().removeItem(item); // Fallback to removeField by equal() } private int findIndexByIdentity(Item item) { - int position=0; - for (Iterator<Item> i=item.getParent().getItemIterator(); i.hasNext(); ) { - Item child=i.next(); - if (item==child) { + int position = 0; + for (Iterator<Item> i = item.getParent().getItemIterator(); i.hasNext(); ) { + Item child = i.next(); + if (item == child) { return position; } position++; @@ -172,7 +179,7 @@ public class Evaluation { /** Removes an item, prefers the one at/close to the given position if there are multiple ones */ public void removeItem(int position,Item item) { - Item removeCandidate=item.getParent().getItem(position); + Item removeCandidate = item.getParent().getItem(position); if (removeCandidate.equals(item)) // Remove based on position item.getParent().removeItem(position); else @@ -189,7 +196,7 @@ public class Evaluation { if (!(item instanceof SegmentItem)) { return item; } - CompositeItem converted = null; + CompositeItem converted; if (item instanceof AndSegmentItem) { converted = new AndItem(); } else if (item instanceof PhraseSegmentItem) { @@ -238,16 +245,21 @@ public class Evaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert - * @param parent the parent of this item, or null to set the root - * @param index the index at which to insert this into the parent - * @param desiredParentType the desired type of the composite which contains item when this returns + * @param items the items to insert + * @param parent the parent of these items, or null to set the root + * @param index the index at which to insert these into the parent + * @param desiredParentType the desired type of the composite which contains items when this returns */ - public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType desiredParentType) { if (isEmpty(parent)) { - CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); - newParent.addItem(item); - query.getModel().getQueryTree().setRoot(newParent); + if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + query.getModel().getQueryTree().setRoot(items.get(0)); + } + else { + CompositeItem newParent = (CompositeItem) desiredParentType.createItemClass(); + items.forEach(item -> newParent.addItem(item)); + query.getModel().getQueryTree().setRoot(newParent); + } return; } @@ -260,34 +272,37 @@ public class Evaluation { } if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) ) - && equalIndexNameIfParentIsPhrase(item, parent)) { - addItem(parent, index, item, desiredParentType); + && equalIndexNameIfParentIsPhrase(items, parent)) { + for (Item item : items) + addItem(parent, index, item, desiredParentType); } - else if (incompatible(desiredParentType, parent)) { - insertIncompatibleItem(item, parent, query, desiredParentType); + else if (parent.items().isEmpty()) { + CompositeItem parentsParent = parent.getParent(); + CompositeItem newParent = newParent(desiredParentType); + items.forEach(item -> newParent.addItem(item)); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + } + else if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + addItem(parent, index, items.get(0), desiredParentType); } else { - insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + insertWithDesiredParentType(items, parent, desiredParentType); } } + /** Returns true if this item represents an empty query *tree*. */ private boolean isEmpty(Item item) { if (item == null) return true; if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; return false; } - /** Returns true if the desired type cannot have childCandidate as a child */ - private boolean incompatible(TermType desiredParentType, Item childCandidate) { - return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; - } - private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { if (parent instanceof NotItem) { if (index == 0 && parent.getItem(0) == null) { // Case 1: The current positive is null and we are adding a positive parent.setItem(0, item); } - else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite + else if (index <= 1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass(); positiveComposite.addItem(parent.getItem(0)); positiveComposite.addItem(index, item); @@ -313,50 +328,45 @@ public class Evaluation { } /** A special purpose check used to simplify the above */ - private boolean equalIndexNameIfParentIsPhrase(Item item,CompositeItem parent) { + private boolean equalIndexNameIfParentIsPhrase(List<Item> items, CompositeItem parent) { if ( ! (parent instanceof PhraseItem)) return true; - if ( ! (item instanceof IndexedItem)) return true; - - return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName()); - } - - private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); + var phrase = (PhraseItem)parent; - newParent.addItem(item); - parent.addItem(newParent); + for (Item item : items) { + if ( ! (item instanceof IndexedItem)) continue; + var indexedItem = (IndexedItem)item; + if (! indexedItem.getIndexName().equals(phrase.getIndexName())) return false; + } + return true; } - private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - // Create new parent - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); - - // Save previous parent parent + private void insertWithDesiredParentType(List<Item> items, CompositeItem parent, TermType desiredParentType) { CompositeItem parentsParent = parent.getParent(); - // Add items to new parent - newParent.addItem(parent); - newParent.addItem(item); + CompositeItem newParent = newParent(desiredParentType); + + if (! (parentsParent instanceof QueryTree) && parentsParent.getItemType() == newParent.getItemType()) { // Collapse + newParent = parentsParent; + } - // Insert new parent as root or child of old parents parent - if (parentsParent == null) { - query.getModel().getQueryTree().setRoot(newParent); + for (Item item : items) + newParent.addItem(item); + if (desiredParentType == TermType.EQUIV || desiredParentType == TermType.PHRASE) { // insert new parent below the current + parent.addItem(newParent); } - else { - parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + else { // insert new parent above the current + newParent.addItem(parent); + if (newParent != parentsParent) // Insert new parent as root or child of old parent's parent + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } - private Item combineItems(Item first,Item second,TermType termType) { + private CompositeItem newParent(TermType desiredParentType) { + return desiredParentType == TermType.DEFAULT ? new AndItem() : (CompositeItem)desiredParentType.createItemClass(); + } + + private Item combineItems(Item first, Item second, TermType termType) { if (first instanceof NullItem) { return second; } else if (first instanceof NotItem) { @@ -378,6 +388,10 @@ public class Evaluation { return composite; } else { + if (combined instanceof EquivItem) { + first = makeEquivCompatible(first); + second = makeEquivCompatible(second); + } combined.addItem(first); combined.addItem(second); // Also works for nots return combined; @@ -394,6 +408,22 @@ public class Evaluation { } } + private Item makeEquivCompatible(Item item) { + if (item instanceof AndItem || item instanceof WeakAndItem) { + PhraseItem phrase = new PhraseItem(); + List<Item> children = ((CompositeItem)item).items(); + if (children.isEmpty()) return phrase; + String index = ((IndexedItem)children.get(0)).getIndexName(); + for (var child : ((CompositeItem)item).items()) + phrase.addItem(child); + phrase.setIndexName(index); + return phrase; + } + else { + return item; // Compatible, or can't be made so + } + } + private CompositeItem createType(TermType termType) { if (termType == TermType.DEFAULT) { if (query.getModel().getType() == Query.Type.ANY) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 6098fbee327..6ed09926ccb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -60,7 +60,12 @@ public class Match { /** Returns a new item representing this match */ public Item toItem(String label) { - var newItem = new WordItem(getReplaceValue(), label); + return toItem(label, getReplaceValue()); + } + + /** Returns a new item representing this match */ + public Item toItem(String label, String term) { + var newItem = new WordItem(term, label); newItem.setWeight(item.getWeight()); return newItem; } @@ -81,4 +86,9 @@ public class Match { return true; } + @Override + public String toString() { + return "match of " + item + " at " + position + " to be replaced by " + replaceValue; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java index 605f3a23e10..b7083414e85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; /** - * The Matches referenced by a particular context name in a rule evaluation + * The matches referenced by a particular context name in a rule evaluation * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java index dd6610d1184..e2756afbb2e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java @@ -37,7 +37,7 @@ public class RuleEngine { // Probably create indices on the first term like Prolog implementations use to boolean matchedAnything = false; - Evaluation evaluation = new Evaluation(query, traceLevel); + Evaluation evaluation = new Evaluation(query, rules, traceLevel); if (traceLevel >= 2) evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':"); for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 2a00843a85c..1da16b4d277 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -190,8 +190,9 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { if (context == null) return; - if (matchReferences != null && matchReferences.contains(context)) + if (matchReferences != null && matchReferences.contains(context)) { currentContext = context; + } } public void leaving(String context) { @@ -209,7 +210,7 @@ public class RuleEvaluation { */ public void addMatch(FlattenedItem item, String replaceString) { evaluation.makeParentMutable(item.getItem()); - Match match = new Match(item,replaceString); + Match match = new Match(item, replaceString); if (currentContext != null) { ReferencedMatches matches = getReferencedMatches(currentContext); if (matches == null) { @@ -241,8 +242,8 @@ public class RuleEvaluation { public Evaluation getEvaluation() { return evaluation; } /** Adds an item to the query being evaluated in a way consistent with the query type */ - public void addItem(Item item, TermType termType) { - evaluation.addItem(item,termType); + public void addItems(List<Item> items, TermType termType) { + items.forEach(item -> evaluation.addItem(item, termType)); } public void removeItem(Item item) { @@ -262,13 +263,13 @@ public class RuleEvaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert + * @param items the items to insert * @param parent the parent of this item, or null to set the root * @param index the index at which to insert this into the parent * @param termType the kind of item to index, this decides the resulting structure */ - public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item, parent, index, termType); + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType termType) { + evaluation.insertItems(items, parent, index, termType); } /** Returns a read-only view of the items of this */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java index 77bffa5778e..a49272a73ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -14,28 +15,28 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public abstract class CompositeCondition extends Condition { - private List<Condition> conditions=new java.util.ArrayList<>(); + private final List<Condition> conditions = new java.util.ArrayList<>(); public CompositeCondition() { } public void preMatchHook(RuleEvaluation e) { super.preMatchHook(e); - if (e.getTraceLevel()>=3) { - e.trace(3,"Evaluating '" + this + "'" + " at " + e.currentItem()); + if (e.getTraceLevel() >= 3) { + e.trace(3, "Evaluating '" + this + "'" + " at " + e.currentItem()); e.indentTrace(); } } public void postMatchHook(RuleEvaluation e) { - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.unindentTrace(); } } protected boolean hasOpenChoicepoint(RuleEvaluation evaluation) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); if (subCondition.hasOpenChoicepoint(evaluation)) return true; } @@ -48,8 +49,8 @@ public abstract class CompositeCondition extends Condition { } /** Sets the condition at the given index */ - public void setCondition(int index,Condition condition) { - conditions.set(index,condition); + public void setCondition(int index, Condition condition) { + conditions.set(index, condition); } /** Returns the number of subconditions */ @@ -74,7 +75,7 @@ public abstract class CompositeCondition extends Condition { * @throws IndexOutOfBoundsException if there is no condition at this index */ public Condition removeCondition(int i) { - Condition condition=conditions.remove(i); + Condition condition = conditions.remove(i); condition.setParent(null); return condition; } @@ -82,20 +83,22 @@ public abstract class CompositeCondition extends Condition { /** Returns an iterator of the immediate children of this condition */ public Iterator<Condition> conditionIterator() { return conditions.iterator(); } + public List<Condition> conditions() { return Collections.unmodifiableList(conditions); } + public void makeReferences(RuleBase rules) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition condition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition condition = i.next(); condition.makeReferences(rules); } } /** Whether this should be output with parentheses, default is parent!=null */ protected boolean useParentheses() { - return getParent()!=null; + return getParent() != null; } protected String toInnerString(String conditionSeparator) { - if (getLabel()!=null) + if (getLabel() != null) return getLabel() + ":(" + conditionsToString(conditionSeparator) + ")"; else if (useParentheses()) return "(" + conditionsToString(conditionSeparator) + ")"; @@ -104,8 +107,8 @@ public abstract class CompositeCondition extends Condition { } protected final String conditionsToString(String conditionSeparator) { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { + StringBuilder buffer = new StringBuilder(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { buffer.append(i.next().toString()); if (i.hasNext()) buffer.append(conditionSeparator); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java index 0d3c93619b5..8d6016cb060 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java @@ -14,20 +14,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Condition { /** The parent of this condition, or null if this is not nested */ - private CompositeCondition parent=null; + private CompositeCondition parent = null; /** * The label of this condition, or null if none. * Specified by label:condition * The label is also the default context is no context is speficied explicitly */ - private String label=null; + private String label; /** * The name space refered by this match, or null if the default (query) * Specified by namespace.condition in rules. */ - private String nameSpace=null; + private String nameSpace = null; /** * The name of the context created by this, or null if none @@ -36,9 +36,9 @@ public abstract class Condition { private String contextName; /** Position constraints of the terms matched by this condition */ - private Anchor anchor=Anchor.NONE; + private Anchor anchor = Anchor.NONE; - public static enum Anchor { + public enum Anchor { NONE, START, END, BOTH; public static Anchor create(boolean start,boolean end) { if (start && end) return Anchor.BOTH; @@ -49,32 +49,32 @@ public abstract class Condition { } public Condition() { - this(null,null); + this(null, null); } public Condition(String label) { - this(label,null); + this(label, null); } - public Condition(String label,String context) { - this.label=label; - this.contextName=context; + public Condition(String label, String context) { + this.label = label; + this.contextName = context; } /** * Sets the name whatever is matched by this condition can be refered as, or null - * to make it unreferable + * to make it nonreferable */ - public void setContextName(String contextName) { this.contextName=contextName; } + public void setContextName(String contextName) { this.contextName = contextName; } /** - * Returns the name whatever is matched by this condition can be refered as, or null + * Returns the name whatever is matched by this condition can be referred as, or null * if it is unreferable */ public String getContextName() { return contextName; } /** Returns whether this is referable, returns context!=null by default */ - protected boolean isReferable() { return contextName!=null; } + protected boolean isReferable() { return contextName != null; } /** Sets the label of this. Set to null to use the default */ public String getLabel() { return label; } @@ -95,15 +95,15 @@ public abstract class Condition { void setParent(CompositeCondition parent) { this.parent=parent; } /** Sets a positional constraint on this condition */ - public void setAnchor(Anchor anchor) { this.anchor=anchor; } + public void setAnchor(Anchor anchor) { this.anchor = anchor; } /** Returns the positional constraint on this anchor. This is never null */ public Anchor getAnchor() { return anchor; } /** - * <p>Returns whether this condition matches the given evaluation + * Returns whether this condition matches the given evaluation * at the <i>current</i> location of the evaluation. Calls the doesMatch - * method of each condition subtype.</p> + * method of each condition subtype. */ public final boolean matches(RuleEvaluation e) { // TODO: With this algoritm, each choice point will move to the next choice on each reevaluation @@ -138,32 +138,32 @@ public abstract class Condition { /** Check start anchor. Trace level 4 if no match */ protected boolean matchesStartAnchor(RuleEvaluation e) { - if (anchor!=Anchor.START && anchor!=Anchor.BOTH) return true; - if (e.getPosition()==0) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the start, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.START && anchor != Anchor.BOTH) return true; + if (e.getPosition() == 0) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the start, which " + e.currentItem() + " isn't"); return false; } /** Check start anchor. Trace level 4 if no match */ protected boolean matchesEndAnchor(RuleEvaluation e) { - if (anchor!=Anchor.END && anchor!=Anchor.BOTH) return true; - if (e.getPosition()>=e.items().size()) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the end, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.END && anchor != Anchor.BOTH) return true; + if (e.getPosition() >= e.items().size()) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the end, which " + e.currentItem() + " isn't"); return false; } - protected void traceResult(boolean matches,RuleEvaluation e) { - if (matches && e.getTraceLevel()>=3) - e.trace(3,"Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); - if (!matches && e.getTraceLevel()>=4) - e.trace(4,"Did not match '" + this + "' at " + e.currentItem()); + protected void traceResult(boolean matches, RuleEvaluation e) { + if (matches && e.getTraceLevel() >= 3) + e.trace(3, "Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); + if (!matches && e.getTraceLevel() >= 4) + e.trace(4, "Did not match '" + this + "' at " + e.currentItem()); } protected String getMatchInfoString(RuleEvaluation e) { - String matchInfo=getMatchInfo(e); - if (matchInfo==null) return ""; + String matchInfo = getMatchInfo(e); + if (matchInfo == null) return ""; return " as '" + matchInfo + "'"; } @@ -212,28 +212,28 @@ public abstract class Condition { public void makeReferences(RuleBase rules) { } protected String getLabelString() { - if (label==null) return ""; + if (label == null) return ""; return label + ":"; } /** Whether the label matches the current item, true if there is no current item */ protected boolean labelMatches(RuleEvaluation e) { - FlattenedItem flattenedItem=e.currentItem(); - if (flattenedItem==null) return true; - TermItem item=flattenedItem.getItem(); - if (item==null) return true; - return labelMatches(item,e); - } - - protected boolean labelMatches(TermItem evaluationTerm,RuleEvaluation e) { - String indexName=evaluationTerm.getIndexName(); - String label=getLabel(); - if (label==null) - label=e.getCurrentLabel(); - if ("".equals(indexName) && label==null) return true; + FlattenedItem flattenedItem = e.currentItem(); + if (flattenedItem == null) return true; + TermItem item = flattenedItem.getItem(); + if (item == null) return true; + return labelMatches(item, e); + } + + protected boolean labelMatches(TermItem evaluationTerm, RuleEvaluation e) { + String indexName = evaluationTerm.getIndexName(); + String label = getLabel(); + if (label == null) + label = e.getCurrentLabel(); + if ("".equals(indexName) && label == null) return true; if (indexName.equals(label)) return true; - if (e.getTraceLevel()>=4) - e.trace(4,"'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); + if (e.getTraceLevel() >= 4) + e.trace(4, "'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); return false; } @@ -242,13 +242,14 @@ public abstract class Condition { protected boolean isDefaultContextName() { return false; } + @Override public String toString() { - String contextString=""; - String nameSpaceString=""; - if (contextName!=null && !isDefaultContextName()) - contextString=contextName + "/"; - if (getNameSpace()!=null) - nameSpaceString=getNameSpace() + "."; + String contextString = ""; + String nameSpaceString = ""; + if (contextName != null && !isDefaultContextName()) + contextString = contextName + "/"; + if (getNameSpace() != null) + nameSpaceString = getNameSpace() + "."; return contextString + nameSpaceString + toInnerString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java index bfddb55566c..e468526fbe0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.querytransform.PhraseMatcher; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -11,8 +10,6 @@ import com.yahoo.prelude.semantics.engine.FlattenedItem; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; -import java.util.Map; - /** * A reference to a named condition * @@ -24,7 +21,7 @@ public class ConditionReference extends Condition { private String conditionName; /** - * The actual condition references by this, or null if not initialized or not found, + * The actual condition referenced by this, or null if not initialized or not found, * or if this is really an automata reference */ private NamedCondition namedCondition; @@ -33,7 +30,7 @@ public class ConditionReference extends Condition { * True if this condition should be looked up in the automata * annotations of the item instead of by reference to another item */ - private boolean automataLookup=false; + private boolean automataLookup = false; public ConditionReference(String conditionName) { this(null,conditionName); @@ -42,37 +39,36 @@ public class ConditionReference extends Condition { public ConditionReference(String label,String conditionName) { super(label); Validator.ensureNotNull("Name of referenced condition",conditionName); - this.conditionName=conditionName; + this.conditionName = conditionName; setContextName(conditionName); } /** Returns the name of the referenced rule, never null */ public String getConditionName() { return conditionName; } - public void setConditionName(String name) { this.conditionName=name; } + public void setConditionName(String name) { this.conditionName = name; } public boolean doesMatch(RuleEvaluation e) { if (automataLookup) return automataMatch(e); - if (namedCondition==null) - throw new EvaluationException("Condition reference '" + conditionName + - "' not found or not initialized"); + if (namedCondition == null) + throw new EvaluationException("Condition reference '" + conditionName + "' not found or not initialized"); return namedCondition.matches(e); } private boolean automataMatch(RuleEvaluation e) { - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; - Object annotation=current.getItem().getAnnotation(conditionName); - if (annotation==null) return false; + Object annotation = current.getItem().getAnnotation(conditionName); + if (annotation == null) return false; if (! (annotation instanceof PhraseMatcher.Phrase)) return false; - PhraseMatcher.Phrase phrase=(PhraseMatcher.Phrase)annotation; + PhraseMatcher.Phrase phrase = (PhraseMatcher.Phrase)annotation; - Choicepoint choicePoint=e.getChoicepoint(this,true); - boolean matches=automataMatchPhrase(phrase,e); + Choicepoint choicePoint = e.getChoicepoint(this,true); + boolean matches = automataMatchPhrase(phrase,e); if (!matches && e.isInNegation()) { // TODO: Temporary hack! Works for single items only e.addMatch(current,null); @@ -84,40 +80,41 @@ public class ConditionReference extends Condition { return matches; } - private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase,RuleEvaluation e) { - for (PhraseMatcher.Phrase.MatchIterator i=phrase.itemIterator(); i.hasNext(); ) { + private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase, RuleEvaluation e) { + for (PhraseMatcher.Phrase.MatchIterator i = phrase.itemIterator(); i.hasNext(); ) { i.next(); - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; if (!labelMatches(e.currentItem().getItem(),e)) return false; if (!e.isInNegation()) - e.addMatch(current,i.getReplace()); + e.addMatch(current, i.getReplace()); e.next(); } - if (phrase.getLength()>phrase.getBackedLength()) return false; // The underlying composite item has changed + if (phrase.getLength() > phrase.getBackedLength()) return false; // The underlying composite item has changed return true; } public void makeReferences(RuleBase ruleBase) { - namedCondition=ruleBase.getCondition(conditionName); - if (namedCondition==null) { // Then this may reference some automata value, if we have an automata + namedCondition = ruleBase.getCondition(conditionName); + if (namedCondition == null) { // Then this may reference some automata value, if we have an automata if (ruleBase.usesAutomata()) - automataLookup=true; + automataLookup = true; else throw new RuleBaseException("Referenced condition '" + conditionName + - "' does not exist in " + ruleBase); + "' does not exist in " + ruleBase); } } protected boolean hasOpenChoicepoint(RuleEvaluation e) { - if (namedCondition==null) return false; + if (namedCondition == null) return false; return namedCondition.getCondition().hasOpenChoicepoint(e); } protected boolean isDefaultContextName() { - return getContextName()==null || getContextName().equals(conditionName); + return getContextName() == null || getContextName().equals(conditionName); } + @Override protected String toInnerString() { return "[" + conditionName + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java index 0d5ce78baf0..af7aab23b85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java @@ -44,7 +44,7 @@ public class LiteralPhraseProduction extends TermProduction { /** Returns a read only view of the terms produced by this, never null */ public List<String> getTerms() { return Collections.unmodifiableList(terms); } - public void produce(RuleEvaluation e,int offset) { + public void produce(RuleEvaluation e, int offset) { PhraseItem newPhrase = new PhraseItem(); newPhrase.setIndexName(getLabel()); for (String term : terms) @@ -52,13 +52,13 @@ public class LiteralPhraseProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); - insertMatch(e, matched, newPhrase, offset); + insertMatch(e, matched, List.of(newPhrase), offset); } else { newPhrase.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newPhrase + "'"); - e.addItem(newPhrase, getTermType()); + e.addItems(List.of(newPhrase), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java index ed21c9643c3..66507719a11 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A literal term produced by a production rule * @@ -63,13 +65,13 @@ public class LiteralTermProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); newItem.setWeight(matched.getItem().getWeight()); - insertMatch(e, matched, newItem, offset); + insertMatch(e, matched, List.of(newItem), offset); } else { newItem.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newItem + "'"); - e.addItem(newItem, getTermType()); + e.addItems(List.of(newItem), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java index a267d274d5a..cbb0b4ab10a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java @@ -33,7 +33,7 @@ public class NamedCondition { e.indentTrace(); } - boolean matches=condition.matches(e); + boolean matches = condition.matches(e); if (e.getTraceLevel() >= 3) { e.unindentTrace(); @@ -50,6 +50,7 @@ public class NamedCondition { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return "[" + conditionName + "] :- " + condition.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java index a971020ea90..debc3150959 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java @@ -13,20 +13,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Production { /** True to add, false to replace, default true */ - protected boolean replacing=true; + protected boolean replacing = true; /** The (0-base) position of this term in the productions of this rule */ - private int position=0; + private int position = 0; /** The weight (strength) of this production as a percentage (default is 100) */ - private int weight=100; + private int weight = 100; /** Creates a produced template term with no label and the default type */ public Production() { } /** True to replace, false to add, if this production can do both. Default true. */ - public void setReplacing(boolean replacing) { this.replacing=replacing; } + public void setReplacing(boolean replacing) { this.replacing = replacing; } public int getPosition() { return position; } @@ -45,7 +45,7 @@ public abstract class Production { * @param offset the offset position at which to produce this. Offsets are used to produce multiple items * at one position, inserted in the right order. */ - public abstract void produce(RuleEvaluation e,int offset); + public abstract void produce(RuleEvaluation e, int offset); /** * Called to add the references into the condition of this rule made by this production @@ -55,6 +55,7 @@ public abstract class Production { void addMatchReferences(Set<String> matchReferences) { } /** All instances of this produces a parseable string output */ + @Override public final String toString() { return toInnerString() + (getWeight()!=100 ? ("!" + getWeight()) : ""); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 46f855ca95d..5d20075cd97 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -54,6 +54,7 @@ public class ProductionList { } } + @Override public String toString() { StringBuilder buffer = new StringBuilder(); for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java index 20ebc41e57f..b320d10b43d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java @@ -19,18 +19,18 @@ public abstract class ProductionRule { private Condition condition; /** What is produced when this rule is true */ - private ProductionList production=new ProductionList(); + private ProductionList production = new ProductionList(); /** The set of match name Strings which the production part of this rule references */ - private Set<String> matchReferences=new java.util.LinkedHashSet<>(); + private final Set<String> matchReferences = new java.util.LinkedHashSet<>(); /** Sets what must be true for this rule to be true */ - public void setCondition(Condition condition) { this.condition=condition; } + public void setCondition(Condition condition) { this.condition = condition; } public Condition getCondition() { return condition; } /** Sets what is produced when this rule is true */ - public void setProduction(ProductionList production) { this.production=production; } + public void setProduction(ProductionList production) { this.production = production; } public ProductionList getProduction() { return production; } @@ -64,6 +64,7 @@ public abstract class ProductionRule { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return condition.toString() + " " + getSymbol() + " " + production.toString(); } @@ -80,21 +81,27 @@ public abstract class ProductionRule { * This default implementation returns false; */ public boolean isLoop() { - // TODO: There are many more possible loops, we should probably detect - // a few more obvious ones + // TODO: There are many more possible loops, we should probably detect a few more obvious ones if (conditionIsEllipsAndOtherNameSpacesOnly(getCondition())) return true; + if (producesItself()) return true; return false; } private boolean conditionIsEllipsAndOtherNameSpacesOnly(Condition condition) { if (condition instanceof EllipsisCondition) return true; if (! (condition instanceof CompositeCondition)) return false; - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition child= i.next(); - if (child.getNameSpace()==null && conditionIsEllipsAndOtherNameSpacesOnly(child)) + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition child = i.next(); + if (child.getNameSpace() == null && conditionIsEllipsAndOtherNameSpacesOnly(child)) return true; } return false; } + private boolean producesItself() { + return production.productionList() + .stream() + .anyMatch(p -> (p instanceof ReferenceTermProduction) && ((ReferenceTermProduction)p).producesAll()); + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index af7abf325e7..00562d3953e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -1,15 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; import java.util.Set; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.TermType; import com.yahoo.prelude.semantics.engine.EvaluationException; import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.ReferencedMatches; import com.yahoo.prelude.semantics.engine.RuleEvaluation; -import com.yahoo.protect.Validator; /** * A term produced by a production rule which takes its actual term value @@ -19,90 +23,114 @@ import com.yahoo.protect.Validator; */ public class ReferenceTermProduction extends TermProduction { - private String reference; + private final String reference; + private final boolean produceAll; /** * Creates a new produced reference term * * @param reference the label of the condition this should take it's value from */ - public ReferenceTermProduction(String reference) { + public ReferenceTermProduction(String reference, boolean produceAll) { super(); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of the term to produce */ - public ReferenceTermProduction(String reference, TermType termType) { + public ReferenceTermProduction(String reference, TermType termType, boolean produceAll) { super(termType); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from */ - public ReferenceTermProduction(String label, String reference) { + public ReferenceTermProduction(String label, String reference, boolean produceAll) { super(label); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of term to produce */ - public ReferenceTermProduction(String label, String reference, TermType termType) { + public ReferenceTermProduction(String label, String reference, TermType termType, boolean produceAll) { super(label, termType); - setReference(reference); - } - - /** The label of the condition this should take its value from, never null */ - public void setReference(String reference) { - Validator.ensureNotNull("reference name of a produced reference term",reference); - this.reference = reference; + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** Returns the label of the condition this should take its value from, never null */ public String getReference() { return reference; } + public boolean producesAll() { return produceAll; } + + @Override void addMatchReferences(Set<String> matchReferences) { matchReferences.add(reference); } - public void produce(RuleEvaluation e, int offset) { + public void produce(RuleEvaluation e, int ignored) { ReferencedMatches referencedMatches = e.getReferencedMatches(reference); if (referencedMatches == null) throw new EvaluationException("Referred match '" + reference + "' not found"); if (replacing) { - replaceMatches(e, referencedMatches); + e.removeMatches(referencedMatches); + } + + var match = referencedMatches.matchIterator().next(); + if (produceAll) { + // produce all terms in the condition + NamedCondition namedCondition = e.getEvaluation().ruleBase().getCondition(referencedMatches.getContextName()); + ChoiceCondition choices = (ChoiceCondition)namedCondition.getCondition(); + List<Item> items = new ArrayList<>(); + for (Iterator<Condition> i = choices.conditionIterator(); i.hasNext();) { + Condition condition = i.next(); + if (condition instanceof TermCondition) { + items.add(match.toItem(getLabel(), ((TermCondition)condition).term())); + } + else if (condition instanceof SequenceCondition) { + PhraseItem phrase = new PhraseItem(getLabel()); + for (var term : ((SequenceCondition)condition).conditions()) + phrase.addItem(match.toItem(getLabel(), ((TermCondition)term).term())); + items.add(phrase); + } + else { + // Until we validate this at construction time + throw new EvaluationException("Could not produce all terms in " + namedCondition + " as it is " + + "not a term or sequence condition"); + } + } + produce(e, match, items, 0); } else { - addMatches(e, referencedMatches); + // produce just the matching term + produce(e, match, List.of(referencedMatches.toItem(getLabel())), 0); } } - public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.removeMatches(referencedMatches); - insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0); - } - - private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.addItem(referencedItem, getTermType()); + private void produce(RuleEvaluation e, Match match, List<Item> items, int offset) { + if (replacing) + insertMatch(e, match, items, offset); + else + e.addItems(items, getTermType()); } + @Override public String toInnerTermString() { return getLabelString() + "[" + reference + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java index a2bbf72a53b..eaff66a0bb0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.semantics.engine.NameSpace; import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.engine.RuleEvaluation; @@ -49,6 +48,9 @@ public class TermCondition extends Condition { } } + public String term() { return term; } + + @Override public String toInnerString() { return getLabelString() + term; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index 29e4982ac17..41d15bc9262 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A new term produced by a production rule * @@ -59,9 +61,9 @@ public abstract class TermProduction extends Production { * Inserts newItem at the position of this match * TODO: Move to ruleevaluation */ - protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { + protected void insertMatch(RuleEvaluation e, Match matched, List<Item> newItems, int offset) { if (getWeight() != 100) - newItem.setWeight(getWeight()); + newItems.forEach(item -> item.setWeight(getWeight())); int insertPosition = matched.getPosition() + offset; // This check is necessary (?) because earlier items may have been removed @@ -71,9 +73,9 @@ public abstract class TermProduction extends Production { insertPosition = matched.getParent().getItemCount(); } - e.insertItem(newItem, matched.getParent(), insertPosition,getTermType()); + e.insertItems(newItems, matched.getParent(), insertPosition, getTermType()); if (e.getTraceLevel() >= 6) - e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " + + e.trace(6, "Inserted items '" + newItems + "' at position " + insertPosition + " producing " + e.getEvaluation().getQuery().getModel().getQueryTree()); } diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 1d1c446f6e1..6fae5c97cd2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -72,6 +72,7 @@ public class StatisticsSearcher extends Searcher { private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; private static final String RELEVANCE_AT_10_METRIC = "relevance.at_10"; + private static final String QUERY_ITEM_COUNT = "query_item_count"; @SuppressWarnings("unused") // all the work is done by the callback private final PeakQpsReporter peakQpsReporter; @@ -264,6 +265,8 @@ public class StatisticsSearcher extends Searcher { addRelevanceMetrics(query, execution, result); + addItemCountMetric(query, metricContext); + return result; } @@ -396,6 +399,10 @@ public class StatisticsSearcher extends Searcher { } } + private void addItemCountMetric(Query query, Metric.Context context) { + metric.set(QUERY_ITEM_COUNT, query.getModel().getQueryTree().treeSize(), context); + } + /** * Returns the relative start time from request was received by jdisc */ diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 623c38fa9f0..1c38bb8f876 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -235,18 +235,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private static final Map<String, CompoundName> propertyAliases; static { Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>(); - addAliases(Query.getArgumentType(), propertyAliasesBuilder); - addAliases(Ranking.getArgumentType(), propertyAliasesBuilder); - addAliases(Model.getArgumentType(), propertyAliasesBuilder); - addAliases(Presentation.getArgumentType(), propertyAliasesBuilder); - addAliases(Select.getArgumentType(), propertyAliasesBuilder); + addAliases(Query.getArgumentType(), CompoundName.empty, propertyAliasesBuilder); propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } - private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) { - CompoundName prefix = getPrefix(arguments); + private static void addAliases(QueryProfileType arguments, CompoundName prefix, Map<String, CompoundName> aliases) { for (FieldDescription field : arguments.fields().values()) { for (String alias : field.getAliases()) aliases.put(alias, prefix.append(field.getName())); + if (field.getType() instanceof QueryProfileFieldType) { + var type = ((QueryProfileFieldType) field.getType()).getQueryProfileType(); + if (type != null) + addAliases(type, prefix.append(type.getComponentIdAsCompoundName()), aliases); + } } } @@ -1048,6 +1048,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return new SessionId(requestId, getRanking().getProfile()); } + @Deprecated // TODO: Remove on Vespa 8 public boolean hasEncodableProperties() { if ( ! ranking.getProperties().isEmpty()) return true; if ( ! ranking.getFeatures().isEmpty()) return true; @@ -1064,39 +1065,29 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * @param buffer the buffer to encode to * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data * @return the encoded length + * @deprecated do not use */ + @Deprecated // TODO: Remove on Vespa 8 public int encodeAsProperties(ByteBuffer buffer, boolean encodeQueryData) { // Make sure we don't encode anything here if we have turned the property feature off // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of // property maps - that's ok (probably we should simplify by just always turning the feature on) if (! hasEncodableProperties()) return 0; - int start = buffer.position(); - int mapCountPosition = buffer.position(); buffer.putInt(0); // map count will go here - int mapCount = 0; - - // TODO: Push down mapCount += ranking.getProperties().encode(buffer, encodeQueryData); if (encodeQueryData) { mapCount += ranking.getFeatures().encode(buffer); - - // TODO: Push down if (presentation.getHighlight() != null) { mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer); } - - // TODO: Push down mapCount += MapEncoder.encodeMap("model", createModelMap(), buffer); } mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, model.getDocumentDb(), buffer); - mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(), buffer); - buffer.putInt(mapCountPosition, mapCount); - return buffer.position() - start; } @@ -1126,7 +1117,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Prepares this for binary serialization. - * <p> + * * This must be invoked after all changes have been made to this query before it is passed * on to a receiving backend. Calling it is somewhat expensive, so it should only happen once. * If a prepared query is cloned, it stays prepared. diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 016e22839c1..360dcd38d3b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -47,7 +47,7 @@ public class ClusterMonitor<T> { public ClusterMonitor(NodeManager<T> manager, boolean startPingThread) { nodeManager = manager; - monitorThread = new MonitorThread("search.clustermonitor"); + monitorThread = new MonitorThread("search.clustermonitor." + manager.name()); if (startPingThread) { monitorThread.start(); } @@ -81,7 +81,9 @@ public class ClusterMonitor<T> { /** * Returns the monitor of the given node, or null if this node has not been added + * @deprecated Will be removed in Vespa 8. */ + @Deprecated(forRemoval = true, since = "7.537") public BaseNodeMonitor<T> getNodeMonitor(T node) { return nodeMonitors.get(node); } @@ -147,6 +149,7 @@ public class ClusterMonitor<T> { private class MonitorThread extends Thread { MonitorThread(String name) { super(name); + setDaemon(true); } public void run() { diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 51ae2ee432d..097d714b47b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -51,7 +51,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod * @param internal whether or not this cluster is internal (part of the same installation) */ public ClusterSearcher(ComponentId id, List<T> connections, boolean internal) { - this(id, connections, new Hasher<T>(), internal); + this(id, connections, new Hasher<>(), internal); } public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal) { @@ -68,6 +68,9 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod } } + @Override + public String name() { return getIdString(); } + /** Pinging a node, called from ClusterMonitor */ @Override public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { @@ -112,7 +115,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (k == null) { b.append("null\n"); } else { - b.append(k.toString()).append('\n'); + b.append(k).append('\n'); } } traceAsString = b.toString(); @@ -303,7 +306,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod private class Pinger implements Callable<Pong> { - private T connection; + private final T connection; public Pinger(T connection) { this.connection = connection; diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 1a74194d694..db583a65606 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -11,6 +11,9 @@ import java.util.concurrent.Executor; */ public interface NodeManager<T> { + /** Name to identify Nodemanager */ + default String name() { return ""; } + /** Called when a failed node is working (ready for production) again */ void working(T node); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 3cf2903b88f..e09b1d51733 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -46,17 +46,12 @@ import java.util.stream.Collectors; public class Dispatcher extends AbstractComponent { public static final String DISPATCH = "dispatch"; - private static final String INTERNAL = "internal"; - private static final String PROTOBUF = "protobuf"; private static final String TOP_K_PROBABILITY = "topKProbability"; private static final String INTERNAL_METRIC = "dispatch_internal"; private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; - /** If enabled, search queries will use protobuf rpc */ - public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF); - /** If set will control computation of how many hits will be fetched from each partition.*/ public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY); @@ -79,8 +74,6 @@ public class Dispatcher extends AbstractComponent { argumentType = new QueryProfileType(DISPATCH); argumentType.setStrict(true); argumentType.setBuiltin(true); - argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType)); - argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType)); argumentType.addField(new FieldDescription(TOP_K_PROBABILITY, FieldType.doubleType)); argumentType.freeze(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 03b0f092abb..bd0415fa449 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -11,17 +11,17 @@ import java.util.Arrays; */ public class LeanHit implements Comparable<LeanHit> { - private final byte [] gid; + private final byte[] gid; private final double relevance; - private final byte [] sortData; + private final byte[] sortData; private final int partId; private final int distributionKey; private FeatureData matchFeatures; - public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { + public LeanHit(byte[] gid, int partId, int distributionKey, double relevance) { this(gid, partId, distributionKey, relevance, null); } - public LeanHit(byte [] gid, int partId, int distributionKey, double relevance, byte [] sortData) { + public LeanHit(byte[] gid, int partId, int distributionKey, double relevance, byte[] sortData) { this.gid = gid; this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; this.sortData = sortData; @@ -31,8 +31,8 @@ public class LeanHit implements Comparable<LeanHit> { } public double getRelevance() { return relevance; } - public byte [] getGid() { return gid; } - public byte [] getSortData() { return sortData; } + public byte[] getGid() { return gid; } + public byte[] getSortData() { return sortData; } public boolean hasSortData() { return sortData != null; } public int getPartId() { return partId; } public int getDistributionKey() { return distributionKey; } @@ -51,7 +51,7 @@ public class LeanHit implements Comparable<LeanHit> { return (res != 0) ? res : compareData(gid, o.gid); } - private static int compareData(byte [] left, byte [] right) { + private static int compareData(byte[] left, byte[] right) { int i = Arrays.mismatch(left, right); if (i < 0) { return 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index fe74180cad3..7a5ef94069d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -65,6 +65,11 @@ public class RpcProtobufFillInvoker extends FillInvoker { @Override protected void sendFillRequest(Result result, String summaryClass) { + if (! documentDb.getDocsumDefinitionSet().hasDocsum(summaryClass)) { + // TODO Vespa 8: + // throw new IllegalArgumentException("invalid summary="+summaryClass); + log.fine("invalid presentation.summary="+summaryClass); + } ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); result.getQuery().trace(false, 5, "Sending ", hitsByNode.size(), " summary fetch requests with jrt/protobuf"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index f90abba330a..36d7e7a85a9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -84,6 +84,9 @@ public class SearchCluster implements NodeManager<Node> { this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), nodesByHost, groups); } + @Override + public String name() { return clusterId; } + public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { for (var group : orderedGroups()) { for (var node : group.nodes()) diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 8925c647ad2..b65953935f0 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler; -import ai.vespa.cloud.Environment; -import ai.vespa.cloud.Zone; import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.ComponentSpecification; @@ -75,6 +73,7 @@ import java.util.logging.Logger; * @author Steinar Knutsen * @author bratseth */ +@SuppressWarnings("deprecation") // super class is deprecated public class SearchHandler extends LoggingRequestHandler { private static final Logger log = Logger.getLogger(SearchHandler.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index db6b07e0b0d..31641b5c2f0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -8,7 +8,9 @@ import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.search.Query; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.rendering.RendererRegistry; import java.util.ArrayList; @@ -30,19 +32,26 @@ public class Presentation implements Cloneable { public static final String TIMING = "timing"; public static final String SUMMARY = "summary"; public static final String SUMMARY_FIELDS = "summaryFields"; + public static final String TENSORS = "tensors"; /** The (short) name of the parameter holding the name of the return format to use */ public static final String FORMAT = "format"; static { - argumentType=new QueryProfileType(PRESENTATION); + argumentType = new QueryProfileType(PRESENTATION); argumentType.setStrict(true); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding")); argumentType.addField(new FieldDescription(TIMING, "boolean", "timing")); argumentType.addField(new FieldDescription(SUMMARY, "string", "summary")); - argumentType.addField(new FieldDescription(FORMAT, "string", "format template")); argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields")); + QueryProfileType formatArgumentType = new QueryProfileType(FORMAT); + formatArgumentType.setBuiltin(true); + formatArgumentType.setStrict(true); + formatArgumentType.addField(new FieldDescription("", "string", "format template")); + formatArgumentType.addField(new FieldDescription(TENSORS, "string", "format.tensors")); + formatArgumentType.freeze(); + argumentType.addField(new FieldDescription(FORMAT, new QueryProfileFieldType(formatArgumentType), "format")); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } @@ -53,7 +62,7 @@ public class Presentation implements Cloneable { /** The terms to highlight in the result (only used by BoldingSearcher, may be removed later). */ private List<IndexedItem> boldingData = null; - /** Whether or not to do highlighting */ + /** Whether to do highlighting */ private boolean bolding = true; /** The summary class to be shown */ @@ -65,6 +74,9 @@ public class Presentation implements Cloneable { /** Whether optional timing data should be rendered */ private boolean timing = false; + /** Whether to renders tensors in short form */ + private boolean tensorShortForm = false; + /** Set of explicitly requested summary fields, instead of summary classes */ private Set<String> summaryFields = LazySet.newHashSet(); @@ -98,14 +110,10 @@ public class Presentation implements Cloneable { this.format = (format != null) ? format : RendererRegistry.defaultRendererId.toSpecification(); } - /** - * Get the name of the format desired for result rendering. - */ + /** Get the name of the format desired for result rendering. */ public String getFormat() { return format.getName(); } - /** - * Set the desired format for result rendering. If null, use the default renderer. - */ + /** Set the desired format for result rendering. If null, use the default renderer. */ public void setFormat(String format) { setRenderer(ComponentSpecification.fromString(format)); } @@ -132,21 +140,7 @@ public class Presentation implements Cloneable { } } - @Override - public boolean equals(Object o) { - if ( ! (o instanceof Presentation)) return false; - Presentation p = (Presentation) o; - return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); - } - - @Override - public int hashCode() { - return QueryHelper.combineHash(bolding, summary); - } - - /** - * @return whether to add optional timing data to the rendered result - */ + /** Returns whether to add optional timing data to the rendered result. */ public boolean getTiming() { return timing; } @@ -166,20 +160,13 @@ public class Presentation implements Cloneable { return summaryFields; } - /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */ - public void prepare() { - if (highlight != null) - highlight.prepare(); - } - /** * Parse the given string as a comma delimited set of field names and * overwrite the set of summary fields. Whitespace will be trimmed. If you * want to add or remove fields programmatically, use * {@link #getSummaryFields()} and modify the returned set. * - * @param asString - * the summary fields requested, e.g. "price,author,title" + * @param asString the summary fields requested, e.g. "price,author,title" */ public void setSummaryFields(String asString) { summaryFields.clear(); @@ -189,5 +176,53 @@ public class Presentation implements Cloneable { } + /** + * Returns whether tensors should use short form in JSON and textual representations, see + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a> + * and <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form</a>. + * Default is false. + */ + public boolean getTensorShortForm() { return tensorShortForm; } + + /** + * Sets whether tensors should use short form in JSON and textual representations from a string. + * + * @param value a string which must be either 'short' or 'long' + * @throws IllegalArgumentException if any other value is passed + */ + public void setTensorShortForm(String value) { + tensorShortForm = toTensorShortForm(value); + } + + private boolean toTensorShortForm(String value) { + switch (value) { + case "short": return true; + case "long": return false; + default: throw new IllegalArgumentException("Value must be 'long' or 'short', not '" + value + "'"); + } + } + + public void setTensorShortForm(boolean tensorShortForm) { + this.tensorShortForm = tensorShortForm; + } + + /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */ + public void prepare() { + if (highlight != null) + highlight.prepare(); + } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Presentation)) return false; + Presentation p = (Presentation) o; + return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); + } + + @Override + public int hashCode() { + return QueryHelper.combineHash(bolding, summary); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index 2fb16ef503f..115a2f6dbdc 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -7,6 +7,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; /** * The root node of a query tree. This is always present above the actual semantic root to ease query manipulation, @@ -179,4 +180,23 @@ public class QueryTree extends CompositeItem { } } + /** + * Returns the total number of items in this query tree. + */ + public int treeSize() { + if (isEmpty()) return 0; + return(countItemsRecursively(getItemIterator().next())); + } + + private int countItemsRecursively(Item item) { + int children = 0; + if (item instanceof CompositeItem) { + CompositeItem composite = (CompositeItem)item; + for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) { + children += countItemsRecursively(i.next()); + } + } + return children + 1; + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 90d9eab687d..f58395fd5bb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -87,7 +87,6 @@ public class QueryProfileProperties extends Properties { */ @Override public void set(CompoundName name, Object value, Map<String, String> context) { - // TODO: Refactor try { name = unalias(name, context); @@ -101,42 +100,8 @@ public class QueryProfileProperties extends Properties { if (runtimeReference != null && ! runtimeReference.getSecond().isOverridable(name.rest(runtimeReference.getFirst().size()), context)) return; - // Check types - if ( ! profile.getTypes().isEmpty()) { - QueryProfileType type; - QueryProfileType explicitTypeFromField = null; - for (int i = 0; i < name.size(); i++) { - if (explicitTypeFromField != null) - type = explicitTypeFromField; - else - type = profile.getType(name.first(i), context); - if (type == null) continue; - - String localName = name.get(i); - FieldDescription fieldDescription = type.getField(localName); - if (fieldDescription == null && type.isStrict()) - throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); - - // TODO: In addition to strictness, check legality along the way - - if (fieldDescription != null) { - if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, new ConversionContext(localName, - profile.getRegistry(), - embedder, - context)); - if (value == null) - throw new IllegalInputException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); - } - else if (fieldDescription.getType() instanceof QueryProfileFieldType) { - // If a type is specified, use that instead of the type implied by the name - explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); - } - } - - } - } + if ( ! profile.getTypes().isEmpty()) + value = convertByType(name, value, context); if (value instanceof String && value.toString().startsWith("ref:")) { if (profile.getRegistry() == null) @@ -164,6 +129,53 @@ public class QueryProfileProperties extends Properties { } } + private Object convertByType(CompoundName name, Object value, Map<String, String> context) { + QueryProfileType type; + QueryProfileType explicitTypeFromField = null; + for (int i = 0; i < name.size(); i++) { + if (explicitTypeFromField != null) + type = explicitTypeFromField; + else + type = profile.getType(name.first(i), context); + if (type == null) continue; + + String localName = name.get(i); + FieldDescription fieldDescription = type.getField(localName); + if (fieldDescription == null && type.isStrict()) + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + // TODO: In addition to strictness, check legality along the way + + if (fieldDescription != null) { + if (i == name.size() - 1) { // at the end of the path, check the assignment type + var conversionContext = new ConversionContext(localName, profile.getRegistry(), embedder, context); + var convertedValue = fieldDescription.getType().convertFrom(value, conversionContext); + if (convertedValue == null + && fieldDescription.getType() instanceof QueryProfileFieldType + && ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType() != null) { + // Try the value of the query profile itself instead + var queryProfileValueDescription = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType().getField(""); + if (queryProfileValueDescription != null) { + convertedValue = queryProfileValueDescription.getType().convertFrom(value, conversionContext); + if (convertedValue == null) + throw new IllegalInputException("'" + value + "' is neither a " + + fieldDescription.getType().toInstanceDescription() + " nor a " + + queryProfileValueDescription.getType().toInstanceDescription()); + } + } else if (convertedValue == null) + throw new IllegalInputException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); + + value = convertedValue; + } else if (fieldDescription.getType() instanceof QueryProfileFieldType) { + // If a type is specified, use that instead of the type implied by the name + explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } + } + + } + return value; + } + @Override public void clearAll(CompoundName name, Map<String, String> context) { if (references == null) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index fb1552e549b..39ec3e1c647 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -10,6 +10,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.yolean.UncheckedInterruptedException; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; /** * A set of compiled query profiles. @@ -24,14 +29,47 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; @Inject - public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + public CompiledQueryProfileRegistry(QueryProfilesConfig config, Executor executor) { QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); typeRegistry = registry.getTypeRegistry(); - for (QueryProfile inputProfile : registry.allComponents()) { - register(QueryProfileCompiler.compile(inputProfile, this)); + int maxConcurrent = 1; // TODO hold this one after Concurrency issue has been found: Math.max(1, (int)(Runtime.getRuntime().availableProcessors() * 0.20)); + BlockingQueue<CompiledQueryProfile> doneQ = new LinkedBlockingQueue<>(); + int started = 0; + int completed = 0; + try { + for (QueryProfile inputProfile : registry.allComponents()) { + abortIfInterrupted(); + if (started++ >= maxConcurrent) { + register(doneQ.take()); + completed++; + } + executor.execute(() -> { + Thread self = Thread.currentThread(); + int prevPriority = self.getPriority(); + try { + self.setPriority(Thread.MIN_PRIORITY); + doneQ.add(QueryProfileCompiler.compile(inputProfile, this)); + } finally { + self.setPriority(prevPriority); + } + }); + } + while (completed < started) { + register(doneQ.take()); + completed++; + } + } catch (InterruptedException e) { + throw new UncheckedInterruptedException("Interrupted while waiting for compiled query profiles", true); } } + // Query profile construction is very expensive and triggers no operations that automatically throws on interrupt + // We need to manually check the interrupt flag in case the container reconfigurer should shut down + private void abortIfInterrupted() { + if (Thread.interrupted()) + throw new UncheckedInterruptedException("Interrupted while building query profile registry", true); + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index ceeed9d9167..c3bd4f7b962 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -19,6 +19,7 @@ import java.util.Set; /** * @author bratseth */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber<QueryProfilesConfig> { private final ConfigSubscriber subscriber = new ConfigSubscriber(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 1d6f584bfaa..a0f38f61672 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -81,7 +81,7 @@ public class FieldDescription implements Comparable<FieldDescription> { /** * Creates a field description * - * @param name the name of the field + * @param name the name of the field, empty means it describes the value held by the query profile itself * @param type the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType} * @param aliases a list of aliases, never null. Aliases are not following dotted * (meaning they are global, not that they cannot contain dots) and are case insensitive. @@ -89,8 +89,6 @@ public class FieldDescription implements Comparable<FieldDescription> { * @param overridable whether this can be overridden when first set in a profile. Default: true */ public FieldDescription(CompoundName name, FieldType type, List<String> aliases, boolean mandatory, boolean overridable) { - if (name.isEmpty()) - throw new IllegalArgumentException("Illegal name ''"); for (String nameComponent : name.asList()) QueryProfile.validateName(nameComponent); this.name = name; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java index 87f39b88981..1b036b96e01 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java @@ -38,7 +38,6 @@ public class PropertyAliases extends Properties { */ protected CompoundName unalias(CompoundName nameOrAlias) { if (aliases.isEmpty()) return nameOrAlias; - if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName()); return (properName != null) ? properName : nameOrAlias; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 9a09c23b23b..243915662d2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -122,12 +122,16 @@ public class QueryProperties extends Properties { if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString(); } } - else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { - if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); - if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); - if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); - if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); + else if (key.first().equals(Presentation.PRESENTATION)) { + if (key.size() == 2) { + if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); + if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); + if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); + if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); + if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); + } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { + if (key.last().equals(Presentation.TENSORS)) return query.getPresentation().getTensorShortForm(); + } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride")) { // featureoverride is deprecated return query.getRanking().getFeatures().getObject(key.rest().toString()); @@ -273,17 +277,27 @@ public class QueryProperties extends Properties { throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } } - else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { - if (key.last().equals(Presentation.BOLDING)) - query.getPresentation().setBolding(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY)) - query.getPresentation().setSummary(asString(value, "")); - else if (key.last().equals(Presentation.FORMAT)) - query.getPresentation().setFormat(asString(value,"")); - else if (key.last().equals(Presentation.TIMING)) - query.getPresentation().setTiming(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY_FIELDS)) - query.getPresentation().setSummaryFields(asString(value,"")); + else if (key.first().equals(Presentation.PRESENTATION)) { + if (key.size() == 2) { + if (key.last().equals(Presentation.BOLDING)) + query.getPresentation().setBolding(asBoolean(value, true)); + else if (key.last().equals(Presentation.SUMMARY)) + query.getPresentation().setSummary(asString(value, "")); + else if (key.last().equals(Presentation.FORMAT)) + query.getPresentation().setFormat(asString(value, "")); + else if (key.last().equals(Presentation.TIMING)) + query.getPresentation().setTiming(asBoolean(value, true)); + else if (key.last().equals(Presentation.SUMMARY_FIELDS)) + query.getPresentation().setSummaryFields(asString(value, "")); + else + throwIllegalParameter(key.last(), Presentation.PRESENTATION); + } + else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { + if (key.last().equals(Presentation.TENSORS)) + query.getPresentation().setTensorShortForm(asString(value, "")); + else + throwIllegalParameter(key.last(), Presentation.FORMAT); + } else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java index 83daf6398c3..fff8935eefb 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java @@ -5,17 +5,20 @@ import static com.yahoo.prelude.searcher.PosSearcher.POSITION_PARSING; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.search.Query; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.google.inject.Inject; import java.util.List; /** - * If default position has not been set, it will be set here. + * If position attribute has not been set, it will be set here. * * @author baldersheim */ @@ -23,6 +26,17 @@ import java.util.List; @Before(PhaseNames.TRANSFORMED_QUERY) public class DefaultPositionSearcher extends Searcher { + private final boolean useV8GeoPositions; + + @Inject + public DefaultPositionSearcher(DocumentmanagerConfig cfg) { + this.useV8GeoPositions = cfg.usev8geopositions(); + } + + DefaultPositionSearcher() { + this.useV8GeoPositions = false; + } + @Override public com.yahoo.search.Result search(Query query, Execution execution) { Location location = query.getRanking().getLocation(); @@ -40,6 +54,12 @@ public class DefaultPositionSearcher extends Searcher { location.setAttribute(facts.getDefaultPosition(null)); } } + if (useV8GeoPositions && (location != null) && (location.getAttribute() != null)) { + var geoLoc = new GeoLocationItem(location); + query.getModel().getQueryTree().and(geoLoc); + location = null; + query.getRanking().setLocation(location); + } return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index f9a91558254..2ba507a83b8 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -76,10 +76,10 @@ import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WR // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { - private static final CompoundName WRAP_ALL_MAPS = new CompoundName("renderer.json.jsonMaps"); + private static final CompoundName WRAP_DEEP_MAPS = new CompoundName("renderer.json.jsonMaps"); + private static final CompoundName WRAP_WSETS = new CompoundName("renderer.json.jsonWsets"); private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); - private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors"); // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; @@ -125,18 +125,46 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private JsonGenerator generator; private FieldConsumer fieldConsumer; private Deque<Integer> renderedChildren; - private boolean debugRendering; - private boolean jsonMaps; + static class FieldConsumerSettings { + boolean debugRendering = false; + boolean jsonDeepMaps = false; + boolean jsonWsets = false; + boolean jsonMapsAll = false; + boolean jsonWsetsAll = false; + boolean tensorShortForm = false; + boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } + void init() { + this.debugRendering = false; + this.jsonDeepMaps = false; + this.jsonWsets = false; + this.jsonMapsAll = false; + this.jsonWsetsAll = false; + this.tensorShortForm = false; + } + void getSettings(Query q) { + if (q == null) { + init(); + return; + } + var props = q.properties(); + this.debugRendering = props.getBoolean(DEBUG_RENDERING_KEY, false); + this.jsonDeepMaps = props.getBoolean(WRAP_DEEP_MAPS, false); + this.jsonWsets = props.getBoolean(WRAP_WSETS, false); + // we may need more fine tuning, but for now use the same query parameters here: + this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, false); + this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, false); + this.tensorShortForm = q.getPresentation().getTensorShortForm(); + } + } + private final FieldConsumerSettings fieldConsumerSettings = new FieldConsumerSettings(); private LongSupplier timeSource; private OutputStream stream; - private boolean tensorShortFormRendering = false; - public JsonRenderer() { this(null); } - /** + /** * Creates a json renderer using a custom executor. * Using a custom executor is useful for tests to avoid creating new threads for each renderer registry. */ @@ -151,8 +179,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * method will be invoked when creating the first renderer instance, but not * for each fresh clone used by individual results. * + * @deprecated Will be removed in Vespa 8. Override the individual render methods of {@link JsonRenderer} to alter + * rendering behaviour. Override {@link #createFieldConsumer(boolean)} and sub-class {@link FieldConsumer} + * to alter rendering of hit fields. * @return an object mapper for the internal JsonFactory */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 make private protected static ObjectMapper createJsonCodec() { return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); } @@ -160,9 +192,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void init() { super.init(); - debugRendering = false; - jsonMaps = false; - setGenerator(null, debugRendering); + fieldConsumerSettings.init(); + setGenerator(null, fieldConsumerSettings); renderedChildren = null; timeSource = System::currentTimeMillis; stream = null; @@ -171,10 +202,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); - debugRendering = getDebugRendering(getResult().getQuery()); - jsonMaps = getWrapAllMaps(getResult().getQuery()); - tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery()); - setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); + fieldConsumerSettings.getSettings(getResult().getQuery()); + setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), fieldConsumerSettings); renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); renderTrace(getExecution().trace()); @@ -204,20 +233,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeEndObject(); } - private boolean getWrapAllMaps(Query q) { - return q != null && q.properties().getBoolean(WRAP_ALL_MAPS, false); - } - - private boolean getDebugRendering(Query q) { - return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false); - } - - private boolean getTensorShortFormRendering(Query q) { - if (q == null || q.properties().get(TENSOR_FORMAT) == null) - return false; - return q.properties().getString(TENSOR_FORMAT).equalsIgnoreCase("short"); - } - protected void renderTrace(Trace trace) throws IOException { if (!trace.traceNode().children().iterator().hasNext()) return; if (getResult().getQuery().getTraceLevel() == 0) return; @@ -520,17 +535,26 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return null; } - private void setGenerator(JsonGenerator generator, boolean debugRendering) { + private void setGenerator(JsonGenerator generator, FieldConsumerSettings settings) { this.generator = generator; - this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, debugRendering, jsonMaps); + this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, settings); } + /** Override this method to use a custom {@link FieldConsumer} sub-class to render fields */ + protected FieldConsumer createFieldConsumer(boolean debugRendering) { + fieldConsumerSettings.debugRendering = debugRendering; + return createFieldConsumer(generator, fieldConsumerSettings); + } + + /** @deprecated Will be removed in Vespa 8. Use {@link #createFieldConsumer(boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove method protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) { - return createFieldConsumer(generator, debugRendering, this.jsonMaps); + fieldConsumerSettings.debugRendering = debugRendering; + return createFieldConsumer(generator, fieldConsumerSettings); } - private FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering, boolean jsonMaps) { - return new FieldConsumer(generator, debugRendering, tensorShortFormRendering, jsonMaps); + private FieldConsumer createFieldConsumer(JsonGenerator generator, FieldConsumerSettings settings) { + return new FieldConsumer(generator, settings); } /** @@ -548,23 +572,39 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer { private final JsonGenerator generator; - private final boolean debugRendering; - private final boolean jsonMaps; - private final boolean tensorShortForm; - + private final FieldConsumerSettings settings; private MutableBoolean hasFieldsField; + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove public FieldConsumer(JsonGenerator generator, boolean debugRendering) { this(generator, debugRendering, false); } + + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) { this(generator, debugRendering, tensorShortForm, false); } + + /** Invoke this from your constructor when sub-classing {@link FieldConsumer} */ + protected FieldConsumer(boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { + this(null, debugRendering, tensorShortForm, jsonMaps); + } + + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { this.generator = generator; - this.debugRendering = debugRendering; - this.tensorShortForm = tensorShortForm; - this.jsonMaps = jsonMaps; + this.settings = new FieldConsumerSettings(); + this.settings.debugRendering = debugRendering; + this.settings.tensorShortForm = tensorShortForm; + this.settings.jsonDeepMaps = jsonMaps; + } + + FieldConsumer(JsonGenerator generator, FieldConsumerSettings settings) { + this.generator = generator; + this.settings = settings; } /** @@ -578,14 +618,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { /** Call before rendering a field to the generator */ void ensureFieldsField() throws IOException { if (hasFieldsField.get()) return; - generator.writeObjectFieldStart(FIELDS); + generator().writeObjectFieldStart(FIELDS); hasFieldsField.set(true); } /** Call after all fields in a hit to close the "fields" field of the JSON object */ void endHitFields() throws IOException { if ( ! hasFieldsField.get()) return; - generator.writeEndObject(); + generator().writeEndObject(); this.hasFieldsField = null; } @@ -594,7 +634,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { try { if (shouldRender(name, value)) { ensureFieldsField(); - generator.writeFieldName(name); + generator().writeFieldName(name); renderFieldContents(value); } } @@ -608,8 +648,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { try { if (shouldRenderUtf8Value(name, length)) { ensureFieldsField(); - generator.writeFieldName(name); - generator.writeUTF8String(utf8Data, offset, length); + generator().writeFieldName(name); + generator().writeUTF8String(utf8Data, offset, length); } } catch (IOException e) { @@ -618,7 +658,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected boolean shouldRender(String name, Object value) { - if (debugRendering) return true; + if (settings.debugRendering) return true; if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; // StringFieldValue cannot hold a null, so checking length directly is OK: @@ -628,47 +668,68 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected boolean shouldRenderUtf8Value(String name, int length) { - if (debugRendering) return true; + if (settings.debugRendering) return true; if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; if (length == 0) return false; return true; } - private static Inspector deepWrapAsMap(Inspector data) { - if (data.type() == Type.ARRAY) { - var map = new Value.ObjectValue(); - for (int i = 0; i < data.entryCount(); i++) { - Inspector obj = data.entry(i); - if (map != null && obj.type() == Type.OBJECT && obj.fieldCount() == 2) { - Inspector key = obj.field("key"); - Inspector value = obj.field("value"); - if (key.type() == Type.STRING && value.valid()) { - map.put(key.asString(), deepWrapAsMap(value)); - } else { - map = null; - } - } else { - map = null; - } + private Inspector maybeConvertMap(Inspector data) { + var map = new Value.ObjectValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + if (obj.type() != Type.OBJECT || obj.fieldCount() != 2) { + return null; + } + Inspector key = obj.field("key"); + Inspector value = obj.field("value"); + if (! key.valid()) return null; + if (! value.valid()) return null; + if (key.type() != Type.STRING && !settings.jsonMapsAll) { + return null; } - if (map != null) { - return map; + if (settings.convertDeep()) { + value = deepMaybeConvert(value); } - var array = new Value.ArrayValue(); - for (int i = 0; i < data.entryCount(); i++) { - Inspector obj = data.entry(i); - array.add(deepWrapAsMap(obj)); + if (key.type() == Type.STRING) { + map.put(key.asString(), value); + } else { + map.put(key.toString(), value); } - return array; } - if (data.type() == Type.OBJECT) { - var object = new Value.ObjectValue(); - for (var entry : data.fields()) { - object.put(entry.getKey(), deepWrapAsMap(entry.getValue())); + return map; + } + + private Inspector maybeConvertWset(Inspector data) { + var wset = new Value.ObjectValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + if (obj.type() != Type.OBJECT || obj.fieldCount() != 2) { + return null; + } + Inspector item = obj.field("item"); + Inspector weight = obj.field("weight"); + if (! item.valid()) return null; + if (! weight.valid()) return null; + // TODO support non-integer weights? + if (weight.type() != Type.LONG) return null; + if (item.type() == Type.STRING) { + wset.put(item.asString(), weight.asLong()); + } else if (settings.jsonWsetsAll) { + wset.put(item.toString(), weight.asLong()); + } else { + return null; } - return object; } - return data; + return wset; + } + + private Inspector convertInsideObject(Inspector data) { + var object = new Value.ObjectValue(); + for (var entry : data.fields()) { + object.put(entry.getKey(), deepMaybeConvert(entry.getValue())); + } + return object; } private static Inspector wrapAsMap(Inspector data) { @@ -688,19 +749,61 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return map; } - private void renderInspector(Inspector data) throws IOException { - Inspector asMap = jsonMaps ? deepWrapAsMap(data) : wrapAsMap(data); - if (asMap != null) { - renderInspectorDirect(asMap); - } else { - renderInspectorDirect(data); + private Inspector deepMaybeConvert(Inspector data) { + if (data.type() == Type.ARRAY) { + if (settings.jsonDeepMaps) { + var map = maybeConvertMap(data); + if (map != null) return map; + } + if (settings.jsonWsets) { + var wset = maybeConvertWset(data); + if (wset != null) return wset; + } + } + if (data.type() == Type.OBJECT) { + return convertInsideObject(data); + } + return data; + } + + private Inspector convertTopLevelArray(Inspector data) { + if (data.entryCount() > 0) { + var map = maybeConvertMap(data); + if (map != null) return map; + if (settings.jsonWsets) { + var wset = maybeConvertWset(data); + if (wset != null) return wset; + } + if (settings.convertDeep()) { + var array = new Value.ArrayValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + array.add(deepMaybeConvert(obj)); + } + return array; + } } + return data; + } + + private Inspector maybeConvertData(Inspector data) throws IOException { + if (data.type() == Type.ARRAY) { + return convertTopLevelArray(data); + } + if (settings.convertDeep() && data.type() == Type.OBJECT) { + return convertInsideObject(data); + } + return data; + } + + private void renderInspector(Inspector data) throws IOException { + renderInspectorDirect(maybeConvertData(data)); } private void renderInspectorDirect(Inspector data) throws IOException { StringBuilder intermediate = new StringBuilder(); JsonRender.render(data, intermediate, true); - generator.writeRawValue(intermediate.toString()); + generator().writeRawValue(intermediate.toString()); } protected void renderFieldContents(Object field) throws IOException { @@ -714,68 +817,75 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void accept(Object field) throws IOException { if (field == null) { - generator.writeNull(); + generator().writeNull(); } else if (field instanceof Boolean) { - generator.writeBoolean((Boolean)field); + generator().writeBoolean((Boolean)field); } else if (field instanceof Number) { renderNumberField((Number) field); } else if (field instanceof TreeNode) { - generator.writeTree((TreeNode) field); + generator().writeTree((TreeNode) field); } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); } else if (field instanceof FeatureData) { - generator.writeRawValue(((FeatureData)field).toJson(tensorShortForm)); + generator().writeRawValue(((FeatureData)field).toJson(settings.tensorShortForm)); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { - generator.writeRawValue(((JsonProducer) field).toJson()); + generator().writeRawValue(((JsonProducer) field).toJson()); } else if (field instanceof StringFieldValue) { - generator.writeString(((StringFieldValue)field).getString()); + generator().writeString(((StringFieldValue)field).getString()); } else if (field instanceof TensorFieldValue) { renderTensor(((TensorFieldValue)field).getTensor()); } else if (field instanceof FieldValue) { // the null below is the field which has already been written ((FieldValue) field).serialize(null, new JsonWriter(generator)); } else { - generator.writeString(field.toString()); + generator().writeString(field.toString()); } } private void renderNumberField(Number field) throws IOException { if (field instanceof Integer) { - generator.writeNumber(field.intValue()); + generator().writeNumber(field.intValue()); } else if (field instanceof Float) { - generator.writeNumber(field.floatValue()); + generator().writeNumber(field.floatValue()); } else if (field instanceof Double) { - generator.writeNumber(field.doubleValue()); + generator().writeNumber(field.doubleValue()); } else if (field instanceof Long) { - generator.writeNumber(field.longValue()); + generator().writeNumber(field.longValue()); } else if (field instanceof Byte || field instanceof Short) { - generator.writeNumber(field.intValue()); + generator().writeNumber(field.intValue()); } else if (field instanceof BigInteger) { - generator.writeNumber((BigInteger) field); + generator().writeNumber((BigInteger) field); } else if (field instanceof BigDecimal) { - generator.writeNumber((BigDecimal) field); + generator().writeNumber((BigDecimal) field); } else { - generator.writeNumber(field.doubleValue()); + generator().writeNumber(field.doubleValue()); } } private void renderTensor(Optional<Tensor> tensor) throws IOException { if (tensor.isEmpty()) { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - generator.writeEndArray(); - generator.writeEndObject(); + generator().writeStartObject(); + generator().writeArrayFieldStart("cells"); + generator().writeEndArray(); + generator().writeEndObject(); return; } - if (tensorShortForm) { - generator.writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); + if (settings.tensorShortForm) { + generator().writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); } else { - generator.writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); + generator().writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); } } + private JsonGenerator generator() { + if (generator == null) + throw new UnsupportedOperationException("Generator required but not assigned. " + + "All accept() methods must be overridden when sub-classing FieldConsumer"); + return generator; + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index c507069b948..9374027504e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,6 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -18,7 +19,6 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.Renderer; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; - import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -41,6 +41,8 @@ import java.util.concurrent.Executors; */ public class Execution extends com.yahoo.processing.execution.Execution { + /** @deprecated - applications should define their own summary class instead */ + @Deprecated(since = "7", forRemoval = true) public static final String ATTRIBUTEPREFETCH = "attributeprefetch"; /** @@ -532,18 +534,23 @@ public class Execution extends com.yahoo.processing.execution.Execution { } /** - * Fill hit properties with values from all in-memory attributes. - * This can be done with good performance on many more hits than - * those for which fill is called with the final summary class, so - * if filtering can be done using only in-memory attribute data, - * this method should be preferred over {@link #fill} to get that data for filtering. - * <p> - * Calling this on already filled results has no cost. + * Fill hit properties with values from some in-memory attributes. + * Not all attributes are included, and *which* attributes are + * subject to change depending on what Vespa needs internally. + * + * Applications should prefer to define their own summary class + * with only the in-memory attributes they need, and call + * fill(result, "foo") with the name of their own summary class + * instead of "foo". * + * @deprecated use fill(Result, String) + * + * TODO Remove on Vespa 9. * @param result the result to fill */ + @Deprecated public void fillAttributes(Result result) { - fill(result, ATTRIBUTEPREFETCH); + fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 13a9f9510cd..2dfa6ef3e3a 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -95,7 +96,7 @@ public class FieldFiller extends Searcher { for (Documentdb db : config.documentdb()) { for (Summaryclass summary : db.summaryclass()) { Set<String> attributes; - if (Execution.ATTRIBUTEPREFETCH.equals(summary.name())) { + if (SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summary.name())) { attributes = new HashSet<>(summary.fields().size()); for (Fields f : summary.fields()) { attributes.add(f.name()); @@ -134,11 +135,11 @@ public class FieldFiller extends Searcher { } if (intersectionOfAttributes.containsAll(summaryFields)) { - if ( ! Execution.ATTRIBUTEPREFETCH.equals(summaryClass)) { - execution.fill(result, Execution.ATTRIBUTEPREFETCH); + if (! SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summaryClass)) { + execution.fill(result, SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } } else { - // Yes, summaryClass may be Execution.ATTRIBUTEPREFETCH here + // Yes, summaryClass may be SORTABLE_ATTRIBUTES_SUMMARY_CLASS here if ( ! summaryDb.hasAll(summaryFields, summaryClass, result.getQuery().getModel().getRestrict())) { execution.fill(result, null); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index c0c5b0ee0b0..846f70fd26b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -400,7 +400,7 @@ public class YqlParser implements Parser { case NON_EMPTY: return ensureNonEmpty(ast); default: - throw newUnexpectedArgumentException(names.get(0), DOT_PRODUCT, NEAREST_NEIGHBOR, + throw newUnexpectedArgumentException(names.get(0), DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, RANGE, RANK, USER_QUERY, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE, USER_INPUT, NON_EMPTY); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index ab9da8ccee5..f20fc85b448 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -20,6 +20,7 @@ import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_S /** * Generates mail-specific query metrics. */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class MetricsSearcher extends Searcher { private static final CompoundName metricsearcherId=new CompoundName("metricsearcher.id"); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java new file mode 100644 index 00000000000..e252a230d4f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.fs4.MapEncoder; +import com.yahoo.prelude.fastsearch.DocumentDatabase; +import com.yahoo.prelude.query.Highlight; +import com.yahoo.search.Query; +import com.yahoo.search.dispatch.rpc.ProtobufSerialization; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Encodes the query in binary form. + * + * @author bratseth + */ +class QueryEncoder { + + /** + * Encodes properties of this query. + * + * @param buffer the buffer to encode to + * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data + * @return the encoded length + */ + static int encodeAsProperties(Query query, ByteBuffer buffer, boolean encodeQueryData) { + // Make sure we don't encode anything here if we have turned the property feature off + // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of + // property maps - that's ok (probably we should simplify by just always turning the feature on) + if (! hasEncodableProperties(query)) return 0; + + int start = buffer.position(); + int mapCountPosition = buffer.position(); + buffer.putInt(0); // map count will go here + int mapCount = 0; + mapCount += query.getRanking().getProperties().encode(buffer, encodeQueryData); + if (encodeQueryData) { + mapCount += query.getRanking().getFeatures().encode(buffer); + if (query.getPresentation().getHighlight() != null) { + mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, + query.getPresentation().getHighlight().getHighlightTerms(), buffer); + } + mapCount += MapEncoder.encodeMap("model", createModelMap(query), buffer); + } + mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, + query.getModel().getDocumentDb(), buffer); + mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(query), buffer); + buffer.putInt(mapCountPosition, mapCount); + return buffer.position() - start; + } + + static boolean hasEncodableProperties(Query query) { + if ( ! query.getRanking().getProperties().isEmpty()) return true; + if ( ! query.getRanking().getFeatures().isEmpty()) return true; + if ( query.getRanking().getFreshness() != null) return true; + if ( query.getModel().getSearchPath() != null) return true; + if ( query.getModel().getDocumentDb() != null) return true; + if ( query.getPresentation().getHighlight() != null && + ! query.getPresentation().getHighlight().getHighlightItems().isEmpty()) return true; + return false; + } + + private static Map<String, Boolean> createCacheSettingMap(Query query) { + if (query.getGroupingSessionCache() && query.getRanking().getQueryCache()) { + Map<String, Boolean> cacheSettingMap = new HashMap<>(); + cacheSettingMap.put("grouping", true); + cacheSettingMap.put("query", true); + return cacheSettingMap; + } + if (query.getGroupingSessionCache()) + return Collections.singletonMap("grouping", true); + if (query.getRanking().getQueryCache()) + return Collections.singletonMap("query", true); + return Collections.emptyMap(); + } + + private static Map<String, String> createModelMap(Query query) { + Map<String, String> m = new HashMap<>(); + if (query.getModel().getSearchPath() != null) m.put("searchpath", query.getModel().getSearchPath()); + + int traceLevel = ProtobufSerialization.getTraceLevelForBackend(query); + if (traceLevel > 0) m.put("tracelevel", String.valueOf(traceLevel)); + + return m; + } + +} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index e2233d51ae4..b2e4821f164 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -3,13 +3,10 @@ package com.yahoo.vespa.streamingvisitors; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AckToken; -import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; @@ -41,7 +38,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -187,7 +183,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setLibraryParameter("location", af); } - if (query.hasEncodableProperties()) { + if (QueryEncoder.hasEncodableProperties(query)) { encodeQueryData(query, 1, ed); params.setLibraryParameter("rankproperties", ed.getEncodedData()); } @@ -254,7 +250,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { ed.setReturned(query.getModel().getQueryTree().getRoot().encode(buf)); break; case 1: - ed.setReturned(query.encodeAsProperties(buf, true)); + ed.setReturned(QueryEncoder.encodeAsProperties(query, buf, true)); break; case 2: throw new IllegalArgumentException("old aggregation no longer exists!"); diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index 46117374e59..39ea6435393 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -69,6 +69,7 @@ TOKEN : <EQUALS: "="> | <EXCLAMATION: "!"> | <INCLUDEDIRECTIVE: "@include"> | + <LANGUAGEDIRECTIVE: "@language"> | <LARGER: ">"> | <LARGEREQUALS: ">="> | <LEFTBRACE: "("> | @@ -86,8 +87,8 @@ TOKEN : <SLASH: "/"> | <SMALLER: "<"> | <SMALLEREQUALS: "<="> | + <STAR: "*"> | <STEMMINGDIRECTIVE: "@stemming"> | - <LANGUAGEDIRECTIVE: "@language"> | <SUPERDIRECTIVE: "@super"> | <IDENTIFIER: (~[ "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf", @@ -312,10 +313,14 @@ NamespaceProduction namespaceProduction() : ReferenceTermProduction referenceTermProduction() : { String reference; + boolean produceAll = false; } { - <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET> - { return new ReferenceTermProduction(reference); } + <LEFTSQUAREBRACKET> + reference = referenceIdentifier() + (<STAR> { produceAll = true; })? + <RIGHTSQUAREBRACKET> + { return new ReferenceTermProduction(reference, produceAll); } } LiteralTermProduction literalTermProduction() : diff --git a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def index e2856e137f0..c58f9944d61 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def +++ b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def @@ -24,7 +24,7 @@ jvm.stacksize int default=512 restart jvm.compressedClassSpaceSize int default=32 restart ## Base value of maximum direct memory size (in megabytes) -jvm.baseMaxDirectMemorySize int default=75 restart +jvm.baseMaxDirectMemorySize int default=16 restart ## Amount of direct memory used for caching. (in megabytes) jvm.directMemorySizeCache int default=0 restart diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index 11424cc7e4e..f0c29e64839 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -490,6 +490,18 @@ public class QueryCanonicalizerTestCase { assertFalse(shoe.usePositionData()); } + @Test + public void queryTreeExceedsAllowedSize() { + Query query = new Query(); + QueryTree tree = query.getModel().getQueryTree(); + tree.setRoot(new WordItem("A")); + tree.and(new WordItem("B")); + + assertNull(QueryCanonicalizer.canonicalize(query)); + query.properties().set("maxQueryItems", 2); + assertEquals("Query tree exceeds allowed item count. Configured limit: 2 - Item count: 3", QueryCanonicalizer.canonicalize(query)); + } + private void assertCanonicalized(String canonicalForm, String expectedError, Item root) { Query query = new Query(); query.getModel().getQueryTree().setRoot(root); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index 2a7e9da2992..ef3526f2fb1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -32,7 +32,6 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public class QuotingSearcherTestCase { public static QuotingSearcher createQuotingSearcher(String configId) { diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java index eb69372c22b..17eb4120b84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -29,7 +29,7 @@ public class ConditionTestCase { var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); TermCondition term = new TermCondition("foo", linguistics); Query query = new Query("?query=foo"); - assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(term.matches(new Evaluation(query, null).freshRuleEvaluation())); } @Test @@ -41,11 +41,11 @@ public class ConditionTestCase { sequence.addCondition(term1); sequence.addCondition(term2); Query query = new Query("?query=foo+bar"); - assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2).freshRuleEvaluation())); + assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3).freshRuleEvaluation())); + assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -57,11 +57,11 @@ public class ConditionTestCase { choice.addCondition(term1); choice.addCondition(term2); Query query1 = new Query("?query=foo+bar"); - assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1).freshRuleEvaluation())); + assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2).freshRuleEvaluation())); + assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3).freshRuleEvaluation())); + assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -75,13 +75,13 @@ public class ConditionTestCase { ProductionRule rule = new ReplacingProductionRule(); rule.setCondition(reference); rule.setProduction(new ProductionList()); - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); Query query = new Query("?query=foo"); - assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query, null).freshRuleEvaluation())); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java index ca78cae70ea..c398f0ed99e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java @@ -17,8 +17,8 @@ public class MatchOnlyIfNotOnlyTermTestCase extends RuleBaseAbstractTestCase { @Test public void testMatch() { - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake snl"); - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake saturday night live"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake snl"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake saturday night live"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java index 3051dd77190..1d6eea8d7de 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java @@ -14,7 +14,7 @@ public class OrPhraseTestCase extends RuleBaseAbstractTestCase { @Test public void testReplacing1() { - assertSemantics("OR (AND new york) title:\"software engineer\"","software engineer new york"); + assertSemantics("OR title:\"software engineer\" (AND new york)","software engineer new york"); assertSemantics("title:\"software engineer\"","software engineer"); // Skip or when there is nothing else } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index b91e9441a2b..bafb2cb6a73 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -32,7 +32,7 @@ public class ProductionRuleTestCase { NamedCondition named = new NamedCondition("brand", term); ConditionReference reference = new ConditionReference("brand"); - TermProduction termProduction = new ReferenceTermProduction("brand", "brand"); + TermProduction termProduction = new ReferenceTermProduction("brand", "brand", false); ProductionList productionList = new ProductionList(); productionList.addProduction(termProduction); @@ -41,7 +41,7 @@ public class ProductionRuleTestCase { rule.setProduction(productionList); // To initialize the condition reference... - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); @@ -49,7 +49,7 @@ public class ProductionRuleTestCase { assertTrue("Brand is referenced", rule.matchReferences().contains("brand")); Query query = new Query("?query=sony"); - RuleEvaluation e = new Evaluation(query).freshRuleEvaluation(); + RuleEvaluation e = new Evaluation(query, null).freshRuleEvaluation(); assertTrue(rule.matches(e)); rule.produce(e); assertEquals("AND brand:sony", query.getModel().getQueryTree().getRoot().toString()); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 76b2d3991c1..bee65db4347 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -128,12 +128,12 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testTypeChange() { - assertSemantics("RANK doors default:typechange","typechange doors"); + assertSemantics("RANK default:typechange doors","typechange doors"); } @Test public void testTypeChangeWithSingularToPluralButNonReplaceWillNotSingularify() { - assertSemantics("RANK door default:typechange","typechange door"); + assertSemantics("RANK default:typechange door","typechange door"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java new file mode 100644 index 00000000000..8f69be2f710 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.Test; + +/** + * @author bratseth + */ +public class SynonymTestCase { + + @Test + public void testReplacingBySynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:foo"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:bar"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:baz"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:word"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:a index1:phrase"); + tester.assertSemantics("index1:other", "index1:other"); + } + + @Test + public void testAddingSynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index2:foo index2:baz index2:bar", "index2:foo"); + tester.assertSemantics("EQUIV index2:bar index2:foo index2:baz", "index2:bar"); + tester.assertSemantics("EQUIV index2:baz index2:foo index2:bar", "index2:baz"); + tester.assertSemantics("EQUIV index2:word index2:\"a phrase\"", "index2:word"); + tester.assertSemantics("EQUIV index2:\"a phrase\" index2:word", "index2:a index2:phrase"); + tester.assertSemantics("index2:other", "index2:other"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr index 9a147887207..32f8e86b59f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr @@ -5,6 +5,4 @@ equiv1 +> =equiv2 =equiv3; testfield:[test] -> =testfield:e1 =testfield:e2 =testfield:e3; -synonymfield:[test] -> =[test]; - [test] :- foo, bar, baz; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr new file mode 100644 index 00000000000..4220f807733 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr @@ -0,0 +1,11 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +index1:[synonyms1] -> =index1:[synonyms1*]; # Replace by equiv(foo, bar, baz) when the query contains foo, bar or baz +index1:[synonyms2] -> =index1:[synonyms2*]; # with phrase + + +index2:[synonyms1] +> =index2:[synonyms1*]; # Add equiv(foo, bar, baz) when the query contains foo, bar or baz +index2:[synonyms2] +> =index2:[synonyms2*]; # with phrase + +[synonyms1] :- foo, baz, bar; +[synonyms2] :- word, a phrase; diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index 1e35b6447ff..fa8d237aee0 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -133,7 +133,7 @@ public class ClusteredConnectionTestCase { /** Represents a connection, e.g over http, in this test */ private static class Connection { - private String id; + private final String id; private boolean inService = true; diff --git a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java index 054b752c067..6c73937f4fa 100644 --- a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java @@ -2,6 +2,7 @@ package com.yahoo.search.query; import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.WordItem; import org.junit.Assert; import org.junit.Test; @@ -41,4 +42,25 @@ public class QueryTreeTest { assertEquals("+(AND p1 p2) -n1.1 -n1.2 -n2.1 -n2.2", tree.toString()); } + @Test + public void getCorrectTreeSize() { + QueryTree nullTree = new QueryTree(new NullItem()); + assertEquals(0, nullTree.treeSize()); + + NotItem not1 = new NotItem(); + not1.addPositiveItem(new WordItem("p1")); + not1.addNegativeItem(new WordItem("n1.1")); + not1.addNegativeItem(new WordItem("n1.2")); + + NotItem not2 = new NotItem(); + not2.addPositiveItem(new WordItem("p2")); + not2.addNegativeItem(new WordItem("n2.1")); + not2.addNegativeItem(new WordItem("n2.2")); + + QueryTree tree = new QueryTree(not1); + tree.and(not2); + + assertEquals(8, tree.treeSize()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java index 9e56fba0e17..4f14ae066cb 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -63,8 +63,8 @@ public class SortingTestCase { private void requireThatChineseHasCorrectRules(Collator col) { final int reorderCodes [] = {UScript.HAN}; - assertEquals("8.0.0.0", col.getUCAVersion().toString()); - assertEquals("153.64.29.0", col.getVersion().toString()); + assertEquals("14.0.0.0", col.getUCAVersion().toString()); + assertEquals("153.112.40.0", col.getVersion().toString()); assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); assertNotEquals("", ((RuleBasedCollator) col).getRules()); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java index a9792049fb7..511921ac047 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -4,7 +4,10 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import org.junit.Test; +import java.util.concurrent.Executors; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -21,7 +24,9 @@ public class CompiledQueryProfileRegistryTest { .value("5"))) .build(); - var registry = new CompiledQueryProfileRegistry(config); + var executor = Executors.newCachedThreadPool(); + var registry = new CompiledQueryProfileRegistry(config, executor); + assertTrue(executor.shutdownNow().isEmpty()); var profile1 = registry.findQueryProfile("profile1"); assertEquals("5", profile1.get("hits")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java index 60b6c6dbe84..3ed044601f0 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertTrue; */ public class DumpToolTestCase { - private String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; + private final String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; @Test public void testNoParameters() { @@ -25,12 +25,13 @@ public class DumpToolTestCase { @Test public void testNoDimensionValues() { - assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).startsWith("a=general-a\n")); + System.out.println(new DumpTool().resolveAndDump("multiprofile1", profileDir)); + assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).contains("a=general-a\n")); } @Test public void testAllParametersSet() { - assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").startsWith("a=general-a\n")); + assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").contains("a=general-a\n")); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java index 4b8edfdde73..fa7508ea5ac 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -47,7 +47,6 @@ public class NameTestCase { assertLegalFieldName("a/b"); assertLegalFieldName("/a/b"); assertLegalFieldName("/a/b/"); - assertIllegalFieldName(""); assertIllegalFieldName("aBc.dooEee.ce_d.-some-other.moreHere", "Could not set 'aBc.dooEee.ce_d.-some-other.moreHere' to 'anyValue'", "Illegal name '-some-other'"); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index f3a71af0b9e..637666a6a19 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -54,6 +54,7 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; import org.junit.Before; import org.junit.Test; @@ -69,6 +70,7 @@ import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Functional testing of {@link JsonRenderer}. @@ -165,11 +167,26 @@ public class JsonRendererTestCase { h.setField("tensor_mixed", new TensorFieldValue(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); h.setField("summaryfeatures", summaryFeatures); - Result r = new Result(new Query("/?format.tensors=short")); - r.hits().add(h); - r.setTotalHitCount(1L); - String summary = render(r); - assertEqualJson(expected, summary); + Result result1 = new Result(new Query("/?presentation.format.tensors=short")); + result1.hits().add(h); + result1.setTotalHitCount(1L); + String summary1 = render(result1); + assertEqualJson(expected, summary1); + + Result result2 = new Result(new Query("/?format.tensors=short")); + result2.hits().add(h); + result2.setTotalHitCount(1L); + String summary2 = render(result2); + assertEqualJson(expected, summary2); + + try { + render(new Result(new Query("/?presentation.format.tensors=unknown"))); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'presentation.format.tensors' to 'unknown': Value must be 'long' or 'short', not 'unknown'", + Exceptions.toMessageString(e)); + } } @Test @@ -1276,7 +1293,9 @@ public class JsonRendererTestCase { " f1: [ 'v1', { mykey1: 'myvalue1', mykey2: 'myvalue2' } ]," + " f2: { i1: 'v2', i2: { mykey3: 'myvalue3' }, i3: 'v3' }," + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + - " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }" + + " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }," + + " f5: { '10001': 'myvalue6', '10002': 'myvalue7' }," + + " f6: { i4: 'v6', i5: { '-17': 'myvalue8', '-42': 'myvalue9' } }" + " }" + " } ]" + "}}"); @@ -1285,6 +1304,8 @@ public class JsonRendererTestCase { h.setField("f2", dataFromSimplified("{ i1: 'v2', i2: [ { key: 'mykey3', value: 'myvalue3' } ], i3: 'v3' }")); h.setField("f3", dataFromSimplified("{ j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }")); h.setField("f4", dataFromSimplified("[ { key: 'mykey4', value: 'myvalue4' }, { key: 'mykey5', value: 'myvalue5' } ]")); + h.setField("f5", dataFromSimplified("[ { key: 10001, value: 'myvalue6' }, { key: 10002, value: 'myvalue7' } ]")); + h.setField("f6", dataFromSimplified("{ i4: 'v6', i5: [ {key: -17, value: 'myvalue8' }, { key: -42, value: 'myvalue9' } ] }")); r.hits().add(h); r.setTotalHitCount(1L); String summary = render(r); @@ -1298,7 +1319,51 @@ public class JsonRendererTestCase { " f1: [ 'v1', [ { key: 'mykey1', value: 'myvalue1' }, { key: 'mykey2', value: 'myvalue2' } ] ]," + " f2: { i1: 'v2', i2: [ { key: 'mykey3', value: 'myvalue3' } ], i3: 'v3' }," + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + - " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }" + + " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }," + + " f5: [ { key: 10001, value: 'myvalue6' }, { key: 10002, value: 'myvalue7' } ]," + + " f6: { i4: 'v6', i5: [ { key: -17, value: 'myvalue8' }, { key: -42, value: 'myvalue9' } ] }" + + " }" + + " } ]" + + "}}"); + r.hits().add(h); + r.setTotalHitCount(1L); + summary = render(r); + assertEqualJson(expected.toString(), summary); + } + + @Test + public void testWsetInFields() throws IOException, InterruptedException, ExecutionException { + Result r = new Result(new Query("/?renderer.json.jsonWsets=true")); + var expected = dataFromSimplified( + "{root: { id:'toplevel', relevance:1.0, fields: { totalCount: 1 }," + + " children: [ { id: 'myHitName', relevance: 1.0," + + " fields: { " + + " f1: [ 'v1', { mykey1: 10, mykey2: 20 } ]," + + " f2: { i1: 'v2', i2: { mykey3: 30 }, i3: 'v3' }," + + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + + " f4: { mykey4: 40, mykey5: 50 }" + + " }" + + " } ]" + + "}}"); + Hit h = new Hit("myHitName"); + h.setField("f1", dataFromSimplified("[ 'v1', [ { item: 'mykey1', weight: 10 }, { item: 'mykey2', weight: 20 } ] ]")); + h.setField("f2", dataFromSimplified("{ i1: 'v2', i2: [ { item: 'mykey3', weight: 30 } ], i3: 'v3' }")); + h.setField("f3", dataFromSimplified("{ j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }")); + h.setField("f4", dataFromSimplified("[ { item: 'mykey4', weight: 40 }, { item: 'mykey5', weight: 50 } ]")); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected.toString(), summary); + + r = new Result(new Query("/?renderer.json.jsonWsets=false")); + expected = dataFromSimplified( + "{root:{id:'toplevel',relevance:1.0,fields:{totalCount:1}," + + " children: [ { id: 'myHitName', relevance: 1.0," + + " fields: { " + + " f1: [ 'v1', [ { item: 'mykey1', weight: 10 }, { item: 'mykey2', weight: 20 } ] ]," + + " f2: { i1: 'v2', i2: [ { item: 'mykey3', weight: 30 } ], i3: 'v3' }," + + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + + " f4: [ { item: 'mykey4', weight: 40 }, { item: 'mykey5', weight: 50 } ]" + " }" + " } ]" + "}}"); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 14fdd047391..9a760bfb0cb 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.searchers; import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; import com.yahoo.config.subscription.RawSource; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -18,7 +19,11 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; + +import java.io.File; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -32,8 +37,8 @@ public class ValidateNearestNeighborTestCase { public ValidateNearestNeighborTestCase() { searcher = new ValidateNearestNeighborSearcher( ConfigGetter.getConfig(AttributesConfig.class, - "raw:", - new RawSource("attribute[5]\n" + + "raw:" + + "attribute[5]\n" + "attribute[0].name simple\n" + "attribute[0].datatype INT32\n" + "attribute[1].name dvector\n" + @@ -57,7 +62,7 @@ public class ValidateNearestNeighborTestCase { "attribute[7].name threetypes\n" + "attribute[7].datatype TENSOR\n" + "attribute[7].tensortype tensor(x{})\n" - ))); + )); } private static TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java index 86ffb1f9830..01e360858c1 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -27,8 +27,8 @@ public class ValidateMatchPhaseSearcherTestCase { public ValidateMatchPhaseSearcherTestCase() { searcher = new ValidateMatchPhaseSearcher( ConfigGetter.getConfig(AttributesConfig.class, - "raw:", - new RawSource("attribute[4]\n" + + "raw:" + + "attribute[4]\n" + "attribute[0].name ok\n" + "attribute[0].datatype INT32\n" + "attribute[0].collectiontype SINGLE\n" + @@ -45,7 +45,7 @@ public class ValidateMatchPhaseSearcherTestCase { "attribute[3].datatype INT32\n" + "attribute[3].collectiontype ARRAY\n" + "attribute[3].fastsearch true" - ))); + )); } private static String getErrorMatch(String attribute) { diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index f6273fdf723..a4fd37145c8 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.test; import com.yahoo.component.chain.Chain; +import com.yahoo.data.JsonProducer; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detection; @@ -15,6 +16,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; @@ -36,10 +38,12 @@ import com.yahoo.search.query.profile.DimensionValues; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.Exceptions; +import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; @@ -69,6 +73,17 @@ import static org.junit.Assert.fail; public class QueryTestCase { @Test + public void testIt() throws Exception { + JSONObject newroot = new JSONObject("{\"key\": 3}"); + var hit = new FastHit(); + hit.setField("data", (JsonProducer)s -> s.append(newroot)); + var field = hit.getField("data"); + if (field instanceof JsonProducer) { + System.out.println((((JsonProducer) field).toJson())); + } + } + + @Test public void testSimpleFunctionality() { Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter")); assertEquals("this is a simple query", q.getModel().getQueryString()); @@ -1078,6 +1093,49 @@ public class QueryTestCase { query.getSelect().getGrouping().toString()); } + /** + * Tests that the value presentation.format.tensors can be set in a query profile. + * This is special because presentation.format is a native query profile. + */ + @Test + public void testSettingNativeQueryProfileValueInQueryProfile() { + { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("default"); + profile.set("presentation.format.tensors", "short", Map.of(), registry); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.findQueryProfile("default")); + assertTrue(query.getPresentation().getTensorShortForm()); + } + + { // Same as above but also set presentation.format + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("default"); + profile.set("presentation.format", "xml", Map.of(), registry); + profile.set("presentation.format.tensors", "short", Map.of(), registry); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.findQueryProfile("default")); + assertEquals("xml", query.getPresentation().getFormat()); + assertTrue(query.getPresentation().getTensorShortForm()); + } + + { // Set presentation.format with a typed query profile type + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType type = new QueryProfileType("mytype"); + type.inherited().add(registry.getType("native")); + registry.getTypeRegistry().register(type); + type.addField(new FieldDescription("ranking.features.query(embedding)", "tensor(x[5])"), + registry.getTypeRegistry()); + QueryProfile profile = new QueryProfile("default"); + profile.setType(type); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo&presentation.format=xml", cRegistry.findQueryProfile("default")); + } + } + private void assertDetectionText(String expectedDetectionText, String queryString, String ... indexSpecs) { Query q = new Query(httpEncode("/?query=" + queryString)); SearchDefinition sd = new SearchDefinition("testSearchDefinition"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 5d3a95efc78..0385de5bebf 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -24,7 +24,9 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; -import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; +import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; + /** * Test translation of fields and sources in YQL to the associated concepts in Vespa. @@ -52,7 +54,7 @@ public class YqlFieldAndSourceTestCase { mockBackend.addResult(query, result); mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); - mockBackend.addSummaryClassByCopy(Execution.ATTRIBUTEPREFETCH, Arrays.asList(FIELD2)); + mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, Arrays.asList(FIELD2)); mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); DocumentdbInfoConfig config = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder() @@ -90,7 +92,7 @@ public class YqlFieldAndSourceTestCase { new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(1) - .name(Execution.ATTRIBUTEPREFETCH) + .name(SORTABLE_ATTRIBUTES_SUMMARY_CLASS) .fields(Arrays.asList(new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(2) @@ -112,7 +114,7 @@ public class YqlFieldAndSourceTestCase { execution.fill(result); assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -123,7 +125,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertTrue(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertTrue(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -134,7 +136,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -145,7 +147,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index cc70f46fca1..1d07cafeda9 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -243,7 +243,6 @@ public class VdsVisitorTestCase { assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); } } - assertEquals(-1, params.getMaxFirstPassHits()); if (qa.maxBucketsPerVisitor != 0) { assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); } else { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index c74133838cf..73f83a9b6e5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -57,6 +57,7 @@ public class LogEntry { return message; } + @SuppressWarnings("deprecation") public static List<LogEntry> parseVespaLog(InputStream log, Instant from) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(log, UTF_8))) { return reader.lines() @@ -104,6 +105,7 @@ public class LogEntry { return Objects.hash(id, at, type, message); } + @SuppressWarnings("deprecation") public static Type typeOf(Level level) { return level.intValue() < Level.INFO.intValue() || level.intValue() == LogLevel.IntValEVENT ? Type.debug : level.intValue() < Level.WARNING.intValue() ? Type.info diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index d4e11163343..b36d7880506 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMoni import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; @@ -50,8 +49,6 @@ public interface ServiceRegistry { NameService nameService(); - GlobalRoutingService globalRoutingService(); - Mailer mailer(); EndpointCertificateProvider endpointCertificateProvider(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index 5363e8d0150..69eda662692 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import java.util.Map; +import java.util.Set; /** * Service that manages archive storage URIs for tenant nodes. @@ -14,7 +15,9 @@ import java.util.Map; */ public interface ArchiveService { - ArchiveBucket createArchiveBucketFor(ZoneId zoneId, boolean sharded); + ArchiveBucket createArchiveBucketFor(ZoneId zoneId); - void updateBucketAndKeyPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName); + void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName); + + void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index 5c979ddfc7b..ce7b56ad1f6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -6,6 +6,8 @@ import com.yahoo.config.provision.zone.ZoneId; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; /** * @author freva @@ -13,15 +15,21 @@ import java.util.Map; */ public class MockArchiveService implements ArchiveService { - public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRoles = new HashMap<>(); + public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRolesForBucket = new HashMap<>(); + public Map<String, Set<String>> authorizedIamRolesForKey = new TreeMap<>(); @Override - public ArchiveBucket createArchiveBucketFor(ZoneId zoneId, boolean sharded) { + public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { return new ArchiveBucket("bucketName", "keyArn"); } @Override - public void updateBucketAndKeyPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) { - authorizedIamRoles.put(bucket, authorizeIamRoleByTenantName); + public void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) { + authorizedIamRolesForBucket.put(bucket, authorizeIamRoleByTenantName); + } + + @Override + public void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles) { + authorizedIamRolesForKey.put(keyArn, tenantAuthorizedIamRoles); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java index a981b11887e..c1d70bf297d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzUser; import java.time.Instant; @@ -14,5 +15,8 @@ import java.util.Collection; */ public interface AccessControlService { boolean approveDataPlaneAccess(AthenzUser user, Instant expiry); + boolean approveSshAccess(TenantName tenantName, Instant expiry); + boolean requestSshAccess(TenantName tenantName); + boolean hasPendingAccessRequests(TenantName tenantName); Collection<AthenzUser> listMembers(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java index c2d4d4a5996..b01f6bb5208 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java @@ -2,15 +2,16 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzGroup; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import java.time.Instant; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -19,13 +20,16 @@ public class AthenzAccessControlService implements AccessControlService { private static final String ALLOWED_OPERATOR_GROUPNAME = "vespa-team"; private static final String DATAPLANE_ACCESS_ROLENAME = "operator-data-plane"; + private final String TENANT_DOMAIN_PREFIX = "vespa.tenant"; private final ZmsClient zmsClient; private final AthenzRole dataPlaneAccessRole; private final AthenzGroup vespaTeam; + private final ZmsClient vespaZmsClient; //TODO: Merge ZMS clients - public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain) { + public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain, ZmsClient vespaZmsClient) { this.zmsClient = zmsClient; + this.vespaZmsClient = vespaZmsClient; this.dataPlaneAccessRole = new AthenzRole(domain, DATAPLANE_ACCESS_ROLENAME); this.vespaTeam = new AthenzGroup(domain, ALLOWED_OPERATOR_GROUPNAME); } @@ -36,7 +40,7 @@ public class AthenzAccessControlService implements AccessControlService { if(!isVespaTeamMember(user)) { throw new IllegalArgumentException(String.format("User %s requires manual approval, please contact Vespa team", user.getName())); } - Map<AthenzUser, String> users = zmsClient.listPendingRoleApprovals(dataPlaneAccessRole); + Map<AthenzIdentity, String> users = zmsClient.listPendingRoleApprovals(dataPlaneAccessRole); if (users.containsKey(user)) { zmsClient.approvePendingRoleMembership(dataPlaneAccessRole, user, expiry, Optional.empty()); return true; @@ -53,6 +57,70 @@ public class AthenzAccessControlService implements AccessControlService { .collect(Collectors.toList()); } + /** + * @return Whether the ssh access role has any pending role membership requests + */ + @Override + public boolean hasPendingAccessRequests(TenantName tenantName) { + var role = sshRole(tenantName); + if (!vespaZmsClient.listRoles(role.domain()).contains(role)) + return false; + var pendingApprovals = vespaZmsClient.listPendingRoleApprovals(role); + return pendingApprovals.containsKey(vespaTeam); + } + + /** + * @return true if access has been granted - false if already member + */ + @Override + public boolean approveSshAccess(TenantName tenantName, Instant expiry) { + var role = sshRole(tenantName); + + if (!vespaZmsClient.listRoles(role.domain()).contains(role)) + vespaZmsClient.createRole(role, Map.of()); + + if (vespaZmsClient.getMembership(role, vespaTeam)) + return false; + + if (!hasPendingAccessRequests(tenantName)) { + vespaZmsClient.addRoleMember(role, vespaTeam, Optional.empty()); + } + // TODO: Pass along auth0 credentials + vespaZmsClient.approvePendingRoleMembership(role, vespaTeam, expiry, Optional.empty()); + return true; + } + + /** + * @return true if access has been requested - false if already member + */ + @Override + public boolean requestSshAccess(TenantName tenantName) { + var role = sshRole(tenantName); + + if (!vespaZmsClient.listRoles(role.domain()).contains(role)) + vespaZmsClient.createRole(role, Map.of()); + + if (vespaZmsClient.getMembership(role, vespaTeam)) + return false; + + vespaZmsClient.addRoleMember(role, vespaTeam, Optional.empty()); + return true; + } + + private AthenzRole sshRole(TenantName tenantName) { + return new AthenzRole(getOrCreateTenantDomain(tenantName), "ssh_access"); + } + + private AthenzDomain getOrCreateTenantDomain(TenantName tenantName) { + var domain = new AthenzDomain(TENANT_DOMAIN_PREFIX + "." + tenantName.value()); + + if (vespaZmsClient.getDomainList(domain.getName()).isEmpty()) { + vespaZmsClient.createSubdomain(new AthenzDomain(TENANT_DOMAIN_PREFIX), tenantName.value()); + } + + return domain; + } + public boolean isVespaTeamMember(AthenzUser user) { return zmsClient.getGroupMembership(vespaTeam, user); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java index 34a8d52b4ca..19eaaa0da87 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java @@ -7,15 +7,11 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import java.util.logging.Logger; - /** * @author bjorncs */ public class AthenzClientFactoryMock extends AbstractComponent implements AthenzClientFactory { - private static final Logger log = Logger.getLogger(AthenzClientFactoryMock.class.getName()); - private final AthenzDbMock athenz; @Inject diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java index a0cc0d1ae1c..f906172dba0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzUser; import java.time.Instant; @@ -28,6 +29,21 @@ public class MockAccessControlService implements AccessControlService { return Set.copyOf(members); } + @Override + public boolean approveSshAccess(TenantName tenantName, Instant expiry) { + return false; + } + + @Override + public boolean requestSshAccess(TenantName tenantName) { + return false; + } + + @Override + public boolean hasPendingAccessRequests(TenantName tenantName) { + return false; + } + public void addPendingMember(AthenzUser user) { pendingMembers.add(user); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 4679f660319..63212c9c200 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -40,7 +40,7 @@ public class ZmsClientMock implements ZmsClient { private final AthenzDbMock athenz; private final AthenzIdentity controllerIdentity; private static final Pattern TENANT_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.(?<tenantDomain>[\\w\\-_]+)\\..*"); - private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.wildcard"); + private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.(?<environment>[\\w\\-_]+)"); public ZmsClientMock(AthenzDbMock athenz, AthenzIdentity controllerIdentity) { this.athenz = athenz; @@ -200,12 +200,12 @@ public class ZmsClientMock implements ZmsClient { } @Override - public Map<AthenzUser,String> listPendingRoleApprovals(AthenzRole athenzRole) { + public Map<AthenzIdentity,String> listPendingRoleApprovals(AthenzRole athenzRole) { return Map.of(); } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason) { + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, Optional<String> reason) { } @Override @@ -255,6 +255,10 @@ public class ZmsClientMock implements ZmsClient { public void deleteRole(AthenzRole athenzRole) { athenz.domains.get(athenzRole.domain()).roles.removeIf(role -> role.name().equals(athenzRole.roleName())); } + + @Override + public void createSubdomain(AthenzDomain parent, String name) {} + @Override public void close() {} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java index 4891fe0ffa7..ed389797b5d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java @@ -132,4 +132,9 @@ public interface BillingDatabaseClient { * @param collectionMethod The collection method for the tenant */ void setCollectionMethod(TenantName tenantName, CollectionMethod collectionMethod); + + /** + * Performs necessary maintenance operations + */ + void maintain(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java index f53025a2e6d..2ac3242f6aa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java @@ -175,4 +175,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { }) .collect(Collectors.toList()); } + + @Override + public void maintain() {} } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index 87e0a82fc0b..9bfd8f9d34e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -36,7 +36,7 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); if (pemEncodedEndpointCertificate == null) - throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Secret store returned null for certificate"); + throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Secret store returned null for certificate"); List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index 9d51c5ca9d1..f0b681fd9c9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; -import org.apache.hc.core5.http.ClassicHttpRequest; import java.util.stream.Stream; @@ -41,7 +40,8 @@ public class ConfigServerException extends RuntimeException { PARENT_HOST_NOT_READY, CERTIFICATE_NOT_READY, LOAD_BALANCER_NOT_READY, - INCOMPLETE_RESPONSE + INCOMPLETE_RESPONSE, + CONFIG_NOT_CONVERGED } public static ConfigServerException readException(byte[] body, String context) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index a0dee6c059f..ae9c5285be6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -23,7 +23,8 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { */ public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty(), Optional.empty(), Optional.empty(), Optional.empty(), - Optional.empty(), Optional.empty(), true); + Optional.empty(), Optional.empty(), true, + Optional.empty()); // This never changes and is only used to create a valid semantic version number, as required by application bundles private static final String majorVersion = "1.0"; @@ -36,11 +37,12 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { private final Optional<String> sourceUrl; private final Optional<String> commit; private final boolean deployedDirectly; + private final Optional<String> bundleHash; /** Public for serialisation only. */ public ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail, Optional<Version> compileVersion, Optional<Instant> buildTime, Optional<String> sourceUrl, - Optional<String> commit, boolean deployedDirectly) { + Optional<String> commit, boolean deployedDirectly, Optional<String> bundleHash) { if (buildNumber.isEmpty() && ( source.isPresent() || authorEmail.isPresent() || compileVersion.isPresent() || buildTime.isPresent() || sourceUrl.isPresent() || commit.isPresent())) throw new IllegalArgumentException("Build number must be present if any other attribute is"); @@ -65,26 +67,30 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { this.sourceUrl = Objects.requireNonNull(sourceUrl, "sourceUrl cannot be null"); this.commit = Objects.requireNonNull(commit, "commit cannot be null"); this.deployedDirectly = deployedDirectly; + this.bundleHash = bundleHash; } /** Create an application package version from a completed build, without an author email */ public static ApplicationVersion from(SourceRevision source, long buildNumber) { return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.empty(), - Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false); + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, + Optional.empty()); } /** Creates a version from a completed build, an author email, and build meta data. */ public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail, Version compileVersion, Instant buildTime) { return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail), - Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty(), false); + Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty(), false, + Optional.empty()); } /** Creates a version from a completed build, an author email, and build meta data. */ public static ApplicationVersion from(Optional<SourceRevision> source, long buildNumber, Optional<String> authorEmail, Optional<Version> compileVersion, Optional<Instant> buildTime, - Optional<String> sourceUrl, Optional<String> commit, boolean deployedDirectly) { - return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly); + Optional<String> sourceUrl, Optional<String> commit, boolean deployedDirectly, + Optional<String> bundleHash) { + return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); } /** Returns a unique identifier for this version or "unknown" if version is not known */ @@ -115,6 +121,11 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns the time this package was built, if known. */ public Optional<Instant> buildTime() { return buildTime; } + /** Returns the hash of app package except deployment/build-meta data */ + public Optional<String> bundleHash() { + return bundleHash; + } + /** Returns the source URL for this application version. */ public Optional<String> sourceUrl() { return sourceUrl.or(() -> source.map(source -> { @@ -174,8 +185,8 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { if (buildNumber().isEmpty() || o.buildNumber().isEmpty()) return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Unknown version sorts first - if (deployedDirectly || o.deployedDirectly) - return Boolean.compare(deployedDirectly, o.deployedDirectly); // Directly deployed versions sort first + if (deployedDirectly != o.deployedDirectly) + return Boolean.compare( ! deployedDirectly, ! o.deployedDirectly); // Directly deployed versions sort first return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index fe1fcece6b1..f6270c78069 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -51,6 +51,9 @@ public interface TesterCloud { /** Tests failed. */ FAILURE, + /** Tests were inconclusive, and need to run again later. */ + INCONCLUSIVE, + /** The tester encountered an exception. */ ERROR, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java deleted file mode 100644 index e14a5a5d562..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing; - -import com.yahoo.config.provision.zone.ZoneId; - -import java.util.Map; - -/** - * A service containing the health status of global rotations. - * - * @author mpolden - */ -public interface GlobalRoutingService { - - /** Returns the health status of each zone behind the given rotation name */ - Map<ZoneId, RotationStatus> getHealthStatus(String rotationName); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java deleted file mode 100644 index 5d51030a329..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.zone.ZoneId; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bratseth - */ -public class MemoryGlobalRoutingService extends AbstractComponent implements GlobalRoutingService { - - private final Map<String, Map<ZoneId, RotationStatus>> status = new HashMap<>(); - - @Override - public Map<ZoneId, RotationStatus> getHealthStatus(String rotationName) { - if (status.isEmpty()) { - return Map.of(ZoneId.from("prod", "us-west-1"), RotationStatus.IN); - } - return Collections.unmodifiableMap(status.getOrDefault(rotationName, Map.of())); - } - - public MemoryGlobalRoutingService setStatus(String rotation, ZoneId zone, RotationStatus status) { - this.status.putIfAbsent(rotation, new HashMap<>()); - this.status.get(rotation).put(zone, status); - return this; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index 3b41550762b..cf6b592b7a5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -81,6 +81,10 @@ public class MockTesterCloud implements TesterCloud { log.add(entry); } + public void clearLog() { + log.clear(); + } + public void set(Status status) { this.status = status; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 8bd7a9e726d..3a5bb96d985 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -233,7 +233,13 @@ enum PathGroup { secretStore(Matcher.tenant, "/application/v4/tenant/{tenant}/secret-store/{*}"), /** Paths used to proxy Horizon metric requests */ - horizonProxy("/horizon/v1/{*}"); + horizonProxy("/horizon/v1/{*}"), + + /** Paths used to list and request access to tenant resources */ + accessRequests(Matcher.tenant, "/application/v4/tenant/{tenant}/access/request/{*}"), + + /** Paths used to approve requests to access tenant resources */ + accessRequestApproval(Matcher.tenant, "/application/v4/tenant/{tenant}/access/approve/{*}"); final List<String> pathSpecs; final List<Matcher> matchers; @@ -281,9 +287,10 @@ enum PathGroup { return EnumSet.complementOf(EnumSet.copyOf(pathGroups)); } - static Set<PathGroup> billingPaths() { + static Set<PathGroup> operatorRestrictedPaths() { var paths = billingPathsNoToken(); paths.add(PathGroup.billingToken); + paths.add(accessRequestApproval); return paths; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 5768a7a3b3b..4d9a16c34f0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -22,7 +22,7 @@ enum Policy { /** Full access to everything. */ operator(Privilege.grant(Action.all()) - .on(PathGroup.allExcept(PathGroup.billingPaths())) + .on(PathGroup.allExcept(PathGroup.operatorRestrictedPaths())) .in(SystemName.all()), Privilege.grant(Action.read) .on(PathGroup.billingPathsNoToken()) @@ -49,7 +49,7 @@ enum Policy { /** Access to create a user tenant in select systems. */ user(Privilege.grant(Action.create, Action.update) .on(PathGroup.user) - .in(SystemName.main, SystemName.cd, SystemName.dev)), + .in(SystemName.main, SystemName.cd)), /** Access to create a tenant. */ tenantCreate(Privilege.grant(Action.create) @@ -130,7 +130,7 @@ enum Policy { /** Read access to all information in select systems. */ classifiedRead(Privilege.grant(Action.read) .on(PathGroup.allExcept(PathGroup.classifiedOperator)) - .in(SystemName.main, SystemName.cd, SystemName.dev)), + .in(SystemName.main, SystemName.cd)), /** Read access to public info. */ publicRead(Privilege.grant(Action.read) @@ -188,6 +188,10 @@ enum Policy { .on(PathGroup.billingList, PathGroup.billing) .in(SystemName.PublicCd, SystemName.Public)), + accessRequests(Privilege.grant(Action.all()) + .on(PathGroup.accessRequests, PathGroup.accessRequestApproval) + .in(SystemName.PublicCd)), + /** Invoice management */ hostedAccountant(Privilege.grant(Action.all()) .on(PathGroup.hostedAccountant, PathGroup.accountant) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index 5cdd12ecb1c..c40c2d4db01 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -52,6 +52,11 @@ public abstract class Role { return new TenantRole(RoleDefinition.developer, tenant); } + /** Returns a {@link RoleDefinition#hostedDeveloper} for the current system and given tenant. */ + public static TenantRole hostedDeveloper(TenantName tenant) { + return new TenantRole(RoleDefinition.hostedDeveloper, tenant); + } + /** Returns a {@link RoleDefinition#administrator} for the current system and given tenant. */ public static TenantRole administrator(TenantName tenant) { return new TenantRole(RoleDefinition.administrator, tenant); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index eeb3bae4431..7f6e77faf03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -60,6 +60,9 @@ public enum RoleDefinition { Policy.billingInformationRead, Policy.secretStoreOperations), + /** Developer for manual deployments for a tenant */ + hostedDeveloper(Policy.developmentDeployment), + /** Admin — the administrative function for user management etc. */ administrator(Policy.tenantUpdate, Policy.tenantManager, @@ -72,7 +75,9 @@ public enum RoleDefinition { Policy.paymentInstrumentDelete, Policy.paymentInstrumentCreate, Policy.planUpdate, - Policy.billingInformationRead), + Policy.billingInformationRead, + Policy.accessRequests + ), /** Headless — the application specific role identified by deployment keys for production */ headless(Policy.submission), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java index 91f7ed2162d..87d4f03b090 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java @@ -13,13 +13,35 @@ import java.util.Objects; * @author smorgrav */ public class TenantInfo { + // Editable as 'Tenant Information - Company Name' + // Viewable in the 'Account - Profile' section as 'Company Name' private final String name; + + // Editable as 'Tenant Information - Email' + // Not displayed outside of 'Edit profile' private final String email; + + // Editable as 'Tenant Information - Website' + // Viewable in the 'Account - Profile' section at bottom of 'Contact Information' private final String website; + + // Editable as 'Contact Information - Contact Name' + // Viewable in the 'Account - Profile' section in 'Contact Information' private final String contactName; + + // Editable as 'Contact Information - Contact Email' + // Viewable in the 'Account - Profile' section in 'Contact Information' private final String contactEmail; + + // Not editable in the account setting + // Not viewable. + // TODO: Remove private final String invoiceEmail; + + // See class for more info private final TenantInfoAddress address; + + // See class for more info private final TenantInfoBillingContact billingContact; TenantInfo(String name, String email, String website, String contactName, String contactEmail, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java index a838a86567a..740adde4519 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java @@ -16,6 +16,8 @@ import java.util.Objects; */ public class TenantInfoAddress { + // All fields are editable in 'Edit Profile - Company Address'. + // The fields are not exposed outside the 'Edit Profile' form. private final String addressLines; private final String postalCodeOrZip; private final String city; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java index 177f0620d85..c875a19d57b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java @@ -7,6 +7,10 @@ import java.util.Objects; * @author smorgrav */ public class TenantInfoBillingContact { + + // All fields are editable in 'Billing - Edit billing contact' + // Only 'name' and 'email' are exposed outside the 'Edit billing contact' form. + // All these fields are required by the billing process. private final String name; private final String email; private final String phone; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index bbf76fcb480..3ef38715c5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -29,7 +29,9 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; @@ -47,6 +49,7 @@ public class Application { private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; private final Optional<ApplicationVersion> latestVersion; + private final SortedSet<ApplicationVersion> versions; private final OptionalLong projectId; private final Optional<IssueId> deploymentIssueId; private final Optional<IssueId> ownershipIssueId; @@ -60,14 +63,14 @@ public class Application { public Application(TenantAndApplicationId id, Instant now) { this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), - new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), List.of()); + new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), new TreeSet<>(), List.of()); } // DO NOT USE! For serialization purposes, only. public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, - Optional<ApplicationVersion> latestVersion, Collection<Instance> instances) { + Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, Collection<Instance> instances) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -80,6 +83,7 @@ public class Application { this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); this.latestVersion = requireNotUnknown(latestVersion); + this.versions = versions; this.instances = instances.stream().collect( Collectors.collectingAndThen(Collectors.toMap(Instance::name, Function.identity(), @@ -105,7 +109,14 @@ public class Application { public OptionalLong projectId() { return projectId; } /** Returns the last submitted version of this application. */ - public Optional<ApplicationVersion> latestVersion() { return latestVersion; } + public Optional<ApplicationVersion> latestVersion() { + return versions.isEmpty() ? Optional.empty() : Optional.of(versions.last()); + } + + /** Returns the currently deployed versions of the application */ + public SortedSet<ApplicationVersion> versions() { + return versions; + } /** * Returns the last deployed validation overrides of this application, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a23cb40dcb1..2b88d6ded9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -87,6 +87,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -370,7 +371,6 @@ public class ApplicationController { try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); Instance instance = application.get().require(job.application().instance()); - rejectOldChange(instance, platform, revision, job, zone); if ( ! applicationPackage.trustedCertificates().isEmpty() && run.testerCertificate().isPresent()) @@ -392,6 +392,8 @@ public class ApplicationController { // the source since it's the same application, so it should have the same warnings NotificationSource source = zone.environment().isManuallyDeployed() ? NotificationSource.from(deployment) : NotificationSource.from(applicationId); + + @SuppressWarnings("deprecation") List<String> warnings = Optional.ofNullable(result.prepareResponse().log) .map(logs -> logs.stream() .filter(log -> log.applicationPackage) @@ -444,6 +446,23 @@ public class ApplicationController { controller.notificationsDb().removeNotifications(notification.source()); } + var oldestDeployedVersion = application.get().productionDeployments().values().stream() + .flatMap(List::stream) + .map(Deployment::applicationVersion) + .filter(version -> ! version.isDeployedDirectly()) + .min(naturalOrder()) + .orElse(ApplicationVersion.unknown); + + var olderVersions = application.get().versions().stream() + .filter(version -> version.compareTo(oldestDeployedVersion) < 0) + .sorted() + .collect(Collectors.toList()); + + // Remove any version not deployed anywhere - but keep one + for (int i = 0; i < olderVersions.size() - 1; i++) { + application = application.withoutVersion(olderVersions.get(i)); + } + store(application); return application; } @@ -801,20 +820,6 @@ public class ApplicationController { } } - private void rejectOldChange(Instance instance, Version platform, ApplicationVersion revision, JobId job, ZoneId zone) { - Deployment deployment = instance.deployments().get(zone); - if (deployment == null) return; - if (!zone.environment().isProduction()) return; - - boolean platformIsOlder = platform.compareTo(deployment.version()) < 0 && !instance.change().isPinned(); - boolean revisionIsOlder = revision.compareTo(deployment.applicationVersion()) < 0 && - !(revision.isUnknown() && controller.system().isCd()); - if (platformIsOlder || revisionIsOlder) - throw new IllegalArgumentException(Text.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + - " are older than the currently deployed (platform: %s, application: %s).", - job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); - } - private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 6e31c93dbdd..cc68d47666d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -45,15 +45,17 @@ public class Instance { private final RotationStatus rotationStatus; private final Map<JobType, Instant> jobPauses; private final Change change; + private final Optional<ApplicationVersion> latestDeployed; /** Creates an empty instance */ public Instance(ApplicationId id) { - this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); + this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty(), Optional.empty()); } /** Creates an empty instance*/ public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, - List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { + List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change, + Optional<ApplicationVersion> latestDeployed) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); @@ -61,6 +63,7 @@ public class Instance { this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"); this.change = Objects.requireNonNull(change, "change cannot be null"); + this.latestDeployed = Objects.requireNonNull(latestDeployed, "latestDeployed cannot be null"); } public Instance withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -87,7 +90,7 @@ public class Instance { else jobPauses.remove(jobType); - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); } public Instance recordActivityAt(Instant instant, ZoneId zone) { @@ -118,15 +121,19 @@ public class Instance { } public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change, latestDeployed); } public Instance with(RotationStatus rotationStatus) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); } public Instance withChange(Change change) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + } + + public Instance withLatestDeployed(ApplicationVersion latestDeployed) { + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, Optional.of(latestDeployed)); } private Instance with(Deployment deployment) { @@ -136,7 +143,7 @@ public class Instance { } private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); } public ApplicationId id() { return id; } @@ -181,6 +188,11 @@ public class Instance { return change; } + /** Returns the application version that last rolled out to this instance. */ + public Optional<ApplicationVersion> latestDeployed() { + return latestDeployed; + } + /** Returns the total quota usage for this instance, excluding temporary deployments **/ public QuotaUsage quotaUsage() { return deployments.values().stream() @@ -199,7 +211,7 @@ public class Instance { @Override public boolean equals(Object o) { if (this == o) return true; - if (! (o instanceof Instance)) return false; + if ( ! (o instanceof Instance)) return false; Instance that = (Instance) o; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 13ef1339e44..06ff381e4dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -21,6 +21,8 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.function.UnaryOperator; /** @@ -43,6 +45,7 @@ public class LockedApplication { private final Set<PublicKey> deployKeys; private final OptionalLong projectId; private final Optional<ApplicationVersion> latestVersion; + private final SortedSet<ApplicationVersion> versions; private final Map<InstanceName, Instance> instances; /** @@ -56,14 +59,14 @@ public class LockedApplication { application.deploymentSpec(), application.validationOverrides(), application.deploymentIssueId(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(), - application.projectId(), application.latestVersion(), application.instances()); + application.projectId(), application.latestVersion(), application.versions(), application.instances()); } private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, - OptionalLong projectId, Optional<ApplicationVersion> latestVersion, + OptionalLong projectId, Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, Map<InstanceName, Instance> instances) { this.lock = lock; this.id = id; @@ -78,6 +81,7 @@ public class LockedApplication { this.deployKeys = deployKeys; this.projectId = projectId; this.latestVersion = latestVersion; + this.versions = versions; this.instances = Map.copyOf(instances); } @@ -85,7 +89,7 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances.values()); + projectId, latestVersion, versions, instances.values()); } LockedApplication withNewInstance(InstanceName instance) { @@ -93,7 +97,7 @@ public class LockedApplication { instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) { @@ -101,7 +105,7 @@ public class LockedApplication { instances.put(instance, modification.apply(instances.get(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication without(InstanceName instance) { @@ -109,49 +113,51 @@ public class LockedApplication { instances.remove(instance); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withNewSubmission(ApplicationVersion latestVersion) { + SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); + applicationVersions.add(latestVersion); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, Optional.of(latestVersion), instances); + projectId, Optional.of(latestVersion), applicationVersions, instances); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } /** Set a major version for this, or set to null to remove any major version override */ @@ -159,13 +165,13 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, deployKeys, projectId, latestVersion, instances); + metrics, deployKeys, projectId, latestVersion, versions, instances); } public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withDeployKey(PublicKey pemDeployKey) { @@ -173,7 +179,7 @@ public class LockedApplication { keys.add(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); } public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { @@ -181,7 +187,15 @@ public class LockedApplication { keys.remove(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, instances); + projectId, latestVersion, versions, instances); + } + + public LockedApplication withoutVersion(ApplicationVersion version) { + SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); + applicationVersions.remove(version); + return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, + projectId, latestVersion, applicationVersions, instances); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 3f4ce7cc49b..bcc3b9b54c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -141,7 +141,7 @@ public abstract class LockedTenant { } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Optional.empty(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 943d6ac7b18..9ed808b3f01 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; @@ -15,9 +13,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -73,7 +68,6 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag hideSharedRoutingEndpoint; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -81,7 +75,6 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -115,13 +108,10 @@ public class RoutingController { public EndpointList readEndpointsOf(DeploymentId deployment) { Set<Endpoint> endpoints = new LinkedHashSet<>(); boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent(); - // Avoid reading application more than once per call to this - Supplier<DeploymentSpec> deploymentSpec = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId())).deploymentSpec()); // To discover the cluster name for a zone-scoped endpoint, we need to read routing policies for (var policy : routingPolicies.read(deployment)) { if (!policy.status().isActive()) continue; for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { - if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, deploymentSpec.get())) continue; endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, controller.zoneRegistry())); endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod)); } @@ -186,17 +176,21 @@ public class RoutingController { return EndpointList.copyOf(endpoints); } - /** Read and return zone-scoped endpoints for given deployments, grouped by their zone */ - public Map<ZoneId, List<Endpoint>> readZoneEndpointsOf(Collection<DeploymentId> deployments) { - var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value)); + /** Read test runner endpoints for given deployments, grouped by their zone */ + public Map<ZoneId, List<Endpoint>> readTestRunnerEndpointsOf(Collection<DeploymentId> deployments) { + TreeMap<ZoneId, List<Endpoint>> endpoints = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { - EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone).not().legacy(); - zoneEndpoints = directEndpoints(zoneEndpoints, deployment.applicationId()); + EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone) + .not().legacy(); + EndpointList directEndpoints = zoneEndpoints.direct(); + if (!directEndpoints.isEmpty()) { + zoneEndpoints = directEndpoints; // Use only direct endpoints if we have any + } if ( ! zoneEndpoints.isEmpty()) { endpoints.put(deployment.zoneId(), zoneEndpoints.asList()); } } - return Collections.unmodifiableMap(endpoints); + return Collections.unmodifiableSortedMap(endpoints); } /** Returns certificate DNS names (CN and SAN values) for given deployment */ @@ -355,17 +349,6 @@ public class RoutingController { Priority.normal)); } - /** Returns direct routing endpoints if any exist and feature flag is set for given application */ - // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant - public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { - boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - EndpointList directEndpoints = endpoints.direct(); - if (hideSharedEndpoint && !directEndpoints.isEmpty()) { - return directEndpoints; - } - return endpoints; - } - /** * Assigns one or more global rotations to given application, if eligible. The given application is implicitly * stored, ensuring that the assigned rotation(s) are persisted when this returns. @@ -386,7 +369,7 @@ public class RoutingController { } /** Returns the routing methods that are available across all given deployments */ - private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) { + private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments) { var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>(); for (var deployment : deployments) { for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) { @@ -397,40 +380,17 @@ public class RoutingController { var routingMethods = new ArrayList<RoutingMethod>(); deploymentsByMethod.forEach((method, supportedDeployments) -> { if (supportedDeployments.containsAll(deployments)) { - if (method.isDirect() && !canRouteDirectlyTo(deployments, deploymentSpec)) return; routingMethods.add(method); } }); return Collections.unmodifiableList(routingMethods); } - /** Returns whether traffic can be directly routed to all given deployments */ - private boolean canRouteDirectlyTo(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) { - return deployments.stream().allMatch(deployment -> canRouteDirectlyTo(deployment, deploymentSpec)); - } - - /** Returns whether traffic can be directly routed to given deployment */ - private boolean canRouteDirectlyTo(DeploymentId deploymentId, DeploymentSpec deploymentSpec) { - if (controller.system().isPublic()) return true; // Public always supports direct routing - if (controller.system().isCd()) return true; // CD deploys directly so we cannot enforce all requirements below - if (deploymentId.zoneId().environment().isManuallyDeployed()) return true; // Manually deployed zones always support direct routing - - // Check Athenz service presence. The test framework uses this identity when sending requests to the - // deployment's container(s). - var athenzService = deploymentSpec.instance(deploymentId.applicationId().instance()) - .flatMap(instance -> instance.athenzService(deploymentId.zoneId().environment(), - deploymentId.zoneId().region())); - if (athenzService.isEmpty()) return false; - return true; - } - /** Compute global endpoints for given routing ID, application and deployments */ private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, DeploymentSpec deploymentSpec) { var endpoints = new ArrayList<Endpoint>(); var directMethods = 0; - var availableRoutingMethods = routingMethodsOfAll(deployments, deploymentSpec); - boolean legacyNamesAvailable = requiresLegacyNames(deploymentSpec, routingId.instance().instance()); - + var availableRoutingMethods = routingMethodsOfAll(deployments); for (var method : availableRoutingMethods) { if (method.isDirect() && ++directMethods > 1) { throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " + @@ -441,21 +401,6 @@ public class RoutingController { .on(Port.fromRoutingMethod(method)) .routingMethod(method) .in(controller.system())); - // Add legacy endpoints - if (legacyNamesAvailable && method == RoutingMethod.shared) { - endpoints.add(Endpoint.of(routingId.instance()) - .target(routingId.endpointId(), cluster, deployments) - .on(Port.plain(4080)) - .legacy() - .routingMethod(method) - .in(controller.system())); - endpoints.add(Endpoint.of(routingId.instance()) - .target(routingId.endpointId(), cluster, deployments) - .on(Port.tls(4443)) - .legacy() - .routingMethod(method) - .in(controller.system())); - } } return endpoints; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 8f37d287c1a..7fc6b927bdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -42,7 +42,6 @@ public class Endpoint { private final Scope scope; private final boolean legacy; private final RoutingMethod routingMethod; - private final boolean tls; private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id, ClusterSpec.Id cluster, URI url, List<Target> targets, Scope scope, Port port, boolean legacy, @@ -63,7 +62,6 @@ public class Endpoint { this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; - this.tls = port.tls; } /** @@ -125,7 +123,7 @@ public class Endpoint { /** Returns whether this endpoint supports TLS connections */ public boolean tls() { - return tls; + return true; } /** Returns whether this requires a rotation to be reachable */ @@ -164,10 +162,9 @@ public class Endpoint { private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) { - String scheme = port.tls ? "https" : "http"; - String separator = separator(system, routingMethod, port.tls); + String separator = "."; String portPart = port.isDefault() ? "" : ":" + port.port; - return URI.create(scheme + "://" + + return URI.create("https://" + sanitize(namePart(name, separator)) + systemPart(system, separator) + sanitize(instancePart(instance, separator)) + @@ -185,13 +182,6 @@ public class Endpoint { return part.replace('_', '-'); } - private static String separator(SystemName system, RoutingMethod routingMethod, boolean tls) { - if (!tls) return "."; - if (routingMethod.isDirect()) return "."; - if (system.isPublic()) return "."; - return "--"; - } - private static String namePart(String name, String separator) { if ("default".equals(name)) return ""; return name + separator; @@ -390,21 +380,19 @@ public class Endpoint { /** Represents an endpoint's HTTP port */ public static class Port { - private static final Port TLS_DEFAULT = new Port(443, true); + private static final Port TLS_DEFAULT = new Port(443); private final int port; - private final boolean tls; - private Port(int port, boolean tls) { + private Port(int port) { if (port < 1 || port > 65535) { throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); } this.port = port; - this.tls = tls; } private boolean isDefault() { - return port == 80 || port == 443; + return port == 443; } /** Returns the default HTTPS port */ @@ -420,12 +408,7 @@ public class Endpoint { /** Create a HTTPS port */ public static Port tls(int port) { - return new Port(port, true); - } - - /** Create a HTTP port */ - public static Port plain(int port) { - return new Port(port, false); + return new Port(port); } } @@ -499,7 +482,7 @@ public class Endpoint { private ClusterSpec.Id cluster; private EndpointId endpointId; private Port port; - private RoutingMethod routingMethod = RoutingMethod.shared; + private RoutingMethod routingMethod = RoutingMethod.sharedLayer4; private boolean legacy = false; private boolean certificateName = false; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 85762cf530a..1354f173633 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; @@ -81,10 +82,14 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL return matching(id -> instance(id).change().hasTargets()); } + /** Returns the subset of instances which are currently deploying a new revision */ + public InstanceList changingRevision() { + return matching(id -> instance(id).change().application().isPresent()); + } + /** Returns the subset of instances which currently have failing jobs on the given version */ public InstanceList failingOn(Version version) { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failing() - .not().withStatus(outOfCapacity) + return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard() .lastCompleted().on(version).isEmpty()); } @@ -93,14 +98,14 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL return matching(id -> ! instance(id).change().isPinned()); } - /** Returns the subset of instances which are not currently failing any jobs. */ + /** Returns the subset of instances which are currently failing a job. */ public InstanceList failing() { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failing().not().withStatus(outOfCapacity).isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard().isEmpty()); } /** Returns the subset of instances which are currently failing an upgrade. */ public InstanceList failingUpgrade() { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failing().not().failingApplicationChange().isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard().not().failingApplicationChange().isEmpty()); } /** Returns the subset of instances which are upgrading (to any version), not considering block windows. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index b088c2fd0fd..da62175089d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; +import com.google.common.hash.Funnel; import com.google.common.hash.Hashing; import com.yahoo.component.Version; import com.yahoo.config.application.FileSystemWrapper; @@ -37,8 +38,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.toMap; @@ -62,6 +66,7 @@ public class ApplicationPackage { private static final String servicesFile = "services.xml"; private final String contentHash; + private final String bundleHash; // Hash of all files except deployment.xml and build-meta private final byte[] zippedContent; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; @@ -102,6 +107,8 @@ public class ApplicationPackage { this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); + + this.bundleHash = calculateBundleHash(); } /** Returns a copy of this with the given certificate appended. */ @@ -117,6 +124,10 @@ public class ApplicationPackage { /** Returns a hash of the content of this package */ public String hash() { return contentHash; } + + public String bundleHash() { + return bundleHash; + } /** Returns the content of this package. The content <b>must not</b> be modified. */ public byte[] zippedContent() { return zippedContent; } @@ -209,6 +220,17 @@ public class ApplicationPackage { return ValidationOverrides.fromXml(validationOverridesContents.toString()); } + // Hashes all files that require a deployment to be forwarded to configservers + private String calculateBundleHash() { + Predicate<String> entryMatcher = name -> !name.endsWith(deploymentFile) && !name.endsWith(buildMetaFile); + SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher); + Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.entrySet().forEach(entry -> { + into.putBytes(entry.getKey().getBytes()); + into.putLong(entry.getValue()); + }); + return Hashing.sha1().hashObject(entryCRCs, funnel).toString(); + } + /** Maps normalized paths to cached content read from a zip archive. */ private static class ZipArchiveCache { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java index f89cc0cc0fc..0eb7f3de3cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java @@ -9,8 +9,12 @@ import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.function.Predicate; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -46,7 +50,7 @@ public class ZipStreamReader { if (entry.getName().equals(name)) continue; - zipOut.putNextEntry(entry); + zipOut.putNextEntry(new ZipEntry(entry.getName())); zipIn.transferTo(zipOut); zipOut.closeEntry(); } @@ -59,6 +63,24 @@ public class ZipStreamReader { } } + public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) { + SortedMap<String, Long> entryCRCs = new TreeMap<>(); + byte[] buffer = new byte[2048]; + try (ZipInputStream zipIn = new ZipInputStream(in); + OutputStream os = new ByteArrayOutputStream()) { + for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { + if (!entryNameMatcher.test(entry.getName())) + continue; + // CRC is not set until entry is read + while ( -1 != zipIn.read(buffer)){} + entryCRCs.put(entry.getName(), entry.getCrc()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return entryCRCs; + } + private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) { try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { byte[] buffer = new byte[2048]; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 8b929cb9d10..21914b87818 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -5,9 +5,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; @@ -37,36 +34,19 @@ public class CuratorArchiveBucketDb { private final ArchiveService archiveService; private final CuratorDb curatorDb; - private final BooleanFlag enableFlag; private final SystemName system; public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); - this.enableFlag = Flags.ENABLE_ONPREM_TENANT_S3_ARCHIVE.bindTo(controller.flagSource()); this.system = controller.zoneRegistry().system(); } - public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant) { - if (enabled(zoneId, tenant)) { - return Optional.of(URI.create(Text.format("s3://%s/%s/", findOrAssignBucket(zoneId, tenant), tenant.value()))); - } else { - return Optional.empty(); - } - } - - private boolean enabled(ZoneId zone, TenantName tenant) { - return system.isPublic() || - enableFlag - .with(FetchVector.Dimension.ZONE_ID, zone.value()) - .with(FetchVector.Dimension.TENANT_ID, tenant.value()) - .value(); - } - - private String findOrAssignBucket(ZoneId zoneId, TenantName tenant) { + public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant, boolean createIfMissing) { return getBucketNameFromCache(zoneId, tenant) .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) - .orElseGet(() -> assignToBucket(zoneId, tenant)); + .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) + .map(bucketName -> URI.create(Text.format("s3://%s/%s/", bucketName, tenant.value()))); } private String assignToBucket(ZoneId zoneId, TenantName tenant) { @@ -92,8 +72,7 @@ public class CuratorArchiveBucketDb { } // We'll have to create a new bucket - var newBucket = archiveService.createArchiveBucketFor(zoneId, tenantsPerBucket().isPresent()) - .withTenant(tenant); + var newBucket = archiveService.createArchiveBucketFor(zoneId).withTenant(tenant); zoneBuckets.add(newBucket); curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); updateArchiveUriCache(zoneId, zoneBuckets); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index d116ef3333c..a0b70eb88ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -5,7 +5,9 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -248,9 +250,9 @@ public class AthenzFacade implements AccessControl { } public boolean hasApplicationAccess( - AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationName applicationName) { + AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationName applicationName, Optional<Zone> zone) { return hasAccess( - action.name(), applicationResourceString(tenantDomain, applicationName), identity); + action.name(), applicationResourceString(tenantDomain, applicationName, zone), identity); } public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { @@ -325,8 +327,10 @@ public class AthenzFacade implements AccessControl { return resourceStringPrefix(tenantDomain) + ".wildcard"; } - private String applicationResourceString(AthenzDomain tenantDomain, ApplicationName applicationName) { - return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.value() + ".wildcard"; + private String applicationResourceString(AthenzDomain tenantDomain, ApplicationName applicationName, Optional<Zone> zone) { + // If environment is not provided, add .wildcard to match .* in the policy resource (* is not allowed in the request) + String environment = zone.map(Zone::environment).map(Environment::value).orElse("wildcard"); + return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.value() + "." + environment; } private enum TenantAction { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java index 6d0b8d751ed..cb2fca3e411 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.handler.ContentChannel; /** @@ -12,7 +12,7 @@ import com.yahoo.jdisc.handler.ContentChannel; * * @author mpolden */ -public abstract class AuditLoggingRequestHandler extends LoggingRequestHandler { +public abstract class AuditLoggingRequestHandler extends ThreadedHttpRequestHandler { private final AuditLogger auditLogger; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index ce03e84f2b9..f03d32aa838 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -7,12 +7,14 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; +import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Change; @@ -23,10 +25,13 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -106,11 +111,42 @@ public class DeploymentStatus { return allJobs; } + /** Whether any jobs both dependent on the dependency, and a dependency for the dependent, are failing. */ + private boolean hasFailures(StepStatus dependency, StepStatus dependent) { + Set<StepStatus> dependents = new HashSet<>(); + fillDependents(dependency, new HashSet<>(), dependents, dependent); + Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(Collectors.toSet()); + + return ! allJobs.matching(job -> criticalJobs.contains(job.id())) + .failingHard() + .isEmpty(); + } + + private boolean fillDependents(StepStatus dependency, Set<StepStatus> visited, Set<StepStatus> dependents, StepStatus current) { + if (visited.contains(current)) + return dependents.contains(current); + + if (dependency == current) + dependents.add(current); + else + for (StepStatus dep : current.dependencies) + if (fillDependents(dependency, visited, dependents, dep)) + dependents.add(current); + + visited.add(current); + return dependents.contains(current); + } + + /** Whether any job is failing on anything older than version, with errors other than lack of capacity in a test zone.. */ + public boolean hasFailures(ApplicationVersion version) { + return ! allJobs.failingHard() + .matching(job -> job.lastTriggered().get().versions().targetApplication().compareTo(version) < 0) + .isEmpty(); + } + /** Whether any jobs of this application are failing with other errors than lack of capacity in a test zone. */ public boolean hasFailures() { - return ! allJobs.failing() - .not().withStatus(RunStatus.outOfCapacity) - .isEmpty(); + return ! allJobs.failingHard().isEmpty(); } /** All job statuses, by job type, for the given instance. */ @@ -128,13 +164,13 @@ public class DeploymentStatus { /** * The set of jobs that need to run for the changes of each instance of the application to be considered complete, - * and any test jobs for any oustanding change, which will likely be needed to lated deploy this change. + * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. */ - public Map<JobId, List<Versions>> jobsToRun() { + public Map<JobId, List<Job>> jobsToRun() { Map<InstanceName, Change> changes = new LinkedHashMap<>(); for (InstanceName instance : application.deploymentSpec().instanceNames()) changes.put(instance, application.require(instance).change()); - Map<JobId, List<Versions>> jobs = jobsToRun(changes); + Map<JobId, List<Job>> jobs = jobsToRun(changes); // Add test jobs for any outstanding change. for (InstanceName instance : application.deploymentSpec().instanceNames()) @@ -150,12 +186,12 @@ public class DeploymentStatus { Collections::unmodifiableMap)); } - private Map<JobId, List<Versions>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { - Map<JobId, Versions> productionJobs = new LinkedHashMap<>(); + private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { + Map<JobId, List<Job>> productionJobs = new LinkedHashMap<>(); changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change, eagerTests))); - Map<JobId, List<Versions>> testJobs = testJobs(productionJobs); - Map<JobId, List<Versions>> jobs = new LinkedHashMap<>(testJobs); - productionJobs.forEach((job, versions) -> jobs.put(job, List.of(versions))); + Map<JobId, List<Job>> testJobs = testJobs(productionJobs); + Map<JobId, List<Job>> jobs = new LinkedHashMap<>(testJobs); + jobs.putAll(productionJobs); // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. jobSteps.forEach((job, step) -> { if ( ! step.isDeclared() || jobs.containsKey(job)) @@ -172,13 +208,13 @@ public class DeploymentStatus { Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion); if (step.completedAt(change, firstProductionJobWithDeployment).isEmpty()) - jobs.merge(job, List.of(versions), DeploymentStatus::union); + jobs.merge(job, List.of(new Job(versions, step.readyAt(change), change)), DeploymentStatus::union); }); return Collections.unmodifiableMap(jobs); } /** The set of jobs that need to run for the given changes to be considered complete. */ - public Map<JobId, List<Versions>> jobsToRun(Map<InstanceName, Change> changes) { + public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { return jobsToRun(changes, false); } @@ -215,72 +251,209 @@ public class DeploymentStatus { * and has not yet started rolling out, due to some other change or a block window being present at the time of submission. */ public Change outstandingChange(InstanceName instance) { - return application.latestVersion().map(Change::of) + return nextVersion(instance).map(Change::of) .filter(change -> application.require(instance).change().application().map(change::upgrades).orElse(true)) .filter(change -> ! jobsToRun(Map.of(instance, change)).isEmpty()) .orElse(Change.empty()); } - /** - * True if the job has already been triggered on the given versions, or if all test types (systemTest, stagingTest), - * restricted to the job's instance if declared in that instance, have successful runs on the given versions. - */ - public boolean isTested(JobId job, Change change) { - Versions versions = Versions.from(change, application, deploymentFor(job), systemVersion); - return allJobs.triggeredOn(versions).get(job).isPresent() - || Stream.of(systemTest, stagingTest) - .noneMatch(testType -> declaredTest(job.application(), testType).map(__ -> allJobs.instance(job.application().instance())) - .orElse(allJobs) - .type(testType) - .successOn(versions).isEmpty()); - } - - private Map<JobId, Versions> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { - ImmutableMap.Builder<JobId, Versions> jobs = ImmutableMap.builder(); + /** The next application version to roll out to instance. */ + private Optional<ApplicationVersion> nextVersion(InstanceName instance) { + return Optional.ofNullable(instanceSteps().get(instance)).stream() + .flatMap(this::allDependencies) + .flatMap(step -> step.instance.latestDeployed().stream()) + .min(naturalOrder()) + .or(application::latestVersion); + } + + private Stream<InstanceStatus> allDependencies(StepStatus step) { + return step.dependencies.stream() + .flatMap(dep -> Stream.concat(Stream.of(dep), allDependencies(dep))) + .filter(InstanceStatus.class::isInstance) + .map(InstanceStatus.class::cast) + .distinct(); + } + + /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ + public Optional<Instant> verifiedAt(JobId job, Versions versions) { + Optional<Instant> triggeredAt = allJobs.get(job) + .flatMap(status -> status.runs().values().stream() + .filter(run -> run.versions().equals(versions)) + .findFirst()) + .map(Run::start); + Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest, versions); + Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest, versions); + if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt; + Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt; + return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt; + } + + /** Earliest instant when versions were tested for the given instance */ + private Optional<Instant> testedAt(ApplicationId instance, JobType type, Versions versions) { + return declaredTest(instance, type).map(__ -> allJobs.instance(instance.instance())) + .orElse(allJobs) + .type(type).asList().stream() + .flatMap(status -> RunList.from(status) + .on(versions) + .status(RunStatus.success) + .asList().stream() + .map(Run::start)) + .min(naturalOrder()); + } + + private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { + Map<JobId, List<Job>> jobs = new LinkedHashMap<>(); jobSteps.forEach((job, step) -> { // When computing eager test jobs for outstanding changes, assume current upgrade completes successfully. Optional<Deployment> deployment = deploymentFor(job) - .map(existing -> assumeUpgradesSucceed ? new Deployment(existing.zone(), - existing.applicationVersion(), - change.platform().orElse(existing.version()), - existing.at(), - existing.metrics(), - existing.activity(), - existing.quota(), - existing.cost()) - : existing); - if ( job.application().instance().equals(instance) - && job.type().isProduction() - && step.completedAt(change).isEmpty()) - jobs.put(job, Versions.from(change, application, deployment, systemVersion)); + .map(existing -> assumeUpgradesSucceed ? withChange(existing, change.withoutApplication()) : existing); + if (job.application().instance().equals(instance) && job.type().isProduction()) { + + List<Job> toRun = new ArrayList<>(); + List<Change> changes = changes(job, step, change); + if (changes.isEmpty()) return; + for (Change partial : changes) { + toRun.add(new Job(Versions.from(partial, application, deployment, systemVersion), + step.readyAt(partial, Optional.of(job)), + partial)); + // Assume first partial change is applied before the second. + deployment = deployment.map(existing -> withChange(existing, partial)); + } + jobs.put(job, toRun); + } }); - return jobs.build(); + return jobs; } - /** The production jobs that need to run to complete roll-out of the given change to production. */ - public Map<JobId, Versions> productionJobs(InstanceName instance, Change change) { - return productionJobs(instance, change, false); + private static Deployment withChange(Deployment deployment, Change change) { + return new Deployment(deployment.zone(), + change.application().orElse(deployment.applicationVersion()), + change.platform().orElse(deployment.version()), + deployment.at(), + deployment.metrics(), + deployment.activity(), + deployment.quota(), + deployment.cost()); + } + + /** Changes to deploy with the given job, possibly split in two steps. */ + private List<Change> changes(JobId job, StepStatus step, Change change) { + // Signal strict completion criterion by depending on job itself. + if (step.completedAt(change, Optional.of(job)).isPresent()) + return List.of(); + + if (change.platform().isEmpty() || change.application().isEmpty() || change.isPinned()) + return List.of(change); + + if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() + || step.completedAt(change.withoutPlatform(), Optional.of(job)).isPresent()) + return List.of(change); + + // For a dual change, where both target remain, we determine what to run by looking at when the two parts became ready: + // for deployments, we look at dependencies; for tests, this may be overridden by what is already deployed. + JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); + if (job.type().isTest()) { + Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); + Optional<Instant> revisionDeployedAt = jobSteps.get(deployment).completedAt(change.withoutPlatform(), Optional.of(deployment)); + + // If only the revision has deployed, then we expect to test that first. + if (platformDeployedAt.isEmpty() && revisionDeployedAt.isPresent()) return List.of(change.withoutPlatform(), change); + + // If only the upgrade has deployed, then we expect to test that first, with one exception: + // The revision has caught up to the upgrade at the deployment job; and either + // the upgrade is failing between deployment and here, or + // the specified rollout is leading or simultaneous; and + // the revision is now blocked by waiting for the production test to verify the upgrade. + // In this case we must abandon the production test on the pure upgrade, so the revision can be deployed. + if (platformDeployedAt.isPresent() && revisionDeployedAt.isEmpty()) { + if (jobSteps.get(deployment).readyAt(change, Optional.of(deployment)) + .map(ready -> ! now.isBefore(ready)).orElse(false)) { + switch (rollout) { + // If separate rollout, this test should keep blocking the revision, unless there are failures. + case separate: return hasFailures(jobSteps.get(deployment), jobSteps.get(job)) ? List.of(change) : List.of(change.withoutApplication(), change); + // If leading rollout, this test should now expect the two changes to fuse and roll together. + case leading: return List.of(change); + // If simultaneous rollout, this test should now expect the revision to run ahead. + case simultaneous: return List.of(change.withoutPlatform(), change); + } + } + return List.of(change.withoutApplication(), change); + } + // If neither is deployed, then neither is ready, and we guess like for deployments. + // If both are deployed, then we need to follow normal logic for whatever is ready. + } + + Optional<Instant> platformReadyAt = step.dependenciesCompletedAt(change.withoutApplication(), Optional.of(job)); + Optional<Instant> revisionReadyAt = step.dependenciesCompletedAt(change.withoutPlatform(), Optional.of(job)); + + // If neither change is ready, we guess based on the specified rollout. + if (platformReadyAt.isEmpty() && revisionReadyAt.isEmpty()) { + switch (rollout) { + case separate: return List.of(change.withoutApplication(), change); // Platform should stay ahead. + case leading: return List.of(change); // They should eventually join. + case simultaneous: return List.of(change.withoutPlatform(), change); // Revision should get ahead. + } + } + + // If only the revision is ready, we run that first. + if (platformReadyAt.isEmpty()) return List.of(change.withoutPlatform(), change); + + // If only the platform is ready, we run that first. + if (revisionReadyAt.isEmpty()) { + return List.of(change.withoutApplication(), change); + } + + // Both changes are ready for this step, and we look to the specified rollout to decide. + boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get()); + boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get()); + switch (rollout) { + case separate: // Let whichever change rolled out first, keep rolling first, unless upgrade alone is failing. + return (platformReadyFirst || platformReadyAt.get().equals(Instant.EPOCH)) // Assume platform was first if no jobs have run yet. + ? step.job().flatMap(jobs()::get).flatMap(JobStatus::firstFailing).isPresent() + ? List.of(change) // Platform was first, but is failing. + : List.of(change.withoutApplication(), change) // Platform was first, and is OK. + : revisionReadyFirst + ? List.of(change.withoutPlatform(), change) // Revision was first. + : List.of(change); // Both ready at the same time, probably due to earlier failure. + case leading: // When one change catches up, they fuse and continue together. + return List.of(change); + case simultaneous: // Revisions are allowed to run ahead, but the job where it caught up should have both changes. + return platformReadyFirst ? List.of(change) : List.of(change.withoutPlatform(), change); + default: throw new IllegalStateException("Unknown upgrade rollout policy"); + } } /** The test jobs that need to run prior to the given production deployment jobs. */ - public Map<JobId, List<Versions>> testJobs(Map<JobId, Versions> jobs) { - Map<JobId, List<Versions>> testJobs = new LinkedHashMap<>(); + public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) { + Map<JobId, List<Job>> testJobs = new LinkedHashMap<>(); for (JobType testType : List.of(systemTest, stagingTest)) { - jobs.forEach((job, versions) -> { + jobs.forEach((job, versionsList) -> { if (job.type().isProduction() && job.type().isDeployment()) { declaredTest(job.application(), testType).ifPresent(testJob -> { - if (allJobs.successOn(versions).get(testJob).isEmpty()) - testJobs.merge(testJob, List.of(versions), DeploymentStatus::union); + for (Job productionJob : versionsList) + if (allJobs.successOn(productionJob.versions()).get(testJob).isEmpty()) + testJobs.merge(testJob, List.of(new Job(productionJob.versions(), + jobSteps().get(testJob).readyAt(productionJob.change), + productionJob.change)), + DeploymentStatus::union); }); } }); - jobs.forEach((job, versions) -> { - if ( job.type().isProduction() && job.type().isDeployment() - && allJobs.successOn(versions).type(testType).isEmpty() - && testJobs.keySet().stream() - .noneMatch(test -> test.type() == testType - && testJobs.get(test).contains(versions))) - testJobs.merge(firstDeclaredOrElseImplicitTest(testType), List.of(versions), DeploymentStatus::union); + jobs.forEach((job, versionsList) -> { + for (Job productionJob : versionsList) + if ( job.type().isProduction() && job.type().isDeployment() + && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() + && testJobs.keySet().stream() + .noneMatch(test -> test.type() == testType + && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { + JobId testJob = firstDeclaredOrElseImplicitTest(testType); + testJobs.merge(testJob, + List.of(new Job(productionJob.versions(), + jobSteps.get(testJob).readyAt(productionJob.change), + productionJob.change)), + DeploymentStatus::union); + } }); } return Collections.unmodifiableMap(testJobs); @@ -314,7 +487,7 @@ public class DeploymentStatus { /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */ private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, DeploymentSpec.Step step, List<StepStatus> previous, InstanceName instance) { - if (step.steps().isEmpty()) { + if (step.steps().isEmpty() && ! (step instanceof DeploymentInstanceSpec)) { if (instance == null) return previous; // Ignore test and staging outside all instances. @@ -357,7 +530,7 @@ public class DeploymentStatus { if (step instanceof DeploymentInstanceSpec) { DeploymentInstanceSpec spec = ((DeploymentInstanceSpec) step); - StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()), this); + StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name())); instance = spec.name(); allSteps.add(instanceStatus); previous = List.of(instanceStatus); @@ -416,7 +589,7 @@ public class DeploymentStatus { private final StepType type; private final DeploymentSpec.Step step; - private final List<StepStatus> dependencies; + private final List<StepStatus> dependencies; // All direct dependencies of this step. private final InstanceName instance; private StepStatus(StepType type, DeploymentSpec.Step step, List<StepStatus> dependencies, InstanceName instance) { @@ -462,11 +635,13 @@ public class DeploymentStatus { /** The time at which all dependencies completed on the given change and / or versions. */ Optional<Instant> dependenciesCompletedAt(Change change, Optional<JobId> dependent) { - return dependencies.stream().allMatch(step -> step.completedAt(change, dependent).isPresent()) - ? dependencies.stream().map(step -> step.completedAt(change, dependent).get()) - .max(naturalOrder()) - .or(() -> Optional.of(Instant.EPOCH)) - : Optional.empty(); + Instant latest = Instant.EPOCH; + for (StepStatus step : dependencies) { + Optional<Instant> completedAt = step.completedAt(change, dependent); + if (completedAt.isEmpty()) return Optional.empty(); + latest = latest.isBefore(completedAt.get()) ? completedAt.get() : latest; + } + return Optional.of(latest); } /** The time until which this step is blocked by a change blocker. */ @@ -491,7 +666,7 @@ public class DeploymentStatus { } @Override - public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return readyAt(change, dependent).map(completion -> completion.plus(step().delay())); } @@ -503,15 +678,13 @@ public class DeploymentStatus { private final DeploymentInstanceSpec spec; private final Instant now; private final Instance instance; - private final DeploymentStatus status; private InstanceStatus(DeploymentInstanceSpec spec, List<StepStatus> dependencies, Instant now, - Instance instance, DeploymentStatus status) { + Instance instance) { super(StepType.instance, spec, dependencies, spec.name()); this.spec = spec; this.now = now; this.instance = instance; - this.status = status; } /** @@ -519,10 +692,10 @@ public class DeploymentStatus { * for this instance, or if no more jobs should run for this instance for the given change. */ @Override - public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) && (change.application().isEmpty() || change.application().equals(instance.change().application())) - || status.jobsToRun(Map.of(instance.name(), change)).isEmpty()) + || step().steps().stream().noneMatch(step -> step.concerns(prod))) ? dependenciesCompletedAt(change, dependent) : Optional.empty(); } @@ -600,28 +773,38 @@ public class DeploymentStatus { @Override public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - return super.readyAt(change, Optional.of(job.id())) - .filter(__ -> status.isTested(job.id(), change)); + Optional<Instant> readyAt = super.readyAt(change, dependent); + Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.systemVersion)); + if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty(); + return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt; } /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ @Override - public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { if ( change.isPinned() && change.platform().isPresent() && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); + if ( change.application().isPresent() + && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application()) + && dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait. + return Optional.empty(); + Change fullChange = status.application().require(instance).change(); if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.applicationVersion())) && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.applicationVersion()))) .orElse(false)) return job.lastCompleted().flatMap(Run::end); - return job.lastSuccess() - .filter(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) - && change.application().map(run.versions().targetApplication()::equals).orElse(true)) - .flatMap(Run::end); + return (dependent.equals(job()) ? job.lastSuccess().stream() + : RunList.from(job).status(RunStatus.success).asList().stream()) + .filter(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) + && change.application().map(run.versions().targetApplication()::equals).orElse(true)) + .map(Run::end) + .flatMap(Optional::stream) + .min(naturalOrder()); } }; } @@ -631,16 +814,21 @@ public class DeploymentStatus { JobStatus job = status.instanceJobs(instance).get(testType); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override - public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion); - return job.lastSuccess() - .filter(run -> versions.targetsMatch(run.versions())) - .filter(run -> ! status.jobs() - .instance(instance) - .type(prodType) - .lastCompleted().endedNoLaterThan(run.start()) - .isEmpty()) - .map(run -> run.end().get()); + return dependent.equals(job()) ? job.lastSuccess() + .filter(run -> versions.targetsMatch(run.versions())) + .filter(run -> ! status.jobs() + .instance(instance) + .type(prodType) + .lastCompleted().endedNoLaterThan(run.start()) + .isEmpty()) + .map(run -> run.end().get()) + : RunList.from(job) + .matching(run -> versions.targetsMatch(run.versions())) + .status(RunStatus.success) + .first() + .map(run -> run.end().get()); } }; } @@ -651,7 +839,7 @@ public class DeploymentStatus { JobStatus job = status.instanceJobs(instance).get(jobType); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override - public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return RunList.from(job) .matching(run -> run.versions().targetsMatch(Versions.from(change, status.application, @@ -670,4 +858,39 @@ public class DeploymentStatus { } + public static class Job { + + private final Versions versions; + private final Optional<Instant> readyAt; + private final Change change; + + public Job(Versions versions, Optional<Instant> readyAt, Change change) { + this.versions = versions; + this.readyAt = readyAt; + this.change = change; + } + + public Versions versions() { + return versions; + } + + public Optional<Instant> readyAt() { + return readyAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Job job = (Job) o; + return versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); + } + + @Override + public int hashCode() { + return Objects.hash(versions, readyAt, change); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index c6e4d02acfa..22df5ca559e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -24,7 +24,7 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus /** Returns the subset of applications which have changes left to deploy; blocked, or deploying */ public DeploymentStatusList withChanges() { - return matching(status -> status.application().instances().values().stream() + return matching(status -> status.application().productionInstances().values().stream() .anyMatch(instance -> instance.change().hasTargets() || status.outstandingChange(instance.name()).hasTargets())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 4e8f17b6098..3be21aec608 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -97,12 +97,9 @@ public class DeploymentTrigger { && status.instanceSteps().get(instanceName) .readyAt(outstanding) .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) - && acceptNewApplicationVersion(status, instanceName)) { + && acceptNewApplicationVersion(status, instanceName, outstanding.application().get())) { application = application.with(instanceName, - instance -> { - instance = instance.withChange(instance.change().with(outstanding.application().get())); - return instance.withChange(remainingChange(instance, status)); - }); + instance -> withRemainingChange(instance, instance.change().with(outstanding.application().get()), status)); } } applications().store(application); @@ -121,7 +118,7 @@ public class DeploymentTrigger { applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> applications().store(application.with(id.instance(), - instance -> instance.withChange(remainingChange(instance, jobs.deploymentStatus(application.get())))))); + instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get()))))); } /** @@ -201,11 +198,12 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application); Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion()); - Map<JobId, List<Versions>> jobs = status.testJobs(Map.of(job, versions)); + DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(versions, Optional.of(controller.clock().instant()), instance.change()); + Map<JobId, List<DeploymentStatus.Job>> jobs = status.testJobs(Map.of(job, List.of(toTrigger))); if (jobs.isEmpty() || ! requireTests) - jobs = Map.of(job, List.of(versions)); + jobs = Map.of(job, List.of(toTrigger)); jobs.forEach((jobId, versionsList) -> { - trigger(deploymentJob(instance, versionsList.get(0), jobId.type(), status.jobs().get(jobId).get(), clock.instant())); + trigger(deploymentJob(instance, versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant())); }); return List.copyOf(jobs.keySet()); } @@ -277,13 +275,8 @@ public class DeploymentTrigger { /** Overrides the given instance's platform and application changes with any contained in the given change. */ public void forceChange(ApplicationId instanceId, Change change) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - Change newChange = change.onTopOf(application.get().require(instanceId.instance()).change()); - application = application.with(instanceId.instance(), - instance -> instance.withChange(newChange)); - DeploymentStatus newStatus = jobs.deploymentStatus(application.get()); - application = application.with(instanceId.instance(), - instance -> instance.withChange(remainingChange(instance, newStatus))); - applications().store(application); + applications().store(application.with(instanceId.instance(), + instance -> withRemainingChange(instance, change.onTopOf(application.get().require(instanceId.instance()).change()), jobs.deploymentStatus(application.get())))); }); } @@ -300,7 +293,7 @@ public class DeploymentTrigger { default: throw new IllegalArgumentException("Unknown cancellation choice '" + cancellation + "'!"); } applications().store(application.with(instanceId.instance(), - instance -> instance.withChange(change))); + instance -> withRemainingChange(instance, change, jobs.deploymentStatus(application.get())))); }); } @@ -329,19 +322,18 @@ public class DeploymentTrigger { /** Finds the next step to trigger for the given application, if any, and returns these as a list. */ private List<Job> computeReadyJobs(DeploymentStatus status) { List<Job> jobs = new ArrayList<>(); - status.jobsToRun().forEach((job, versionsList) -> { - for (Versions versions : versionsList) - status.jobSteps().get(job).readyAt(status.application().require(job.application().instance()).change()) - .filter(readyAt -> ! clock.instant().isBefore(readyAt)) - .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) - .filter(__ -> abortIfRunning(versionsList, status.jobs().get(job).get())) // Abort and trigger this later if running with outdated parameters. - .ifPresent(readyAt -> { - jobs.add(deploymentJob(status.application().require(job.application().instance()), - versions, - job.type(), - status.instanceJobs(job.application().instance()).get(job.type()), - readyAt)); - }); + Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); + jobsToRun.forEach((job, versionsList) -> { + versionsList.get(0).readyAt() + .filter(readyAt -> ! clock.instant().isBefore(readyAt)) + .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) + .filter(__ -> abortIfRunning(status, jobsToRun, job)) // Abort and trigger this later if running with outdated parameters. + .map(readyAt -> deploymentJob(status.application().require(job.application().instance()), + versionsList.get(0).versions(), + job.type(), + status.instanceJobs(job.application().instance()).get(job.type()), + readyAt)) + .ifPresent(jobs::add); }); return Collections.unmodifiableList(jobs); } @@ -356,36 +348,62 @@ public class DeploymentTrigger { return false; } - /** Returns whether the job is not running, and also aborts it if it's running with outdated versions. */ - private boolean abortIfRunning(List<Versions> versionsList, JobStatus status) { - if ( ! status.isRunning()) - return true; + private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + status.jobs().get(job) + .flatMap(JobStatus::lastTriggered) + .filter(last -> ! last.hasEnded()) + .ifPresent(last -> { + if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) + && versions.versions().sourcesMatchIfPresent(last.versions()))) { + log.log(Level.INFO, "Aborting outdated run " + last); + controller.jobController().abort(last.id()); + } + }); + } - Run last = status.lastTriggered().get(); - if (versionsList.stream().noneMatch(versions -> versions.targetsMatch(last.versions()) - && versions.sourcesMatchIfPresent(last.versions()))) - controller.jobController().abort(last.id()); + /** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */ + private boolean abortIfRunning(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + abortIfOutdated(status, jobs, job); + boolean blocked = status.jobs().get(job).get().isRunning(); + + if ( ! job.type().isTest()) { + Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) + .map(type -> new JobId(job.application(), type)) + .flatMap(status.jobs()::get); + if (productionTest.isPresent()) { + abortIfOutdated(status, jobs, productionTest.get().id()); + // Production deployments are also blocked by their declared tests, if the next versions to run + // for those are not the same as the versions we're considering running in the deployment job now. + if (productionTest.map(JobStatus::id).map(jobs::get) + .map(versions -> ! versions.get(0).versions().targetsMatch(jobs.get(job).get(0).versions())) + .orElse(false)) + blocked = true; + } + } - return false; + return ! blocked; } // ---------- Change management o_O ---------- - private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance) { - if (status.application().require(instance).change().application().isPresent()) return true; // Replacing a previous application change is ok. - if (status.hasFailures()) return true; // Allow changes to fix upgrade problems. - if (status.application().deploymentSpec().instance(instance) // Leading upgrade allows app change to join in. - .map(spec -> spec.upgradeRollout() == DeploymentSpec.UpgradeRollout.leading).orElse(false)) return true; - return status.application().require(instance).change().platform().isEmpty(); + private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance, ApplicationVersion version) { + if (status.application().deploymentSpec().instance(instance).isEmpty()) return false; // Unknown instance. + if (status.hasFailures(version)) return true; // Allow changes to fix upgrade or previous revision problems. + DeploymentInstanceSpec spec = status.application().deploymentSpec().requireInstance(instance); + Change change = status.application().require(instance).change(); + return change.application().isEmpty() || spec.upgradeRevision() != DeploymentSpec.UpgradeRevision.separate; } - private Change remainingChange(Instance instance, DeploymentStatus status) { - Change change = instance.change(); - if (status.jobsToRun(Map.of(instance.name(), instance.change().withoutApplication())).isEmpty()) - change = change.withoutPlatform(); - if (status.jobsToRun(Map.of(instance.name(), instance.change().withoutPlatform())).isEmpty()) - change = change.withoutApplication(); - return change; + private Instance withRemainingChange(Instance instance, Change change, DeploymentStatus status) { + Change remaining = change; + if (status.jobsToRun(Map.of(instance.name(), change.withoutApplication())).isEmpty()) + remaining = remaining.withoutPlatform(); + if (status.jobsToRun(Map.of(instance.name(), change.withoutPlatform())).isEmpty()) { + remaining = remaining.withoutApplication(); + if (change.application().isPresent()) + instance = instance.withLatestDeployed(change.application().get()); + } + return instance.withChange(remaining); } // ---------- Version and job helpers ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 684c497571d..532f19eee21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -83,11 +83,11 @@ import static com.yahoo.config.application.api.Notifications.When.failing; import static com.yahoo.config.application.api.Notifications.When.failingCommit; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -101,6 +101,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.joining; @@ -153,7 +154,7 @@ public class InternalStepRunner implements StepRunner { case installReal: return installReal(id, logger); case startStagingSetup: return startTests(id, true, logger); case endStagingSetup: - case endTests: return endTests(id, logger); + case endTests: return endTests(id, logger); case startTests: return startTests(id, false, logger); case copyVespaLogs: return copyVespaLogs(id, logger); case deactivateReal: return deactivateReal(id, logger); @@ -222,6 +223,7 @@ public class InternalStepRunner implements StepRunner { logger); } + @SuppressWarnings("deprecation") private Optional<RunStatus> deploy(Supplier<ActivateResult> deployment, Instant startTime, DualLogger logger) { try { PrepareResponse prepareResponse = deployment.get().prepareResponse(); @@ -245,20 +247,21 @@ public class InternalStepRunner implements StepRunner { ? Optional.of(deploymentFailed) : Optional.empty(); switch (e.code()) { case CERTIFICATE_NOT_READY: - logger.log("Waiting for certificate to become ready on config server: New application, or old one has expired"); + logger.log("No valid CA signed certificate for app available to config server"); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "Certificate did not become available on config server within (" + timeouts.endpointCertificate() + ")"); + logger.log(WARNING, "CA signed certificate for app not available to config server within " + timeouts.endpointCertificate()); return Optional.of(RunStatus.endpointCertificateTimeout); } return result; case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: + case CONFIG_NOT_CONVERGED: logger.log("Deployment failed with possibly transient error " + e.code() + ", will retry: " + e.getMessage()); return result; case LOAD_BALANCER_NOT_READY: case PARENT_HOST_NOT_READY: - logger.log(e.message()); + logger.log(e.message()); // Consider splitting these messages in summary and details, on config server. return result; case OUT_OF_CAPACITY: logger.log(e.message()); @@ -277,9 +280,9 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Waiting for certificate to become valid: New application, or old one has expired"); + logger.log("Validating CA signed certificate requested for app: not yet available"); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "Controller could not validate certificate within " + + logger.log(WARNING, "CA signed certificate for app not available within " + timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e)); return Optional.of(RunStatus.endpointCertificateTimeout); } @@ -301,7 +304,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installReal(RunId id, boolean setTheStage, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); if (deployment.isEmpty()) { - logger.log(INFO, "Deployment expired before installation was successful."); + logger.log("Deployment expired before installation was successful."); return Optional.of(installationFailed); } @@ -324,15 +327,32 @@ public class InternalStepRunner implements StepRunner { List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), NodeFilter.all() .hostnames(parentHostnames)); - NodeList nodeList = NodeList.of(nodes, parents, services.get()); boolean firstTick = run.convergenceSummary().isEmpty(); + NodeList nodeList = NodeList.of(nodes, parents, services.get()); + ConvergenceSummary summary = nodeList.summary(); if (firstTick) { // Run the first time (for each convergence step). logger.log("######## Details for all nodes ########"); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, true)) .collect(toList())); } - ConvergenceSummary summary = nodeList.summary(); + else if ( ! summary.converged()) { + logger.log("Waiting for convergence of " + summary.services() + " services across " + summary.nodes() + " nodes"); + if (summary.needPlatformUpgrade() > 0) + logger.log(summary.upgradingPlatform() + "/" + summary.needPlatformUpgrade() + " nodes upgrading platform"); + if (summary.needReboot() > 0) + logger.log(summary.rebooting() + "/" + summary.needReboot() + " nodes rebooting"); + if (summary.needRestart() > 0) + logger.log(summary.restarting() + "/" + summary.needRestart() + " nodes restarting"); + if (summary.retiring() > 0) + logger.log(summary.retiring() + " nodes retiring"); + if (summary.upgradingFirmware() > 0) + logger.log(summary.upgradingFirmware() + " nodes upgrading firmware"); + if (summary.upgradingOs() > 0) + logger.log(summary.upgradingOs() + " nodes upgrading OS"); + if (summary.needNewConfig() > 0) + logger.log(summary.needNewConfig() + " application services upgrading"); + } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { @@ -349,14 +369,12 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList - .isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) - .and(nodeList - .not().isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) - ); - if ( ! suspendedTooLong.isEmpty()) { + NodeList suspendedTooLong = nodeList.isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) + .and(nodeList.not().isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) + ); + if ( ! suspendedTooLong.isEmpty() && deployment.get().at().plus(timeouts.statelessNodesDown()).isBefore(controller.clock().instant())) { failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } @@ -398,10 +416,10 @@ public class InternalStepRunner implements StepRunner { } if ( ! firstTick) - logger.log(nodeList.expectedDown().and(nodeList.needsNewConfig()).asList().stream() - .distinct() - .flatMap(node -> nodeDetails(node, false)) - .collect(toList())); + logger.log(FINE, nodeList.expectedDown().and(nodeList.needsNewConfig()).asList().stream() + .distinct() + .flatMap(node -> nodeDetails(node, false)) + .collect(toList())); controller.jobController().locked(id, lockedRun -> { Instant noNodesDownSince = nodeList.allowedDown().size() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null; @@ -459,7 +477,7 @@ public class InternalStepRunner implements StepRunner { /** Returns true iff all calls to endpoint in the deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; @@ -485,7 +503,7 @@ public class InternalStepRunner implements StepRunner { private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { DeploymentId deployment = new DeploymentId(id, zone); - Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readZoneEndpointsOf(Set.of(deployment)); + Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deployment)); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; @@ -593,7 +611,7 @@ public class InternalStepRunner implements StepRunner { deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.routing().readZoneEndpointsOf(deployments); + var endpoints = controller.routing().readTestRunnerEndpointsOf(deployments); if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); @@ -617,9 +635,10 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> endTests(RunId id, DualLogger logger) { - if (deployment(id.application(), id.type()).isEmpty()) { + Optional<Deployment> deployment = deployment(id.application(), id.type()); + if (deployment.isEmpty()) { logger.log(INFO, "Deployment expired before tests could complete."); - return Optional.of(aborted); + return Optional.of(error); } Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate(); @@ -629,7 +648,7 @@ public class InternalStepRunner implements StepRunner { } catch (CertificateExpiredException | CertificateNotYetValidException e) { logger.log(WARNING, "Tester certificate expired before tests could complete."); - return Optional.of(aborted); + return Optional.of(error); } } @@ -645,6 +664,11 @@ public class InternalStepRunner implements StepRunner { logger.log("Tests failed."); controller.jobController().updateTestReport(id); return Optional.of(testFailure); + case INCONCLUSIVE: + long sleepMinutes = Math.max(15, Math.min(120, Duration.between(deployment.get().at(), controller.clock().instant()).toMinutes() / 20)); + logger.log("Tests were inconclusive, and will run again in " + sleepMinutes + " minutes."); + controller.jobController().locked(id, run -> run.sleepingUntil(controller.clock().instant().plusSeconds(60 * sleepMinutes))); + return Optional.of(reset); case ERROR: logger.log(INFO, "Tester failed running its tests!"); controller.jobController().updateTestReport(id); @@ -716,8 +740,12 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> report(RunId id, DualLogger logger) { try { controller.jobController().active(id).ifPresent(run -> { + if (run.status() == reset) + return; + if (run.hasFailed()) sendEmailNotification(run, logger); + updateConsoleNotification(run); }); } @@ -1005,7 +1033,11 @@ public class InternalStepRunner implements StepRunner { } private void log(String... messages) { - log(List.of(messages)); + log(INFO, List.of(messages)); + } + + private void log(Level level, String... messages) { + log(level, List.of(messages)); } private void logAll(List<LogEntry> messages) { @@ -1013,7 +1045,11 @@ public class InternalStepRunner implements StepRunner { } private void log(List<String> messages) { - controller.jobController().log(id, step, INFO, messages); + log(INFO, messages); + } + + private void log(Level level, List<String> messages) { + controller.jobController().log(id, step, level, messages); } private void log(Level level, String message) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 1601af2612b..7ea9e990753 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -49,11 +49,13 @@ import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static java.util.Comparator.naturalOrder; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; @@ -370,7 +372,11 @@ public class JobController { for (Step step : report.allPrerequisites(unlockedRun.steps().keySet())) locks.add(curator.lock(id.application(), id.type(), step)); - locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. + locked(id, run -> { + // If run should be reset, just return here. + if (run.status() == reset) return run.reset(); + + // Store the modified run after it has been written to history, in case the latter fails. Run finishedRun = run.finished(controller.clock().instant()); locked(id.application(), id.type(), runs -> { runs.put(run.id(), finishedRun); @@ -378,14 +384,14 @@ public class JobController { long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); - old.getKey().number() <= last - historyLength + old.getKey().number() <= last - historyLength || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge)); old = oldEntries.next()) { // Make sure we keep the last success and the first failing - if (successes == 1 - && old.getValue().status() == RunStatus.success - && !old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { + if ( successes == 1 + && old.getValue().status() == RunStatus.success + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } @@ -439,7 +445,8 @@ public class JobController { applicationPackage.buildTime(), sourceUrl, revision.map(SourceRevision::commit), - false)); + false, + Optional.of(applicationPackage.bundleHash()))); byte[] diff = application.get().latestVersion() .map(v -> v.buildNumber().getAsLong()) .flatMap(prevBuild -> controller.applications().applicationStore().find(id.tenant(), id.application(), prevBuild)) @@ -512,7 +519,7 @@ public class JobController { long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(), - Optional.empty(), Optional.empty(), Optional.empty(), true); + Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty()); byte[] diff = lastRun.map(run -> run.versions().targetApplication()) .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage)) @@ -594,7 +601,7 @@ public class JobController { .flatMap(List::stream) .map(Deployment::applicationVersion) .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) - .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong())) + .min(naturalOrder()) .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); controller.applications().applicationStore().pruneTesters(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index cec2e698e9e..c54d17c2596 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -12,9 +12,12 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; + /** * A list of deployment jobs that can be filtered in various ways. * @@ -51,6 +54,15 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return matching(job -> job.lastCompleted().isPresent() && ! job.isSuccess()); } + /** Returns the subset of jobs which are currently failing, not out of test capacity, and not aborted. */ + public JobList failingHard() { + return failing().not().outOfTestCapacity().not().withStatus(aborted); + } + + public JobList outOfTestCapacity() { + return matching(job -> job.isOutOfCapacity() && job.id().type().environment().isTest()); + } + public JobList running() { return matching(job -> job.isRunning()); } @@ -76,8 +88,13 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } /** Returns the subset of jobs run for the given instance. */ - public JobList instance(InstanceName instance) { - return matching(job -> job.id().application().instance().equals(instance)); + public JobList instance(InstanceName... instances) { + return instance(Set.of(instances)); + } + + /** Returns the subset of jobs run for the given instance. */ + public JobList instance(Collection<InstanceName> instances) { + return matching(job -> instances.contains(job.id().application().instance())); } /** Returns the subset of jobs of which are production jobs. */ @@ -85,12 +102,12 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return matching(job -> job.id().type().isProduction()); } - /** Returns the jobs with any runs matching the given versions — targets only for system test, everything present otherwise. */ + /** Returns the jobs with any runs matching the given versions — targets only for system test, everything present otherwise. */ public JobList triggeredOn(Versions versions) { return matching(job -> ! RunList.from(job).on(versions).isEmpty()); } - /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ + /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ public JobList successOn(Versions versions) { return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 2b9e3dd0733..c12448ab269 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -32,6 +32,7 @@ public class Run { private final boolean isRedeployment; private final Instant start; private final Optional<Instant> end; + private final Optional<Instant> sleepUntil; private final RunStatus status; private final long lastTestRecord; private final Instant lastVespaLogTimestamp; @@ -42,7 +43,7 @@ public class Run { // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, - RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, + Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); @@ -50,6 +51,7 @@ public class Run { this.isRedeployment = isRedeployment; this.start = start; this.end = end; + this.sleepUntil = sleepUntil; this.status = status; this.lastTestRecord = lastTestRecord; this.lastVespaLogTimestamp = lastVespaLogTimestamp; @@ -62,7 +64,7 @@ public class Run { public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running, + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun); } @@ -76,7 +78,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } @@ -91,52 +93,66 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(startTime)); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run finished(Instant now) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status, + return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), sleepUntil, status == running ? success : status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun); } public Run aborted() { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, aborted, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } + public Run reset() { + requireActive(); + Map<Step, StepInfo> reset = new EnumMap<>(steps); + reset.replaceAll((step, __) -> StepInfo.initial(step)); + return new Run(id, reset, versions, isRedeployment, start, end, sleepUntil, running, -1, lastVespaLogTimestamp, + Optional.empty(), Optional.empty(), testerCertificate, dryRun); + } + public Run with(long lastTestRecord) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun); } + public Run sleepingUntil(Instant instant) { + requireActive(); + return new Run(id, steps, versions, isRedeployment, start, end, Optional.of(instant), status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + } + /** Returns the id of this run. */ public RunId id() { return id; @@ -185,6 +201,11 @@ public class Run { return end; } + /** Returns the instant until which this should sleep. */ + public Optional<Instant> sleepUntil() { + return sleepUntil; + } + /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { return status != running && status != success; @@ -210,7 +231,7 @@ public class Run { return lastVespaLogTimestamp; } - /** Returns the timestamp of the last time no nodes were allowed to be down. */ + /** Returns since when no nodes have been allowed to be down. */ public Optional<Instant> noNodesDownSince() { return noNodesDownSince; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 925bd500199..00cd4bd5c6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -26,7 +26,7 @@ public class RunList extends AbstractFilteringList<Run, RunList> { return from(job.runs().descendingMap().values()); } - /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ + /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ public RunList on(Versions versions) { return matching(run -> matchingVersions(run, versions)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 7aa685d0698..375bdd239d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -33,6 +33,9 @@ public enum RunStatus { success, /** Run was abandoned, due to user intervention or job timeout. */ - aborted + aborted, + + /** Run should be reset to its starting state. Used for production tests. */ + reset } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index d58f3dee95f..2e669808c44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -113,6 +113,7 @@ public enum Step { public static Step.Status of(RunStatus status) { switch (status) { case success : throw new AssertionError("Unexpected run status '" + status + "'!"); + case reset : case aborted : return unfinished; case running : return succeeded; default : return failed; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java index 71832352d4d..24723f84897 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java @@ -6,12 +6,12 @@ import java.util.Objects; import java.util.Optional; /** - * Information about a step. + * Information about a step. Immutable. * * @author hakonhall */ -// @Immutable public class StepInfo { + private final Step step; private final Step.Status status; private final Optional<Instant> startTime; @@ -57,4 +57,5 @@ public class StepInfo { public int hashCode() { return Objects.hash(step, status, startTime); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index 779ce6fa7fe..1e183d44377 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -126,8 +126,9 @@ public class Versions { private static ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { - return max(change.application(), deployment.map(Deployment::applicationVersion)) - .orElseGet(() -> defaultApplicationVersion(application)); + return change.application() + .or(() -> deployment.map(Deployment::applicationVersion)) + .orElseGet(() -> defaultApplicationVersion(application)); } private static ApplicationVersion defaultApplicationVersion(Application application) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java index a1eae23afd3..a05a8cd753f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java @@ -28,7 +28,7 @@ public class ZipBuilder implements AutoCloseable { public void add(byte[] zippedContent) { try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) { for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) { - zipOutputStream.putNextEntry(entry); + zipOutputStream.putNextEntry(new ZipEntry(entry.getName())); zin.transferTo(zipOutputStream); zipOutputStream.closeEntry(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 8156a2e9f3b..e3f69f59d24 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; @@ -15,8 +16,11 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; + /** * Update archive access permissions with roles from tenants * @@ -52,10 +56,20 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { try { var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, + archiveService.updateBucketPolicy(zoneId, archiveBucket, Maps.filterEntries(tenantArchiveAccessRoles, entry -> archiveBucket.tenants().contains(entry.getKey()))) ); + Map<String, List<ArchiveBucket>> bucketsPerKey = archiveBucketDb.buckets(zoneId).stream() + .collect(groupingBy(ArchiveBucket::keyArn)); + bucketsPerKey.forEach((keyArn, buckets) -> { + Set<String> authorizedIamRolesForKey = buckets.stream() + .flatMap(b -> b.tenants().stream()) + .filter(tenantArchiveAccessRoles::containsKey) + .map(tenantArchiveAccessRoles::get) + .collect(Collectors.toSet()); + archiveService.updateKeyPolicy(zoneId, keyArn, authorizedIamRolesForKey); + }); } catch (Exception e) { throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 045960fdce2..36ab2e6f384 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -29,8 +29,8 @@ public class ArchiveUriUpdater extends ControllerMaintainer { private final NodeRepository nodeRepository; private final CuratorArchiveBucketDb archiveBucketDb; - public ArchiveUriUpdater(Controller controller, Duration duration) { - super(controller, duration); + public ArchiveUriUpdater(Controller controller, Duration interval) { + super(controller, interval); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.archiveBucketDb = controller.archiveBucketDb(); @@ -39,6 +39,10 @@ public class ArchiveUriUpdater extends ControllerMaintainer { @Override protected double maintain() { Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); + + tenantsByZone.put(controller().zoneRegistry().systemZone().getVirtualId(), + new HashSet<>(INFRASTRUCTURE_TENANTS)); + for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.deployments().values()) { @@ -52,7 +56,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { tenantsByZone.forEach((zone, tenants) -> { Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); for (TenantName tenant : tenants) { - archiveBucketDb.archiveUriFor(zone, tenant) + archiveBucketDb.archiveUriFor(zone, tenant, true) .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java new file mode 100644 index 00000000000..a7ebaec7c09 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; + +import java.time.Duration; +import java.util.EnumSet; + +/** + * @author olaa + */ +public class BillingDatabaseMaintainer extends ControllerMaintainer { + + public BillingDatabaseMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, EnumSet.of(SystemName.PublicCd)); + } + + @Override + protected double maintain() { + controller().serviceRegistry().billingDatabase().maintain(); + return 1; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 913d6dfeab8..bde39aa3dd6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -75,6 +75,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); + maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer)); } public Upgrader upgrader() { return upgrader; } @@ -132,10 +133,11 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration vcmrMaintainer; private final Duration retriggerMaintainer; private final Duration userManagementMaintainer; + private final Duration billingDatabaseMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); - this.defaultInterval = duration(system.isCd() || system == SystemName.dev ? 1 : 5, MINUTES); + this.defaultInterval = duration(system.isCd() ? 1 : 5, MINUTES); this.outstandingChangeDeployer = duration(3, MINUTES); this.versionStatusUpdater = duration(3, MINUTES); this.readyJobsTrigger = duration(1, MINUTES); @@ -166,6 +168,7 @@ public class ControllerMaintenance extends AbstractComponent { this.vcmrMaintainer = duration(1, HOURS); this.retriggerMaintainer = duration(1, MINUTES); this.userManagementMaintainer = duration(12, HOURS); + this.billingDatabaseMaintainer = duration(5, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3958b3cb282..3a047d33be5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -57,6 +58,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { private List<Application> applications() { return ApplicationList.from(controller().applications().readable()) .withProjectId() + .matching(appliaction -> appliaction.deploymentSpec().steps().stream().anyMatch(step -> step.concerns(Environment.prod))) .asList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index ea76b0684a3..cc00572e760 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -6,7 +6,6 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.Flags; @@ -71,7 +70,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteUnusedCertificates(); deleteOrReportUnmanagedCertificates(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); + log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); return 0.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 85d750d267c..3184449e48b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -78,7 +78,7 @@ public class JobRunner extends ControllerMaintainer { /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ public void advance(Run run) { if ( ! run.hasFailed() - && controller().clock().instant().isAfter(run.start().plus(jobTimeout))) + && controller().clock().instant().isAfter(run.sleepUntil().orElse(run.start()).plus(jobTimeout))) executors.execute(() -> { jobs.abort(run.id()); advance(jobs.run(run.id()).get()); @@ -86,7 +86,7 @@ public class JobRunner extends ControllerMaintainer { else if (run.readySteps().isEmpty()) executors.execute(() -> finish(run.id())); - else + else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) run.readySteps().forEach(step -> executors.execute(() -> advance(run.id(), step))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 021b27456a4..6aa43d4db47 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -292,7 +292,7 @@ public class MetricsReporter extends ControllerMaintainer { } private static int deploymentsFailingUpgrade(JobList jobs) { - return jobs.failing().not().failingApplicationChange().size(); + return jobs.failingHard().not().failingApplicationChange().size(); } private static Duration averageDeploymentDuration(JobList jobs, Instant now) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index aa087f58059..4abad09e1dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -62,8 +62,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { int hourOfDay = instant.atZone(ZoneOffset.UTC).getHour(); int dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek().getValue(); // Upgrade can only be scheduled between 07:00 and 12:59 UTC, Monday-Thursday - return hourOfDay >= 7 && hourOfDay <= 12 && - dayOfWeek < 5; + return hourOfDay >= 7 && hourOfDay <= 12 && dayOfWeek < 5; } private Release releaseIn(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 9f72b68372c..acaf35133d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.InstanceName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 9af838b1f55..8444ed8a374 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -15,18 +15,19 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import java.time.Duration; -import java.util.Collection; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.function.BinaryOperator; -import java.util.function.Function; +import java.util.function.UnaryOperator; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; -import static java.util.Comparator.naturalOrder; +import static java.util.Comparator.reverseOrder; /** * Maintenance job which schedules applications for Vespa version upgrade @@ -54,91 +55,90 @@ public class Upgrader extends ControllerMaintainer { public double maintain() { // Determine target versions for each upgrade policy VersionStatus versionStatus = controller().readVersionStatus(); - Version canaryTarget = controller().systemVersion(versionStatus); - Collection<Version> defaultTargets = targetVersions(Confidence.normal, versionStatus); - Collection<Version> conservativeTargets = targetVersions(Confidence.high, versionStatus); + cancelBrokenUpgrades(versionStatus); - cancelUpgrades(versionStatus, canaryTarget, defaultTargets, conservativeTargets); - upgrade(versionStatus, canaryTarget, defaultTargets, conservativeTargets); - return 1.0; - } + Optional<Integer> targetMajorVersion = targetMajorVersion(); + InstanceList instances = instances(controller().systemVersion(versionStatus)); + for (UpgradePolicy policy : UpgradePolicy.values()) + updateTargets(versionStatus, instances, policy, targetMajorVersion); - /** Returns the target versions for given confidence, one per major version in the system */ - private Collection<Version> targetVersions(Confidence confidence, VersionStatus versionStatus) { - return versionStatus.versions().stream() - // Ensure we never pick a version newer than the system - .filter(v -> !v.versionNumber().isAfter(controller().systemVersion(versionStatus))) - .filter(v -> v.confidence().equalOrHigherThan(confidence)) - .map(VespaVersion::versionNumber) - .collect(Collectors.toMap(Version::getMajor, // Key on major version - Function.identity(), // Use version as value - BinaryOperator.<Version>maxBy(naturalOrder()))) // Pick highest version when merging versions within this major - .values(); + return 1.0; } /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ private InstanceList instances(Version systemVersion) { return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()), systemVersion)) .withDeclaredJobs() + .shuffle(random) + .byIncreasingDeployedVersion() .unpinned(); } - private void cancelUpgrades(VersionStatus versionStatus, Version canaryTarget, Collection<Version> defaultTargets, Collection<Version> conservativeTargets) { - InstanceList instances = instances(controller().systemVersion(versionStatus)); + private void cancelBrokenUpgrades(VersionStatus versionStatus) { // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation) + InstanceList instances = instances(controller().systemVersion(versionStatus)); for (VespaVersion version : versionStatus.versions()) { if (version.confidence() == Confidence.broken) - cancelUpgradesOf(instances.upgradingTo(version.versionNumber()) - .not().with(UpgradePolicy.canary), + cancelUpgradesOf(instances.upgradingTo(version.versionNumber()).not().with(UpgradePolicy.canary), version.versionNumber() + " is broken"); } + } + + private void updateTargets(VersionStatus versionStatus, InstanceList instances, UpgradePolicy policy, Optional<Integer> targetMajorVersion) { + InstanceList remaining = instances.with(policy); + List<Version> targetAndNewer = new ArrayList<>(); + UnaryOperator<InstanceList> cancellationCriterion = policy == UpgradePolicy.canary ? i -> i.not().upgradingTo(targetAndNewer) + : i -> i.failing() + .not().upgradingTo(targetAndNewer); + + Map<ApplicationId, Version> targets = new LinkedHashMap<>(); + for (Version version : targetsForPolicy(versionStatus, policy)) { + targetAndNewer.add(version); + InstanceList eligible = eligibleForVersion(remaining, version, cancellationCriterion, targetMajorVersion); + InstanceList outdated = cancellationCriterion.apply(eligible); + cancelUpgradesOf(outdated.upgrading(), "Upgrading to outdated versions"); + + // Prefer the newest target for each instance. + remaining = remaining.not().matching(eligible.asList()::contains); + for (ApplicationId id : outdated.and(eligible.not().upgrading()).not().changingRevision()) + targets.put(id, version); + } - // Canaries should always try the canary target - cancelUpgradesOf(instances.upgrading() - .not().upgradingTo(canaryTarget) - .with(UpgradePolicy.canary), - "Outdated target version for Canaries"); - - // Cancel *failed* upgrades to earlier versions, as the new version may fix it - String reason = "Failing on outdated version"; - cancelUpgradesOf(instances.upgrading() - .failing() - .not().upgradingTo(defaultTargets) - .with(UpgradePolicy.defaultPolicy), - reason); - cancelUpgradesOf(instances.upgrading() - .failing() - .not().upgradingTo(conservativeTargets) - .with(UpgradePolicy.conservative), - reason); - } - - private void upgrade(VersionStatus versionStatus, Version canaryTarget, Collection<Version> defaultTargets, Collection<Version> conservativeTargets) { - InstanceList instances = instances(controller().systemVersion(versionStatus)); - Optional<Integer> targetMajorVersion = targetMajorVersion(); - upgrade(instances.with(UpgradePolicy.canary), canaryTarget, targetMajorVersion, instances.size()); - defaultTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.defaultPolicy), target, targetMajorVersion, numberOfApplicationsToUpgrade())); - conservativeTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.conservative), target, targetMajorVersion, numberOfApplicationsToUpgrade())); + int numberToUpgrade = policy == UpgradePolicy.canary ? instances.size() : numberOfApplicationsToUpgrade(); + for (ApplicationId id : instances.matching(targets.keySet()::contains).first(numberToUpgrade)) { + log.log(Level.INFO, "Triggering upgrade to " + targets.get(id) + " for " + id); + controller().applications().deploymentTrigger().triggerChange(id, Change.of(targets.get(id))); + } + } + + /** Returns target versions for given confidence, by descending version number. */ + private List<Version> targetsForPolicy(VersionStatus versions, UpgradePolicy policy) { + Version systemVersion = controller().systemVersion(versions); + if (policy == UpgradePolicy.canary) + return List.of(systemVersion); + + Confidence target = policy == UpgradePolicy.defaultPolicy ? Confidence.normal : Confidence.high; + return versions.versions().stream() + .filter(version -> ! version.versionNumber().isAfter(systemVersion) + && version.confidence().equalOrHigherThan(target)) + .map(VespaVersion::versionNumber) + .sorted(reverseOrder()) + .collect(Collectors.toList()); } - private void upgrade(InstanceList instances, Version version, Optional<Integer> targetMajorVersion, int numberToUpgrade) { + private InstanceList eligibleForVersion(InstanceList instances, Version version, UnaryOperator<InstanceList> cancellationCriterion, Optional<Integer> targetMajorVersion) { Change change = Change.of(version); - instances.not().failingOn(version) - .allowMajorVersion(version.getMajor(), targetMajorVersion.orElse(version.getMajor())) - .not().deploying() - .not().hasCompleted(change) // Avoid rescheduling change for instances without production steps - .onLowerVersionThan(version) - .canUpgradeAt(version, controller().clock().instant()) - .shuffle(random) // Shuffle so we do not always upgrade instances in the same order - .byIncreasingDeployedVersion() - .first(numberToUpgrade) - .forEach(instance -> controller().applications().deploymentTrigger().triggerChange(instance, change)); + return instances.not().failingOn(version) + .allowMajorVersion(version.getMajor(), targetMajorVersion.orElse(version.getMajor())) + .not().hasCompleted(change) // Avoid rescheduling change for instances without production steps. + .onLowerVersionThan(version) + .canUpgradeAt(version, controller().clock().instant()); } private void cancelUpgradesOf(InstanceList instances, String reason) { instances = instances.unpinned(); if (instances.isEmpty()) return; - log.info("Cancelling upgrading of " + instances.asList().size() + " instances: " + reason); + log.info("Cancelling upgrading of " + instances.asList() + " instances: " + reason); for (ApplicationId instance : instances.asList()) controller().applications().deploymentTrigger().cancelChange(instance, PLATFORM); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 4b060846090..dee3c822465 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -49,6 +49,8 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; /** * Serializes {@link Application}s to/from slime. @@ -75,6 +77,7 @@ public class ApplicationSerializer { private static final String deployingField = "deployingField"; private static final String projectIdField = "projectId"; private static final String latestVersionField = "latestVersion"; + private static final String versionsField = "versions"; private static final String pinnedField = "pinned"; private static final String deploymentIssueField = "deploymentIssueId"; private static final String ownershipIssueIdField = "ownershipIssueId"; @@ -94,6 +97,7 @@ public class ApplicationSerializer { private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format private static final String assignedRotationsField = "assignedRotations"; private static final String assignedRotationEndpointField = "endpointId"; + private static final String latestDeployedField = "latestDeployed"; // Deployment fields private static final String zoneField = "zone"; @@ -111,6 +115,7 @@ public class ApplicationSerializer { private static final String compileVersionField = "compileVersion"; private static final String buildTimeField = "buildTime"; private static final String sourceUrlField = "sourceUrl"; + private static final String bundleHashField = "bundleHash"; private static final String lastQueriedField = "lastQueried"; private static final String lastWrittenField = "lastWritten"; private static final String lastQueriesPerSecondField = "lastQueriesPerSecond"; @@ -163,6 +168,7 @@ public class ApplicationSerializer { root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField)); application.latestVersion().ifPresent(version -> toSlime(version, root.setObject(latestVersionField))); + versionsToSlime(application, root.setArray(versionsField)); instancesToSlime(application, root.setArray(instancesField)); return slime; } @@ -176,6 +182,7 @@ public class ApplicationSerializer { assignedRotationsToSlime(instance.rotations(), instanceObject); toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); toSlime(instance.change(), instanceObject, deployingField); + instance.latestDeployed().ifPresent(version -> toSlime(version, instanceObject.setObject(latestDeployedField))); } } @@ -221,6 +228,10 @@ public class ApplicationSerializer { object.setString(regionField, zone.region().value()); } + private void versionsToSlime(Application application, Cursor object) { + application.versions().forEach(version -> toSlime(version, object.addObject())); + } + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number)); applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField))); @@ -230,6 +241,7 @@ public class ApplicationSerializer { applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + applicationVersion.bundleHash().ifPresent(bundleHash -> object.setString(bundleHashField, bundleHash)); } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -310,10 +322,11 @@ public class ApplicationSerializer { List<Instance> instances = instancesFromSlime(id, root.field(instancesField)); OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField)); Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField)); + SortedSet<ApplicationVersion> versions = versionsFromSlime(root.field(versionsField)); return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, - deployKeys, projectId, latestVersion, instances); + deployKeys, projectId, latestVersion, versions, instances); } private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) { @@ -321,6 +334,12 @@ public class ApplicationSerializer { .filter(version -> ! version.isUnknown()); } + private SortedSet<ApplicationVersion> versionsFromSlime(Inspector versionsObject) { + SortedSet<ApplicationVersion> versions = new TreeSet<>(); + versionsObject.traverse((ArrayTraverser) (name, object) -> versions.add(applicationVersionFromSlime(object))); + return versions; + } + private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) { List<Instance> instances = new ArrayList<>(); field.traverse((ArrayTraverser) (name, object) -> { @@ -330,12 +349,14 @@ public class ApplicationSerializer { List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); RotationStatus rotationStatus = rotationStatusFromSlime(object); Change change = changeFromSlime(object.field(deployingField)); + Optional<ApplicationVersion> latestDeployed = latestVersionFromSlime(object.field(latestDeployedField)); instances.add(new Instance(id.instance(instanceName), deployments, jobPauses, assignedRotations, rotationStatus, - change)); + change, + latestDeployed)); }); return instances; } @@ -424,8 +445,9 @@ public class ApplicationSerializer { Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); boolean deployedDirectly = object.field(deployedDirectlyField).asBool(); + Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly); + return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 5e005c9467c..212e27ba0c9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -133,4 +133,5 @@ public class BufferedLogStore { public void writeTestReport(RunId id, TestReport report) { store.putTestReport(id, report.toJson().getBytes()); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index caad32d9a17..9b1a03039fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.Version; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index c93b77e9af1..800d5e2750b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -37,6 +37,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCer import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; @@ -82,6 +83,7 @@ class RunSerializer { private static final String numberField = "number"; private static final String startField = "start"; private static final String endField = "end"; + private static final String sleepingUntilField = "sleepingUntil"; private static final String statusField = "status"; private static final String versionsField = "versions"; private static final String isRedeploymentField = "isRedeployment"; @@ -96,6 +98,7 @@ class RunSerializer { private static final String sourceUrlField = "sourceUrl"; private static final String buildField = "build"; private static final String sourceField = "source"; + private static final String bundleHashField = "bundleHash"; private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; @@ -138,6 +141,7 @@ class RunSerializer { runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), + SlimeUtils.optionalInstant(runObject.field(sleepingUntilField)), runStatusOf(runObject.field(statusField).asString()), runObject.field(lastTestRecordField).asLong(), Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), @@ -179,9 +183,10 @@ class RunSerializer { Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField)); boolean deployedDirectly = versionObject.field(deployedDirectlyField).asBool(); + Optional<String> bundleHash = SlimeUtils.optionalString(versionObject.field(bundleHashField)); return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, - compileVersion, buildTime, sourceUrl, commit, deployedDirectly); + compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); } // Don't change this — introduce a separate array instead. @@ -226,6 +231,7 @@ class RunSerializer { runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); + run.sleepUntil().ifPresent(end -> runObject.setLong(sleepingUntilField, end.toEpochMilli())); runObject.setString(statusField, valueOf(run.status())); runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); @@ -362,6 +368,7 @@ class RunSerializer { case error : return "error"; case success : return "success"; case aborted : return "aborted"; + case reset : return "reset"; default: throw new AssertionError("No value defined for '" + status + "'!"); } @@ -378,6 +385,7 @@ class RunSerializer { case "error" : return error; case "success" : return success; case "aborted" : return aborted; + case "reset" : return reset; default: throw new IllegalArgumentException("No run status defined by '" + status + "'!"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 6116b2e27b6..dffc7bf43b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -105,7 +105,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } } - throw new RuntimeException("Failed talking to config servers: " + errorBuilder.toString()); + throw new RuntimeException("Failed talking to config servers: " + errorBuilder); } private Optional<ProxyResponse> proxy(ProxyRequest request, URI url, StringBuilder errorBuilder) { @@ -118,9 +118,9 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C String content = getContent(response); int status = response.getStatusLine().getStatusCode(); if (status / 100 == 5) { - errorBuilder.append("Talking to server ").append(url.getHost()); - errorBuilder.append(", got ").append(status).append(" ") - .append(content).append("\n"); + errorBuilder.append("Talking to server ").append(url.getHost()) + .append(", got ").append(status).append(" ") + .append(content).append("\n"); LOG.log(Level.FINE, () -> Text.format("Got response from %s with status code %d and content:\n %s", url.getHost(), status, content)); return Optional.empty(); @@ -135,8 +135,9 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C // Send response back return Optional.of(new ProxyResponse(request, content, status, url, contentType)); } catch (Exception e) { - errorBuilder.append("Talking to server ").append(url.getHost()); - errorBuilder.append(" got exception ").append(e.getMessage()); + errorBuilder.append("Talking to server ").append(url.getHost()) + .append(" got exception ").append(e.getMessage()) + .append("\n"); LOG.log(Level.FINE, e, () -> "Got exception while sending request to " + url.getHost()); return Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 73a80a0a94a..3b4154759ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -23,7 +23,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; @@ -70,6 +70,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -96,11 +97,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; @@ -139,7 +140,6 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Scanner; @@ -173,7 +173,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private final TestConfigSerializer testConfigSerializer; @Inject - public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx, + public ApplicationApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller, AccessControlRequests accessControlRequests) { super(parentCtx, controller.auditLogger()); @@ -235,6 +235,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/")) return root(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/request/ssh")) return accessRequests(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/info")) return tenantInfo(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/notifications")) return notifications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}/validate")) return validateSecretStore(path.get("tenant"), path.get("name"), request); @@ -286,6 +287,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/request/ssh")) return requestSshAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/access/approve/ssh")) return approveAccessRequest(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/info")) return updateTenantInfo(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowArchiveAccess(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return addSecretStore(path.get("tenant"), path.get("name"), request); @@ -402,6 +405,44 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse accessRequests(String tenantName, HttpRequest request) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); + + var pendingRequests = controller.serviceRegistry().accessControlService().hasPendingAccessRequests(TenantName.from(tenantName)); + var slime = new Slime(); + slime.setObject().setBool("hasPendingRequests", pendingRequests); + return new SlimeJsonResponse(slime); + } + + private HttpResponse requestSshAccess(String tenantName, HttpRequest request) { + if (!isOperator(request)) { + return ErrorResponse.forbidden("Only operators are allowed to request ssh access"); + } + + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + return ErrorResponse.badRequest("Can only request access for cloud tenants"); + + controller.serviceRegistry().accessControlService().requestSshAccess(TenantName.from(tenantName)); + return new MessageResponse("OK"); + + } + + private HttpResponse approveAccessRequest(String tenantName, HttpRequest request) { + var tenant = TenantName.from(tenantName); + + if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) + return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); + + var inspector = toSlime(request.getData()).get(); + var expiry = inspector.field("expiry").valid() ? + Instant.ofEpochMilli(inspector.field("expiry").asLong()) : + Instant.now().plus(1, ChronoUnit.DAYS); + + controller.serviceRegistry().accessControlService().approveSshAccess(tenant, expiry); + return new MessageResponse("OK"); + } + private HttpResponse tenantInfo(String tenantName, HttpRequest request) { return controller.tenants().get(TenantName.from(tenantName)) .filter(tenant -> tenant.type() == Tenant.Type.cloud) @@ -1379,18 +1420,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId) .scope(Endpoint.Scope.zone); if (!legacyEndpoints) { - zoneEndpoints = zoneEndpoints.not().legacy(); + zoneEndpoints = zoneEndpoints.not().legacy().direct(); } - for (var endpoint : controller.routing().directEndpoints(zoneEndpoints, deploymentId.applicationId())) { + for (var endpoint : zoneEndpoints) { toSlime(endpoint, endpointArray.addObject()); } // Add declared endpoints EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application) .targets(deploymentId); if (!legacyEndpoints) { - declaredEndpoints = declaredEndpoints.not().legacy(); + declaredEndpoints = declaredEndpoints.not().legacy().direct(); } - for (var endpoint : controller.routing().directEndpoints(declaredEndpoints, deploymentId.applicationId())) { + for (var endpoint : declaredEndpoints) { toSlime(endpoint, endpointArray.addObject()); } @@ -1439,7 +1480,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setDouble("quota", deployment.quota().rate()); deployment.cost().ifPresent(cost -> response.setDouble("cost", cost)); - controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant()) + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant(), false) .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); Cursor activity = response.setObject("activity"); @@ -1719,7 +1760,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { TenantName tenant = TenantName.from(tenantName); Inspector requestObject = toSlime(request.getData()).get(); controller.tenants().create(accessControlRequests.specification(tenant, requestObject), - accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); + accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); + if (controller.system().isPublic()) { + User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); + TenantInfo info = controller.tenants().require(tenant, CloudTenant.class) + .info() + .withContactName(user.name()) + .withContactEmail(user.email()); + // Store changes + controller.tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { + lockedTenant = lockedTenant.withInfo(info); + controller.tenants().store(lockedTenant); + }); + } return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } @@ -1792,16 +1845,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private ApplicationVersion getApplicationVersion(Application application, Long build) { - // Check whether this is the latest version, and possibly return that. - // Otherwise, look through historic runs for a proper ApplicationVersion. - return application.latestVersion() + return application.versions().stream() .filter(version -> version.buildNumber().stream().anyMatch(build::equals)) - .or(() -> controller.jobController().deploymentStatus(application).jobs() - .asList().stream() - .flatMap(job -> job.runs().values().stream()) - .map(run -> run.versions().targetApplication()) - .filter(version -> version.buildNumber().stream().anyMatch(build::equals)) - .findAny()) + .findFirst() .filter(version -> controller.applications().applicationStore().hasBuild(application.id().tenant(), application.id().application(), build)) @@ -2077,7 +2123,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.routing().readZoneEndpointsOf(deployments), + controller.routing().readTestRunnerEndpointsOf(deployments), controller.applications().reachableContentClustersByZone(deployments))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 30bd040a682..0da8934f2a6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -20,9 +20,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -52,6 +52,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialRe import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.normal; +import static java.util.Comparator.reverseOrder; /** * Implements the REST API for the job controller delegated from the Application API. @@ -230,6 +231,7 @@ class JobControllerApiHandlerHelper { private static String nameOf(RunStatus status) { switch (status) { + case reset: // This means the run will reset and keep running. case running: return "running"; case aborted: return "aborted"; case error: return "error"; @@ -256,7 +258,7 @@ class JobControllerApiHandlerHelper { responseObject.setString("application", id.application().value()); application.projectId().ifPresent(projectId -> responseObject.setLong("projectId", projectId)); - Map<JobId, List<Versions>> jobsToRun = status.jobsToRun(); + Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); Cursor stepsArray = responseObject.setArray("steps"); VersionStatus versionStatus = controller.readVersionStatus(); for (DeploymentStatus.StepStatus stepStatus : status.allSteps()) { @@ -292,22 +294,23 @@ class JobControllerApiHandlerHelper { Cursor latestVersionsObject = stepObject.setObject("latestVersions"); List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker(); + var deployments = application.require(stepStatus.instance()).productionDeployments().values(); latestVersionWithCompatibleConfidenceAndNotNewerThanSystem(versionStatus.versions(), application.deploymentSpec().requireInstance(stepStatus.instance()).upgradePolicy()) .ifPresent(latestPlatform -> { Cursor latestPlatformObject = latestVersionsObject.setObject("platform"); latestPlatformObject.setString("platform", latestPlatform.versionNumber().toFullString()); latestPlatformObject.setLong("at", latestPlatform.committedAt().toEpochMilli()); - latestPlatformObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() - .anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); + latestPlatformObject.setBool("upgrade", change.platform().map(latestPlatform.versionNumber()::isAfter).orElse(true) && deployments.isEmpty() + || deployments.stream().anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); }); application.latestVersion().ifPresent(latestApplication -> { Cursor latestApplicationObject = latestVersionsObject.setObject("application"); toSlime(latestApplicationObject.setObject("application"), latestApplication); latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); - latestApplicationObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() - .anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); + latestApplicationObject.setBool("upgrade", change.application().map(latestApplication::compareTo).orElse(1) > 0 && deployments.isEmpty() + || deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); }); } @@ -330,23 +333,26 @@ class JobControllerApiHandlerHelper { JobStatus jobStatus = status.jobs().get(job).get(); Cursor toRunArray = stepObject.setArray("toRun"); - for (Versions versions : jobsToRun.getOrDefault(job, List.of())) { + for (DeploymentStatus.Job versions : jobsToRun.getOrDefault(job, List.of())) { boolean running = jobStatus.lastTriggered() .map(run -> jobStatus.isRunning() - && versions.targetsMatch(run.versions()) - && (job.type().isProduction() || versions.sourcesMatchIfPresent(run.versions()))) + && versions.versions().targetsMatch(run.versions()) + && (job.type().isProduction() || versions.versions().sourcesMatchIfPresent(run.versions()))) .orElse(false); if (running) continue; // Run will be contained in the "runs" array. Cursor runObject = toRunArray.addObject(); - toSlime(runObject.setObject("versions"), versions); + toSlime(runObject.setObject("versions"), versions.versions()); } toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); }); } + Cursor buildsArray = responseObject.setArray("builds"); + application.versions().stream().sorted(reverseOrder()).forEach(version -> applicationVersionToSlime(buildsArray.addObject(), version)); + return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 3afc2c8fc15..45c4978ab9f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.MessageResponse; @@ -49,7 +49,7 @@ import java.util.stream.Collectors; * @author andreer * @author olaa */ -public class BillingApiHandler extends LoggingRequestHandler { +public class BillingApiHandler extends ThreadedHttpRequestHandler { private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index f18d38e971b..07e3fbe1bb2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -48,7 +48,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private final BillingController billing; private final Clock clock; - public BillingApiHandlerV2(LoggingRequestHandler.Context context, Controller controller) { + public BillingApiHandlerV2(ThreadedHttpRequestHandler.Context context, Controller controller) { super(context, BillingApiHandlerV2::createRestApi); this.applications = controller.applications(); this.tenants = controller.tenants(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index 0ea8d09ba09..e1fc68974cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; @@ -37,7 +37,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private final ChangeManagementAssessor assessor; private final Controller controller; - public ChangeManagementApiHandler(LoggingRequestHandler.Context ctx, Controller controller) { + public ChangeManagementApiHandler(ThreadedHttpRequestHandler.Context ctx, Controller controller) { super(ctx, controller.auditLogger()); this.assessor = new ChangeManagementAssessor(controller.serviceRegistry().configServer().nodeRepository()); this.controller = controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 7c1f049548a..951a22c5ee7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; @@ -51,7 +51,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { private final ControllerMaintenance maintenance; private final Controller controller; - public ControllerApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller, ControllerMaintenance maintenance) { + public ControllerApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller, ControllerMaintenance maintenance) { super(parentCtx, controller.auditLogger()); this.controller = controller; this.maintenance = maintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index 3047d1ed234..eb74f931b2c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; @@ -36,7 +36,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; * @author jonmv */ @SuppressWarnings("unused") // Handler -public class BadgeApiHandler extends LoggingRequestHandler { +public class BadgeApiHandler extends ThreadedHttpRequestHandler { private final static Logger log = Logger.getLogger(BadgeApiHandler.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index b5008a44c6d..158cc6caede 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; @@ -47,11 +47,11 @@ import static java.util.stream.Collectors.toUnmodifiableMap; * @author bratseth */ @SuppressWarnings("unused") // Injected -public class DeploymentApiHandler extends LoggingRequestHandler { +public class DeploymentApiHandler extends ThreadedHttpRequestHandler { private final Controller controller; - public DeploymentApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller) { + public DeploymentApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller) { super(parentCtx); this.controller = controller; } @@ -185,6 +185,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli())); if (jobsToRun.containsKey(job)) { List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream() + .map(DeploymentStatus.Job::versions) .filter(versions -> versions.targetPlatform().equals(statistics.version())) .collect(Collectors.toList()); if ( ! versionsOnThisPlatform.isEmpty()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index c685390c7ed..411e9ec6070 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.auth0.jwt.JWT; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; @@ -16,6 +19,7 @@ import com.yahoo.restapi.Path; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; @@ -94,6 +98,15 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { path.matches("/application/v4/tenant/{tenant}/application/{application}/{*}"); Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from); + final Optional<Zone> zone; + if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { + zone = Optional.of(new Zone(Environment.from(path.get("environment")), RegionName.from(path.get("region")))); + } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { + zone = Optional.of(new Zone(Environment.from(path.get("environment")), RegionName.from(path.get("region")))); + } else { + zone = Optional.empty(); + } + AthenzIdentity identity = principal.getIdentity(); Set<Role> roleMemberships = new CopyOnWriteArraySet<>(); @@ -121,10 +134,21 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { && ! tenant.get().name().value().equals("sandbox")) futures.add(executor.submit(() -> { if ( tenant.get().type() == Tenant.Type.athenz - && hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get())) + && hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get(), zone)) roleMemberships.add(Role.buildService(tenant.get().name(), application.get())); })); + if (identity instanceof AthenzUser + && zone.isPresent() + && tenant.isPresent() + && application.isPresent()) { + Zone z = zone.get(); + futures.add(executor.submit(() -> { + if (canDeployToManualZones(identity, ((AthenzTenant) tenant.get()).domain(), application.get(), z)) + roleMemberships.add(Role.hostedDeveloper(tenant.get().name())); + })); + } + futures.add(executor.submit(() -> { if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/false)) roleMemberships.add(Role.systemFlagsDeployer()); @@ -167,12 +191,22 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } } - private boolean hasDeployerAccess(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application) { + private boolean hasDeployerAccess(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application, Optional<Zone> zone) { try { return athenz.hasApplicationAccess(identity, ApplicationAction.deploy, tenantDomain, - application); + application, + zone); + } catch (ZmsClientException e) { + throw new RuntimeException("Failed to authorize operation: (" + e.getMessage() + ")", e); + } + } + + private boolean canDeployToManualZones(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application, Zone zone) { + if (! zone.environment().isManuallyDeployed()) return false; + try { + return athenz.hasApplicationAccess(identity, ApplicationAction.deploy, tenantDomain, application, Optional.of(zone)); } catch (ZmsClientException e) { throw new RuntimeException("Failed to authorize operation: (" + e.getMessage() + ")", e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index 44871cc58ce..708891c8251 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.vespa.flags.BooleanFlag; @@ -36,7 +36,7 @@ import java.util.stream.Collectors; * * @author valerijf */ -public class HorizonApiHandler extends LoggingRequestHandler { +public class HorizonApiHandler extends ThreadedHttpRequestHandler { private final SystemName systemName; private final HorizonClient client; @@ -46,7 +46,7 @@ public class HorizonApiHandler extends LoggingRequestHandler { EnumSet.of(RoleDefinition.hostedOperator, RoleDefinition.hostedSupporter); @Inject - public HorizonApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller, FlagSource flagSource) { + public HorizonApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller, FlagSource flagSource) { super(parentCtx); this.systemName = controller.system(); this.client = controller.serviceRegistry().horizonClient(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 226a7ca9561..7b66e60b426 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -35,6 +35,7 @@ import com.yahoo.yolean.Exceptions; import java.net.URI; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.logging.Level; @@ -98,22 +99,21 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private HttpResponse endpoints(Path path) { - var instanceId = instanceFrom(path); - var endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) - .sortedBy(Comparator.comparing(Endpoint::name)) - .asList(); - - var deployments = endpoints.stream() - .flatMap(e -> e.deployments().stream()) - .distinct() - .sorted(Comparator.comparing(DeploymentId::dottedString)) - .collect(Collectors.toList()); - - var deploymentsStatus = deployments.stream() - .collect(Collectors.toMap( - deploymentId -> deploymentId, - deploymentId -> controller.routing().of(deploymentId).routingStatus()) - ); + ApplicationId instanceId = instanceFrom(path); + List<Endpoint> endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) + .sortedBy(Comparator.comparing(Endpoint::dnsName)) + .asList(); + + List<DeploymentId> deployments = endpoints.stream() + .flatMap(e -> e.deployments().stream()) + .distinct() + .collect(Collectors.toList()); + + Map<DeploymentId, RoutingStatus> deploymentsStatus = deployments.stream() + .collect(Collectors.toMap( + deploymentId -> deploymentId, + deploymentId -> controller.routing().of(deploymentId).routingStatus()) + ); var slime = new Slime(); var root = slime.setObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index 4e0ace507c3..e0097c295c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; @@ -22,7 +22,7 @@ import java.util.logging.Level; * @author bjorncs */ @SuppressWarnings("unused") // Request handler listed in controller's services.xml -public class SystemFlagsHandler extends LoggingRequestHandler { +public class SystemFlagsHandler extends ThreadedHttpRequestHandler { private static final String API_PREFIX = "/system-flags/v1"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 7e9ae036cc7..33e6632b8e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; @@ -62,7 +62,7 @@ import java.util.stream.Collectors; * @author jonmv */ @SuppressWarnings("unused") // Handler -public class UserApiHandler extends LoggingRequestHandler { +public class UserApiHandler extends ThreadedHttpRequestHandler { private final static Logger log = Logger.getLogger(UserApiHandler.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 0ac4380f560..5b165a9ef37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; @@ -27,11 +27,11 @@ import java.util.stream.Collectors; * @author mpolden */ @SuppressWarnings("unused") -public class ZoneApiHandler extends LoggingRequestHandler { +public class ZoneApiHandler extends ThreadedHttpRequestHandler { private final ZoneRegistry zoneRegistry; - public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry) { + public ZoneApiHandler(ThreadedHttpRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry) { super(parentCtx); this.zoneRegistry = serviceRegistry.zoneRegistry(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 510635b005b..a7416cfa0ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; @@ -34,7 +34,7 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; - public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, + public ZoneApiHandler(ThreadedHttpRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, ConfigServerRestExecutor proxy, Controller controller) { super(parentCtx, controller.auditLogger()); this.zoneRegistry = serviceRegistry.zoneRegistry(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index be8e49cf661..34736c16a6b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -94,20 +93,7 @@ public class RoutingPolicy { return List.of(infraEndpoint.get()); } DeploymentId deployment = new DeploymentId(id.owner(), id.zone()); - List<Endpoint> endpoints = new ArrayList<>(); - endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment).in(system)); - // Add legacy endpoints - if (routingMethod == RoutingMethod.shared) { - endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment) - .on(Port.plain(4080)) - .legacy() - .in(system)); - endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment) - .on(Port.tls(4443)) - .legacy() - .in(system)); - } - return endpoints; + return List.of(endpoint(routingMethod).target(id.cluster(), deployment).in(system)); } /** Returns the region endpoint of this */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index e5eb1382ccf..1f205d6e577 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -64,17 +64,14 @@ public abstract class DeploymentRoutingContext implements RoutingContext { return controller.policies().read(deployment).of(id); } - /** - * Extension of a {@link DeploymentRoutingContext} for deployments using either {@link RoutingMethod#shared} or - * {@link RoutingMethod#sharedLayer4} routing. - */ + /** Extension of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#sharedLayer4} routing */ public static class SharedDeploymentRoutingContext extends DeploymentRoutingContext { private final Clock clock; private final ConfigServer configServer; public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) { - super(deployment, RoutingMethod.shared, controller); + super(deployment, RoutingMethod.sharedLayer4, controller); this.clock = Objects.requireNonNull(clock); this.configServer = Objects.requireNonNull(configServer); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java index 2923c8dff5c..bbd2e6bbb41 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java @@ -10,8 +10,7 @@ import java.time.Instant; import java.util.Objects; /** - * An implementation of {@link RoutingContext} for a zone, using either {@link RoutingMethod#shared} or - * {@link RoutingMethod#sharedLayer4} routing. + * An implementation of {@link RoutingContext} for a zone, using {@link RoutingMethod#sharedLayer4} routing. * * @author mpolden */ @@ -42,7 +41,7 @@ public class SharedZoneRoutingContext implements RoutingContext { @Override public RoutingMethod routingMethod() { - return RoutingMethod.shared; + return RoutingMethod.sharedLayer4; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index 5d244875ae5..c1abe38a2a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -84,9 +84,7 @@ public class DeploymentStatistics { for (Deployment deployment : instance.productionDeployments().values()) allVersions.add(deployment.version()); - JobList failing = status.jobs().failing() - .not().withStatus(RunStatus.outOfCapacity) - .not().withStatus(RunStatus.aborted); + JobList failing = status.jobs().failingHard(); // Add all unsuccessful runs for failing production jobs as any run may have resulted in an incomplete deployment // where a subset of nodes have upgraded. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 1215ddbc2ad..63c0193ba7f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -46,13 +46,13 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -237,14 +237,12 @@ public class ControllerTest { ZoneId usWest = ZoneId.from("prod.us-west-1"); ZoneId usCentral = ZoneId.from("prod.us-central-1"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .instances("beta,default") .endpoint("default", "foo") .region(usWest.region()) .region(usCentral.region()) // Two deployments should result in each DNS alias being registered once .build(); tester.controllerTester().zoneRegistry().setRoutingMethod(List.of(ZoneApiMock.from(usWest), ZoneApiMock.from(usCentral)), - RoutingMethod.shared, RoutingMethod.sharedLayer4); betaContext.submit(applicationPackage).deploy(); @@ -253,12 +251,6 @@ public class ControllerTest { assertFalse(betaDeployments.isEmpty()); Set<ContainerEndpoint> containerEndpoints = Set.of(new ContainerEndpoint("foo", "global", - List.of("beta--app1--tenant1.global.vespa.oath.cloud", - "rotation-id-01"), - OptionalInt.empty(), - RoutingMethod.shared), - new ContainerEndpoint("foo", - "global", List.of("beta.app1.tenant1.global.vespa.oath.cloud", "rotation-id-01"), OptionalInt.empty(), @@ -277,12 +269,6 @@ public class ControllerTest { assertFalse(defaultDeployments.isEmpty()); Set<ContainerEndpoint> containerEndpoints = Set.of(new ContainerEndpoint("foo", "global", - List.of("app1--tenant1.global.vespa.oath.cloud", - "rotation-id-02"), - OptionalInt.empty(), - RoutingMethod.shared), - new ContainerEndpoint("foo", - "global", List.of("app1.tenant1.global.vespa.oath.cloud", "rotation-id-02"), OptionalInt.empty(), @@ -294,8 +280,8 @@ public class ControllerTest { defaultContext.flushDnsUpdates(); } - Map<String, String> rotationCnames = Map.of("beta--app1--tenant1.global.vespa.oath.cloud", "rotation-fqdn-01.", - "app1--tenant1.global.vespa.oath.cloud", "rotation-fqdn-02."); + Map<String, String> rotationCnames = Map.of("beta.app1.tenant1.global.vespa.oath.cloud", "rotation-fqdn-01.", + "app1.tenant1.global.vespa.oath.cloud", "rotation-fqdn-02."); rotationCnames.forEach((cname, data) -> { var record = tester.controllerTester().findCname(cname); assertTrue(record.isPresent()); @@ -303,10 +289,8 @@ public class ControllerTest { assertEquals(data, record.get().data().asString()); }); - Map<ApplicationId, Set<String>> globalDnsNamesByInstance = Map.of(betaContext.instanceId(), Set.of("beta--app1--tenant1.global.vespa.oath.cloud", - "beta.app1.tenant1.global.vespa.oath.cloud"), - defaultContext.instanceId(), Set.of("app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.oath.cloud")); + Map<ApplicationId, Set<String>> globalDnsNamesByInstance = Map.of(betaContext.instanceId(), Set.of("beta.app1.tenant1.global.vespa.oath.cloud"), + defaultContext.instanceId(), Set.of("app1.tenant1.global.vespa.oath.cloud")); globalDnsNamesByInstance.forEach((instance, dnsNames) -> { Set<String> actualDnsNames = tester.controller().routing().readDeclaredEndpointsOf(instance) @@ -333,35 +317,22 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", - "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.yahooapis.com"), + "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(deployment.zone()))); } context.flushDnsUpdates(); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + Optional<Record> record = tester.controllerTester().findCname("app1.tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("app1.tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); List<String> globalDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.instanceId()) .scope(Endpoint.Scope.global) + .sortedBy(Comparator.comparing(Endpoint::dnsName)) .mapToList(Endpoint::dnsName); - assertEquals(List.of("app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.yahooapis.com"), + assertEquals(List.of("app1.tenant1.global.vespa.oath.cloud"), globalDnsNames); } @@ -382,11 +353,11 @@ public class ControllerTest { assertFalse(deployments.isEmpty()); var notWest = Set.of( - "rotation-id-01", "foobar--app1--tenant1.global.vespa.oath.cloud", - "rotation-id-02", "app1--tenant1.global.vespa.oath.cloud", - "rotation-id-03", "all--app1--tenant1.global.vespa.oath.cloud" + "rotation-id-01", "foobar.app1.tenant1.global.vespa.oath.cloud", + "rotation-id-02", "app1.tenant1.global.vespa.oath.cloud", + "rotation-id-03", "all.app1.tenant1.global.vespa.oath.cloud" ); - var west = Sets.union(notWest, Set.of("rotation-id-04", "west--app1--tenant1.global.vespa.oath.cloud")); + var west = Sets.union(notWest, Set.of("rotation-id-04", "west.app1.tenant1.global.vespa.oath.cloud")); for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), @@ -397,24 +368,24 @@ public class ControllerTest { assertEquals(4, tester.controllerTester().nameService().records().size()); - var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + var record1 = tester.controllerTester().findCname("app1.tenant1.global.vespa.oath.cloud"); assertTrue(record1.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); + assertEquals("app1.tenant1.global.vespa.oath.cloud", record1.get().name().asString()); assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); + var record2 = tester.controllerTester().findCname("foobar.app1.tenant1.global.vespa.oath.cloud"); assertTrue(record2.isPresent()); - assertEquals("foobar--app1--tenant1.global.vespa.oath.cloud", record2.get().name().asString()); + assertEquals("foobar.app1.tenant1.global.vespa.oath.cloud", record2.get().name().asString()); assertEquals("rotation-fqdn-01.", record2.get().data().asString()); - var record3 = tester.controllerTester().findCname("all--app1--tenant1.global.vespa.oath.cloud"); + var record3 = tester.controllerTester().findCname("all.app1.tenant1.global.vespa.oath.cloud"); assertTrue(record3.isPresent()); - assertEquals("all--app1--tenant1.global.vespa.oath.cloud", record3.get().name().asString()); + assertEquals("all.app1.tenant1.global.vespa.oath.cloud", record3.get().name().asString()); assertEquals("rotation-fqdn-03.", record3.get().data().asString()); - var record4 = tester.controllerTester().findCname("west--app1--tenant1.global.vespa.oath.cloud"); + var record4 = tester.controllerTester().findCname("west.app1.tenant1.global.vespa.oath.cloud"); assertTrue(record4.isPresent()); - assertEquals("west--app1--tenant1.global.vespa.oath.cloud", record4.get().name().asString()); + assertEquals("west.app1.tenant1.global.vespa.oath.cloud", record4.get().name().asString()); assertEquals("rotation-fqdn-04.", record4.get().data().asString()); } @@ -437,7 +408,7 @@ public class ControllerTest { for (var zone : List.of(west, central)) { assertEquals( "Zone " + zone + " is a member of global endpoint", - Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-01", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -455,13 +426,13 @@ public class ControllerTest { for (var zone : List.of(west, central)) { assertEquals( "Zone " + zone + " is a member of global endpoint", - Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-01", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } assertEquals( "Zone " + east + " is a member of global endpoint", - Set.of("rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-02", "east.app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(east)) ); @@ -478,9 +449,9 @@ public class ControllerTest { assertEquals( "Zone " + zone + " is a member of global endpoint", zone.equals(east) - ? Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", - "rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud") - : Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + ? Set.of("rotation-id-01", "app1.tenant1.global.vespa.oath.cloud", + "rotation-id-02", "east.app1.tenant1.global.vespa.oath.cloud") + : Set.of("rotation-id-01", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -563,6 +534,7 @@ public class ControllerTest { @Test public void testDnsUpdatesWithChangeInRotationAssignment() { // Application 1 is deployed and deleted + String dnsName1 = "app1.tenant1.global.vespa.oath.cloud"; { var context = tester.newDeploymentContext("tenant1", "app1", "default"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -573,11 +545,10 @@ public class ControllerTest { context.submit(applicationPackage).deploy(); assertEquals(1, tester.controllerTester().nameService().records().size()); - { - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + Optional<Record> record = tester.controllerTester().findCname(dnsName1); assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); + assertEquals(dnsName1, record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); } @@ -596,17 +567,13 @@ public class ControllerTest { } context.flushDnsUpdates(); - // Records are removed - List<String> removed = List.of("app1--tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com"); - for (var name : removed) { - Optional<Record> record = tester.controllerTester().findCname(name); - assertTrue(name + " is removed", record.isEmpty()); - } + // Record is removed + Optional<Record> record = tester.controllerTester().findCname(dnsName1); + assertTrue(dnsName1 + " is removed", record.isEmpty()); } // Application 2 is deployed and assigned same rotation as application 1 had before deletion + String dnsName2 = "app2.tenant2.global.vespa.oath.cloud"; { var context = tester.newDeploymentContext("tenant2", "app2", "default"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -617,9 +584,9 @@ public class ControllerTest { context.submit(applicationPackage).deploy(); assertEquals(1, tester.controllerTester().nameService().records().size()); - var record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + var record = tester.controllerTester().findCname(dnsName2); assertTrue(record.isPresent()); - assertEquals("app2--tenant2.global.vespa.oath.cloud", record.get().name().asString()); + assertEquals(dnsName2, record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); } @@ -637,11 +604,11 @@ public class ControllerTest { // DNS records are created for the newly assigned rotation assertEquals(2, tester.controllerTester().nameService().records().size()); - var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + var record1 = tester.controllerTester().findCname(dnsName1); assertTrue(record1.isPresent()); assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - var record2 = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + var record2 = tester.controllerTester().findCname(dnsName2); assertTrue(record2.isPresent()); assertEquals("rotation-fqdn-01.", record2.get().data().asString()); } @@ -722,7 +689,7 @@ public class ControllerTest { var context = tester.newDeploymentContext(); ZoneId zone = ZoneId.from("dev", "us-east-1"); tester.controllerTester().zoneRegistry() - .setRoutingMethod(ZoneApiMock.from(zone), RoutingMethod.shared, RoutingMethod.sharedLayer4); + .setRoutingMethod(ZoneApiMock.from(zone), RoutingMethod.sharedLayer4); // Deploy context.runJob(zone, applicationPackage); @@ -738,7 +705,7 @@ public class ControllerTest { .stream() .map(Endpoint::routingMethod) .collect(Collectors.toSet()); - assertEquals(routingMethods, Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4)); + assertEquals(routingMethods, Set.of(RoutingMethod.sharedLayer4)); // Deployment has stored application meta. assertNotNull(tester.controllerTester().serviceRegistry().applicationStore() @@ -778,12 +745,11 @@ public class ControllerTest { public void testDeletingApplicationThatHasAlreadyBeenDeleted() { var context = tester.newDeploymentContext(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region("us-east-3") .region("us-west-1") .build(); ZoneId zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); - context.runJob(zone, applicationPackage); + context.submit(applicationPackage).runJob(zone, applicationPackage); tester.controller().applications().deactivate(context.instanceId(), zone); tester.controller().applications().deactivate(context.instanceId(), zone); } @@ -914,12 +880,10 @@ public class ControllerTest { .region(zone2.region()) .build(); - // Zone 1 supports shared and sharedLayer4 - tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, - RoutingMethod.sharedLayer4); + // Zone 1 supports sharedLayer4 + tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.sharedLayer4); // Zone 2 supports shared and exclusive - tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, - RoutingMethod.exclusive); + tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.exclusive); context.submit(applicationPackage).deploy(); var expectedRecords = List.of( @@ -935,20 +899,10 @@ public class ControllerTest { new LatencyAliasTarget(HostName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), "dns-zone-1", ZoneId.from("prod.us-east-3")).pack()), - // The 'default' global endpoint, pointing to both zones with shared routing, via rotation - new Record(Record.Type.CNAME, - RecordName.from("application--tenant.global.vespa.oath.cloud"), - RecordData.from("rotation-fqdn-01.")), - // The zone-scoped endpoint pointing to zone 2 with exclusive routing new Record(Record.Type.CNAME, RecordName.from("application.tenant.us-east-3.vespa.oath.cloud"), - RecordData.from("lb-0--tenant:application:default--prod.us-east-3.")), - - // The 'east' global endpoint, pointing to zone 2 with shared routing, via rotation - new Record(Record.Type.CNAME, - RecordName.from("east--application--tenant.global.vespa.oath.cloud"), - RecordData.from("rotation-fqdn-02."))); + RecordData.from("lb-0--tenant:application:default--prod.us-east-3."))); assertEquals(expectedRecords, List.copyOf(tester.controllerTester().nameService().records())); } @@ -988,53 +942,6 @@ public class ControllerTest { } @Test - public void testDeploymentWithSharedAndDirectRouting() { - var context = tester.newDeploymentContext(); - var zone1 = ZoneId.from("prod", "us-west-1"); - var zone2 = ZoneId.from("prod", "us-east-3"); - var applicationPackageBuilder = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()); - tester.controllerTester().zoneRegistry() - .setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, RoutingMethod.sharedLayer4) - .setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, RoutingMethod.sharedLayer4); - Supplier<Set<RoutingMethod>> routingMethods = () -> tester.controller().routing().readEndpointsOf(context.deploymentIdIn(zone1)) - .asList() - .stream() - .map(Endpoint::routingMethod) - .collect(Collectors.toSet()); - - // Without satisfying requirements - context.submit(applicationPackageBuilder.build()).deploy(); - assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); - - // Package satisfying all requirements is submitted, but not deployed yet - applicationPackageBuilder = applicationPackageBuilder.athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")); - var context2 = context.submit(applicationPackageBuilder.build()); - assertEquals("Direct routing endpoint is available after submission and before deploy", - Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4), routingMethods.get()); - context2.deploy(); - - // Global endpoint is added and includes directly routed endpoint name - applicationPackageBuilder = applicationPackageBuilder.endpoint("default", "default"); - context2.submit(applicationPackageBuilder.build()).deploy(); - for (var zone : List.of(zone1, zone2)) { - assertEquals(Set.of("rotation-id-01", - "application.tenant.global.vespa.oath.cloud", - "application--tenant.global.vespa.oath.cloud"), - tester.configServer().containerEndpointNames(context.deploymentIdIn(zone))); - } - List<String> zoneDnsNames = tester.controller().routing().readEndpointsOf(context.deploymentIdIn(zone1)) - .scope(Endpoint.Scope.zone) - .mapToList(Endpoint::dnsName); - assertEquals(List.of("application--tenant.us-west-1.vespa.oath.cloud", - "application.tenant.us-west-1.prod.vespa.yahooapis.com", - "application--tenant.us-west-1.prod.vespa.yahooapis.com", - "application.tenant.us-west-1.vespa.oath.cloud"), - zoneDnsNames); - } - - @Test public void testChangeEndpointCluster() { var context = tester.newDeploymentContext(); var west = ZoneId.from("prod", "us-west-1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index e50c32d0e5d..35cca0e1f1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -33,26 +33,6 @@ public class EndpointTest { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint - "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.plain(4080)).legacy().in(SystemName.main), - - // Legacy endpoint with TLS - "https://a1--t1.global.vespa.yahooapis.com:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).legacy().in(SystemName.main), - - // Main endpoint - "https://a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).in(SystemName.main), - - // Main endpoint in CD - "https://cd--a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).in(SystemName.cd), - - // Main endpoint in CD - "https://cd--i2--a2--t2.global.vespa.oath.cloud:4443/", - Endpoint.of(instance2).target(endpointId, cluster, List.of(deployment2)).on(Port.tls(4443)).in(SystemName.cd), - // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), @@ -99,26 +79,6 @@ public class EndpointTest { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint - "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.plain(4080)).legacy().in(SystemName.main), - - // Legacy endpoint with TLS - "https://a1--t1.global.vespa.yahooapis.com:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).legacy().in(SystemName.main), - - // Main endpoint - "https://a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).in(SystemName.main), - - // Main endpoint in CD - "https://cd--i2--a2--t2.global.vespa.oath.cloud:4443/", - Endpoint.of(instance2).target(endpointId, cluster, List.of(deployment2)).on(Port.tls(4443)).in(SystemName.cd), - - // Main endpoint in CD - "https://cd--a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).in(SystemName.cd), - // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), @@ -161,33 +121,25 @@ public class EndpointTest { var testZone = new DeploymentId(instance1, ZoneId.from("test", "us-north-2")); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint (always contains environment) - "http://a1.t1.us-north-1.prod.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.plain(4080)).legacy().in(SystemName.main), - - // Secure legacy endpoint - "https://a1--t1.us-north-1.prod.vespa.yahooapis.com:4443/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls(4443)).legacy().in(SystemName.main), - // Prod endpoint in main - "https://a1--t1.us-north-1.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + "https://a1.t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.main), // Prod endpoint in CD - "https://cd--a1--t1.us-north-1.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.cd), + "https://cd.a1.t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.cd), // Test endpoint in main - "https://a1--t1.us-north-2.test.vespa.oath.cloud:4443/", - Endpoint.of(instance1).target(cluster, testZone).on(Port.tls(4443)).in(SystemName.main), + "https://a1.t1.us-north-2.test.vespa.oath.cloud/", + Endpoint.of(instance1).target(cluster, testZone).on(Port.tls()).in(SystemName.main), // Non-default cluster in main - "https://c1--a1--t1.us-north-1.vespa.oath.cloud/", + "https://c1.a1.t1.us-north-1.vespa.oath.cloud/", Endpoint.of(instance1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).in(SystemName.main), // Non-default instance in main - "https://i2--a2--t2.us-north-1.vespa.oath.cloud:4443/", - Endpoint.of(instance2).target(cluster, prodZone2).on(Port.tls(4443)).in(SystemName.main), + "https://i2.a2.t2.us-north-1.vespa.oath.cloud/", + Endpoint.of(instance2).target(cluster, prodZone2).on(Port.tls()).in(SystemName.main), // Non-default cluster in public "https://c1.a1.t1.us-north-1.z.vespa-app.cloud/", @@ -195,11 +147,7 @@ public class EndpointTest { // Non-default cluster and instance in public "https://c2.i2.a2.t2.us-north-1.z.vespa-app.cloud/", - Endpoint.of(instance2).target(ClusterSpec.Id.from("c2"), prodZone2).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), - - // Endpoint in main using shared layer 4 - "https://a1.t1.us-north-1.vespa.oath.cloud/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).routingMethod(RoutingMethod.sharedLayer4).in(SystemName.main) + Endpoint.of(instance2).target(ClusterSpec.Id.from("c2"), prodZone2).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); @@ -365,11 +313,11 @@ public class EndpointTest { var tests1 = Map.of( // With default cluster "a1.t1.us-north-1.prod", - Endpoint.of(instance1).target(EndpointId.defaultId(), ClusterSpec.Id.from("default"), List.of(zone)).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(instance1).target(EndpointId.defaultId(), ClusterSpec.Id.from("default"), List.of(zone)).on(Port.tls()).in(SystemName.main), // With non-default cluster "c1.a1.t1.us-north-1.prod", - Endpoint.of(instance1).target(EndpointId.of("ignored1"), ClusterSpec.Id.from("c1"), List.of(zone)).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(instance1).target(EndpointId.of("ignored1"), ClusterSpec.Id.from("c1"), List.of(zone)).on(Port.tls()).in(SystemName.main), // With application endpoint "c2.a1.t1.us-north-1.prod", @@ -381,11 +329,11 @@ public class EndpointTest { var tests2 = Map.of( // With non-default instance and default cluster "i2.a2.t2.us-north-1.prod", - Endpoint.of(instance2).target(EndpointId.defaultId(), ClusterSpec.Id.from("default"), List.of(zone2)).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(instance2).target(EndpointId.defaultId(), ClusterSpec.Id.from("default"), List.of(zone2)).on(Port.tls()).in(SystemName.main), // With non-default instance and cluster "c2.i2.a2.t2.us-north-1.prod", - Endpoint.of(instance2).target(EndpointId.of("ignored2"), ClusterSpec.Id.from("c2"), List.of(zone2)).on(Port.tls(4443)).in(SystemName.main) + Endpoint.of(instance2).target(EndpointId.of("ignored2"), ClusterSpec.Id.from("c2"), List.of(zone2)).on(Port.tls()).in(SystemName.main) ); tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone))); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone2))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 2ee03457046..cf0b46dfba2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -7,6 +7,8 @@ import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayInputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; import java.util.List; import java.util.Map; @@ -14,6 +16,7 @@ import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -108,6 +111,21 @@ public class ApplicationPackageTest { } } + @Test + public void testBundleHashesAreSameWithDifferentDeploymentXml() throws Exception { + var originalPackage = getApplicationZip("original.zip"); + var changedDeploymentXml = getApplicationZip("changed-deployment-xml.zip"); + var changedServices = getApplicationZip("changed-services-xml.zip"); + + // services.xml is changed -> different bundle hash + assertNotEquals(originalPackage.bundleHash(), changedServices.bundleHash()); + assertNotEquals(originalPackage.hash(), changedServices.hash()); + + // deployment.xml is changed -> same bundle hash + assertEquals(originalPackage.bundleHash(), changedDeploymentXml.bundleHash()); + assertNotEquals(originalPackage.hash(), changedDeploymentXml.hash()); + } + private static Map<String, String> unzip(byte[] zip) { return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) .entries().stream() @@ -115,4 +133,8 @@ public class ApplicationPackageTest { entry -> new String(entry.contentOrThrow(), UTF_8))); } + private ApplicationPackage getApplicationZip(String path) throws Exception { + return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java index d6ec01a2d57..1a052b6a578 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java @@ -16,7 +16,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class CuratorArchiveBucketDbTest { @@ -29,19 +29,22 @@ public class CuratorArchiveBucketDbTest { Set.of(new ArchiveBucket("existingBucket", "keyArn").withTenant(TenantName.defaultName()))); // Finds existing bucket in db - assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName())); + assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName(), true)); // Assigns to existing bucket while there is space IntStream.range(0, 29).forEach(i -> assertEquals( Optional.of(URI.create("s3://existingBucket/tenant" + i + "/")), bucketDb - .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i)))); + .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i), true))); // Creates new bucket when existing buckets are full - assertEquals(Optional.of(URI.create("s3://bucketName/lastDrop/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.from("lastDrop"))); + assertEquals(Optional.of(URI.create("s3://bucketName/lastDrop/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.from("lastDrop"), true)); // Creates new bucket when there are no existing buckets in zone - assertEquals(Optional.of(URI.create("s3://bucketName/firstInZone/")), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("firstInZone"))); + assertEquals(Optional.of(URI.create("s3://bucketName/firstInZone/")), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("firstInZone"), true)); + + // Does not create bucket if not required + assertEquals(Optional.empty(), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("newTenant"), false)); // Lists all buckets by zone Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 29).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 91a12d3b465..bc5a70b1fa0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -57,9 +57,10 @@ public class ApplicationPackageBuilder { private OptionalInt majorVersion = OptionalInt.empty(); private String instances = "default"; private String upgradePolicy = null; + private String upgradeRevision = "latest"; private String upgradeRollout = null; private String globalServiceId = null; - private String athenzIdentityAttributes = null; + private String athenzIdentityAttributes = "athenz-domain='domain' athenz-service='service'"; private String searchDefinition = "search test { }"; private boolean explicitSystemTest = false; private boolean explicitStagingTest = false; @@ -80,6 +81,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder upgradeRevision(String upgradeRevision) { + this.upgradeRevision = upgradeRevision; + return this; + } + public ApplicationPackageBuilder upgradeRollout(String upgradeRollout) { this.upgradeRollout = upgradeRollout; return this; @@ -195,7 +201,12 @@ public class ApplicationPackageBuilder { public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) { this.athenzIdentityAttributes = Text.format("athenz-domain='%s' athenz-service='%s'", domain.value(), - service.value()); + service.value()); + return this; + } + + public ApplicationPackageBuilder withoutAthenzIdentity() { + this.athenzIdentityAttributes = null; return this; } @@ -248,9 +259,10 @@ public class ApplicationPackageBuilder { } xml.append(">\n"); xml.append(" <instance id='").append(instances).append("'>\n"); - if (upgradePolicy != null || upgradeRollout != null) { + if (upgradePolicy != null || upgradeRevision != null || upgradeRollout != null) { xml.append(" <upgrade "); if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); + if (upgradeRevision != null) xml.append("revision='").append(upgradeRevision).append("' "); if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); xml.append("/>\n"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 699721b128c..9c5f7e3376a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -164,14 +164,13 @@ public class DeploymentContext { } - /** Completely deploy the latest change */ + /** Completely deploy the current change */ public DeploymentContext deploy() { Application application = application(); assertTrue("Application package submitted", application.latestVersion().isPresent()); assertFalse("Submission is not already deployed", application.instances().values().stream() .anyMatch(instance -> instance.deployments().values().stream() .anyMatch(deployment -> deployment.applicationVersion().equals(lastSubmission)))); - assertEquals(application.latestVersion(), instance().change().application()); completeRollout(application.deploymentSpec().instances().size() > 1); for (var instance : application().instances().values()) { assertFalse(instance.change().hasTargets()); @@ -334,20 +333,20 @@ public class DeploymentContext { /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */ public DeploymentContext completeRollout(boolean multiInstance) { triggerJobs(); - Map<ApplicationId, Set<JobType>> jobsByInstance = new HashMap<>(); + Map<ApplicationId, Map<JobType, Versions>> jobsByInstance = new HashMap<>(); List<Run> activeRuns; while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty()) for (Run run : activeRuns) { - Set<JobType> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashSet<>()); - if (jobs.add(run.id().type())) { - runJob(run.id().type(), run.id().application()); - if (multiInstance) { - tester.outstandingChangeDeployer().run(); - } - triggerJobs(); - } else { - throw new AssertionError("Job '" + run.id() + "' was run twice"); + Map<JobType, Versions> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>()); + Versions previous = jobs.put(run.id().type(), run.versions()); + if (run.versions().equals(previous)) { + throw new AssertionError("Job '" + run.id() + "' was run twice on same versions"); } + runJob(run.id().type(), run.id().application()); + if (multiInstance) { + tester.outstandingChangeDeployer().run(); + } + triggerJobs(); } assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 102dfde16ec..67fa2f87794 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -107,6 +107,40 @@ public class DeploymentTriggerTest { } @Test + public void separateRevisionMakesApplicationChangeWaitForPreviousToComplete() { + DeploymentContext app = tester.newDeploymentContext(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradeRevision(null) // separate by default, but we override this in test builder + .region("us-east-3") + .test("us-east-3") + .build(); + + app.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); + Optional<ApplicationVersion> v0 = app.lastSubmission(); + + app.submit(applicationPackage); + Optional<ApplicationVersion> v1 = app.lastSubmission(); + assertEquals(v0, app.instance().change().application()); + + // Eager tests still run before new revision rolls out. + app.runJob(systemTest).runJob(stagingTest); + + // v0 rolls out completely. + app.runJob(testUsEast3); + assertEquals(Optional.empty(), app.instance().change().application()); + + // v1 starts rolling when v0 is done. + tester.outstandingChangeDeployer().run(); + assertEquals(v1, app.instance().change().application()); + + // v1 fails, so v2 starts immediately. + app.runJob(productionUsEast3).failDeployment(testUsEast3); + app.submit(applicationPackage); + Optional<ApplicationVersion> v2 = app.lastSubmission(); + assertEquals(v2, app.instance().change().application()); + } + + @Test public void leadingUpgradeAllowsApplicationChangeWhileUpgrading() { var applicationPackage = new ApplicationPackageBuilder().region("us-east-3") .upgradeRollout("leading") @@ -155,9 +189,10 @@ public class DeploymentTriggerTest { tester.controllerTester().upgradeSystem(new Version("8.9")); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest); + tester.clock().advance(Duration.ofMinutes(1)); tester.triggerJobs(); - // Jobs are not aborted when the new submission remains outstanding. + // Upgrade is allowed to proceed ahead of revision change, and is not aborted. app.submit(); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); @@ -347,17 +382,15 @@ public class DeploymentTriggerTest { // Application on (6.1, 1.0.1) Version v1 = Version.fromString("6.1"); - // Application is mid-upgrade when block window begins, and has an outstanding change. + // Application is mid-upgrade when block window begins, and gets an outstanding change. Version v2 = Version.fromString("6.2"); tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); - app.submit(applicationPackage); - app.runJob(stagingTest).runJob(systemTest); // Entering block window will keep the outstanding change in place. tester.clock().advance(Duration.ofHours(1)); - tester.outstandingChangeDeployer().run(); + app.submit(applicationPackage); app.runJob(productionUsWest1); assertEquals(1, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetApplication().buildNumber().getAsLong()); assertEquals(2, app.deploymentStatus().outstandingChange(app.instance().name()).application().get().buildNumber().getAsLong()); @@ -465,18 +498,45 @@ public class DeploymentTriggerTest { } @Test - public void settingANoOpChangeIsANoOp() { + public void downgradingApplicationVersionWorks() { var app = tester.newDeploymentContext().submit().deploy(); ApplicationVersion appVersion0 = app.lastSubmission().get(); + assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + app.submit().deploy(); ApplicationVersion appVersion1 = app.lastSubmission().get(); + assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); - // Triggering a roll-out of an already deployed application is a no-op. - assertEquals(Change.empty(), app.instance().change()); + // Downgrading application version. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); + assertEquals(Change.of(appVersion0), app.instance().change()); + app.runJob(stagingTest) + .runJob(productionUsCentral1) + .runJob(productionUsEast3) + .runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).applicationVersion()); + assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + } + + @Test + public void settingANoOpChangeIsANoOp() { + var app = tester.newDeploymentContext().submit(); + assertEquals(Optional.empty(), app.instance().latestDeployed()); + + app.deploy(); + ApplicationVersion appVersion0 = app.lastSubmission().get(); + assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + + app.submit().deploy(); + ApplicationVersion appVersion1 = app.lastSubmission().get(); + assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); + + // Triggering a roll-out of an already deployed application is a no-op. assertEquals(Change.empty(), app.instance().change()); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion1)); assertEquals(Change.empty(), app.instance().change()); + assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); } @Test @@ -538,19 +598,23 @@ public class DeploymentTriggerTest { // Change has a higher application version than what is deployed -- deployment should trigger. app1.timeOutUpgrade(productionUsCentral1); - assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone(main)).version()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); assertEquals(revision2, app1.deployment(productionUsCentral1.zone(main)).applicationVersion()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. - tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1))); // Enough time for retry + tester.clock().advance(Duration.ofHours(3)); // Enough time for retry tester.triggerJobs(); // Failing job is not retried as change has been deployed app1.assertNotRunning(productionUsCentral1); // Last job has a different deployment target, so tests need to run again. - app1.runJob(systemTest).runJob(stagingTest).runJob(productionEuWest1); - assertFalse(app1.instance().change().hasTargets()); - assertFalse(app1.instanceJobs().get(productionUsCentral1).isSuccess()); + app1.runJob(systemTest) + .runJob(stagingTest) // Eager test of outstanding change, assuming upgrade in west succeeds. + .runJob(productionEuWest1) // Upgrade completes, and revision is the only change. + .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. + .runJob(productionEuWest1); // Finally, west changes revision. + assertEquals(Change.empty(), app1.instance().change()); + assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus()); } @Test @@ -792,6 +856,115 @@ public class DeploymentTriggerTest { } @Test + public void testMultipleInstancesWithDifferentChanges() { + DeploymentContext i1 = tester.newDeploymentContext("t", "a", "i1"); + DeploymentContext i2 = tester.newDeploymentContext("t", "a", "i2"); + DeploymentContext i3 = tester.newDeploymentContext("t", "a", "i3"); + DeploymentContext i4 = tester.newDeploymentContext("t", "a", "i4"); + ApplicationPackage applicationPackage = ApplicationPackageBuilder + .fromDeploymentXml("<deployment version='1'>\n" + + " <upgrade revision='separate' />\n" + + " <parallel>\n" + + " <instance id='i1'>\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <delay hours='6' />\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='i2'>\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " </prod>\n" + + " </instance>\n" + + " </parallel>\n" + + " <instance id='i3'>\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <delay hours='18' />\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='i4'>\n" + + " <test />\n" + + " <staging />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + // Package is submitted, and change propagated to the two first instances. + i1.submit(applicationPackage); + Optional<ApplicationVersion> v0 = i1.lastSubmission(); + tester.outstandingChangeDeployer().run(); + assertEquals(v0, i1.instance().change().application()); + assertEquals(v0, i2.instance().change().application()); + assertEquals(Optional.empty(), i3.instance().change().application()); + assertEquals(Optional.empty(), i4.instance().change().application()); + + // Tests run in i4, as they're declared there, and i1 and i2 get to work + i4.runJob(systemTest).runJob(stagingTest); + i1.runJob(productionUsEast3); + i2.runJob(productionUsEast3); + + // Since the post-deployment delay of i1 is incomplete, i3 doesn't yet get the change. + tester.outstandingChangeDeployer().run(); + assertEquals(v0, i1.instance().latestDeployed()); + assertEquals(v0, i2.instance().latestDeployed()); + assertEquals(Optional.empty(), i1.instance().change().application()); + assertEquals(Optional.empty(), i2.instance().change().application()); + assertEquals(Optional.empty(), i3.instance().change().application()); + assertEquals(Optional.empty(), i4.instance().change().application()); + + // When the delay is done, i3 gets the change. + tester.clock().advance(Duration.ofHours(6)); + tester.outstandingChangeDeployer().run(); + assertEquals(Optional.empty(), i1.instance().change().application()); + assertEquals(Optional.empty(), i2.instance().change().application()); + assertEquals(v0, i3.instance().change().application()); + assertEquals(Optional.empty(), i4.instance().change().application()); + + // v0 begins roll-out in i3, and v1 is submitted and rolls out in i1 and i2 some time later + i3.runJob(productionUsEast3); // v0 + tester.clock().advance(Duration.ofHours(12)); + i1.submit(applicationPackage); + Optional<ApplicationVersion> v1 = i1.lastSubmission(); + i4.runJob(systemTest).runJob(stagingTest); + i1.runJob(productionUsEast3); // v1 + i2.runJob(productionUsEast3); // v1 + assertEquals(v1, i1.instance().latestDeployed()); + assertEquals(v1, i2.instance().latestDeployed()); + assertEquals(Optional.empty(), i1.instance().change().application()); + assertEquals(Optional.empty(), i2.instance().change().application()); + assertEquals(v0, i3.instance().change().application()); + assertEquals(Optional.empty(), i4.instance().change().application()); + + // After some time, v2 also starts rolling out to i1 and i2, but does not complete in i2 + tester.clock().advance(Duration.ofHours(3)); + i1.submit(applicationPackage); + Optional<ApplicationVersion> v2 = i1.lastSubmission(); + i4.runJob(systemTest).runJob(stagingTest); + i1.runJob(productionUsEast3); // v2 + tester.clock().advance(Duration.ofHours(3)); + + // v1 is all done in i1 and i2, but does not yet roll out in i3; v2 is not completely rolled out there yet. + tester.outstandingChangeDeployer().run(); + assertEquals(v0, i3.instance().change().application()); + + // i3 completes v0, which rolls out to i4; v1 is ready for i3, but v2 is not. + i3.runJob(testUsEast3); + assertEquals(Optional.empty(), i3.instance().change().application()); + tester.outstandingChangeDeployer().run(); + assertEquals(v2, i1.instance().latestDeployed()); + assertEquals(v1, i2.instance().latestDeployed()); + assertEquals(v0, i3.instance().latestDeployed()); + assertEquals(Optional.empty(), i1.instance().change().application()); + assertEquals(v2, i2.instance().change().application()); + assertEquals(v1, i3.instance().change().application()); + assertEquals(v0, i4.instance().change().application()); + } + + @Test public void testMultipleInstances() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .instances("instance1,instance2") @@ -998,48 +1171,65 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - // Upgrade instance 1; a failure in any instance allows an application change to accompany the upgrade. + // Upgrade instance 1; upgrade rolls out first, with revision following. // The new platform won't roll out to the conservative instance until the normal one is upgraded. - app2.failDeployment(systemTest); app1.submit(applicationPackage); assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app1.instance().change()); + // Upgrade platform. app2.runJob(systemTest); - app1.jobAborted(stagingTest) - .runJob(stagingTest) + app1.runJob(stagingTest) .runJob(productionUsWest1) .runJob(productionUsEast3); - app1.runJob(stagingTest); // Tests with only the outstanding application change. - app2.runJob(systemTest); // Tests with only the outstanding application change. + // Upgrade revision + tester.clock().advance(Duration.ofSeconds(1)); // Ensure we see revision as rolling after upgrade. + app2.runJob(systemTest); // R + app1.runJob(stagingTest) // R + .runJob(productionUsWest1); // R + // productionUsEast3 won't change revision before its production test has completed for the upgrade, which is one of the last jobs! tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(productionAwsUsEast1a); - tester.triggerJobs(); app1.runJob(testAwsUsEast1a); + tester.clock().advance(Duration.ofSeconds(1)); + app1.runJob(productionAwsUsEast1a); // R + app1.runJob(testAwsUsEast1a); // R app1.runJob(productionApNortheast2); app1.runJob(productionApNortheast1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(testApNortheast1); app1.runJob(testApNortheast2); + app1.runJob(productionApNortheast2); // R + app1.runJob(productionApNortheast1); // R app1.runJob(testUsEast3); app1.runJob(productionApSoutheast1); + tester.clock().advance(Duration.ofSeconds(1)); + app1.runJob(productionUsEast3); // R + tester.clock().advance(Duration.ofHours(2)); + app1.runJob(productionEuWest1); // R + tester.clock().advance(Duration.ofMinutes(330)); + app1.runJob(testApNortheast1); // R + app1.runJob(testApNortheast2); // R + app1.runJob(testUsEast3); // R + app1.runJob(productionApSoutheast1); // R + + app1.runJob(stagingTest); // Tests with only the outstanding application change. + app2.runJob(systemTest); // Tests with only the outstanding application change. // Confidence rises to high, for the new version, and instance 2 starts to upgrade. tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(tester.jobs().active().toString(), 1, tester.jobs().active().size()); assertEquals(Change.empty(), app1.instance().change()); assertEquals(Change.of(version), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - app1.runJob(stagingTest); // Never completed successfully with just the upgrade. - app2.runJob(systemTest) // Never completed successfully with just the upgrade. - .runJob(productionEuWest1) + app2.runJob(productionEuWest1) .failDeployment(testEuWest1); - // Instance 2 failed the last job, and now exist block window, letting application change roll out with the upgrade. + // Instance 2 failed the last job, and now exits block window, letting application change roll out with the upgrade. tester.clock().advance(Duration.ofDays(1)); // Leave block window for revisions. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); @@ -1054,18 +1244,19 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - // Two first instances upgraded and with new revision — last instance gets change from whatever maintainer runs first. + // Two first instances upgraded and with new revision — last instance gets both changes as well. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(version), app3.instance().change()); + assertEquals(Change.of(version).with(app1.lastSubmission().get()), app3.instance().change()); tester.deploymentTrigger().cancelChange(app3.instanceId(), ALL); tester.outstandingChangeDeployer().run(); tester.upgrader().maintain(); - assertEquals(Change.of(app1.application().latestVersion().get()), app3.instance().change()); + assertEquals(Change.of(app1.lastSubmission().get()), app3.instance().change()); app3.runJob(productionEuWest1); tester.upgrader().maintain(); + app1.runJob(stagingTest); app3.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); @@ -1073,22 +1264,410 @@ public class DeploymentTriggerTest { } @Test - public void testChangeCompletion() { - var app = tester.newDeploymentContext().submit().deploy(); - var version = new Version("7.1"); - tester.controllerTester().upgradeSystem(version); + public void testRevisionJoinsUpgradeWithSeparateRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("separate") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); - app.submit(); - tester.triggerJobs(); + // Revision starts rolling, but stays behind. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + + // Upgrade got here first, so attempts to proceed alone, but the upgrade fails. + app.triggerJobs(); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + app.timeOutConvergence(productionUsEast3); + + // Revision is allowed to join. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + app.runJob(productionUsEast3); + + // Platform and revision now proceed together. + app.runJob(stagingTest); + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testProductionTestBlockingDeploymentWithSeparateRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-east-3") + .region("us-west-1") + .delay(Duration.ofHours(1)) + .test("us-east-3") + .upgradeRollout("separate") + .build(); + var app = tester.newDeploymentContext().submit(appPackage) + .runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(productionUsWest1); + tester.clock().advance(Duration.ofHours(1)); + app.runJob(testUsEast3); + assertEquals(Change.empty(), app.instance().change()); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); + + // Revision starts rolling, but waits for production test to verify the upgrade. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).triggerJobs(); + app.assertRunning(productionUsWest1); + app.assertNotRunning(productionUsEast3); + + // Upgrade got here first, so attempts to proceed alone, but the upgrade fails. + app.triggerJobs(); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.timeOutConvergence(productionUsWest1).triggerJobs(); + + // Upgrade now fails between us-east-3 deployment and test, so test is abandoned, and revision unblocked. + app.assertRunning(productionUsEast3); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + app.runJob(productionUsEast3).triggerJobs() + .jobAborted(productionUsWest1).runJob(productionUsWest1); + tester.clock().advance(Duration.ofHours(1)); + app.runJob(testUsEast3); + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testProductionTestNotBlockingDeploymentWithSimultaneousRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-east-3") + .region("us-central-1") + .region("us-west-1") + .delay(Duration.ofHours(1)) + .test("us-east-3") + .test("us-west-1") + .upgradeRollout("simultaneous") + .build(); + var app = tester.newDeploymentContext().submit(appPackage) + .runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(productionUsCentral1).runJob(productionUsWest1); + tester.clock().advance(Duration.ofHours(1)); + app.runJob(testUsEast3).runJob(testUsWest1); + assertEquals(Change.empty(), app.instance().change()); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); + + // Revision starts rolling, and causes production test to abort when it reaches deployment. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).triggerJobs(); + app.assertRunning(productionUsCentral1); + app.assertRunning(productionUsEast3); + + // Revision deploys to first prod zone. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + tester.clock().advance(Duration.ofSeconds(1)); + app.runJob(productionUsEast3); + + // Revision catches up in second prod zone. + app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest).triggerJobs(); + app.jobAborted(productionUsCentral1).triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions()); + app.runJob(productionUsCentral1).triggerJobs(); + + // Revision proceeds alone in third prod zone, making test targets different for the two prod tests. + assertEquals(new Versions(version0, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + app.triggerJobs(); + app.assertNotRunning(testUsEast3); + tester.clock().advance(Duration.ofHours(1)); + + // Test lets revision proceed alone, and us-west-1 is blocked until tested. + app.runJob(testUsEast3).triggerJobs(); + app.assertNotRunning(productionUsWest1); + app.runJob(testUsWest1).runJob(productionUsWest1).runJob(testUsWest1); // Test for us-east-3 is not re-run. + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testVeryLengthyPipelineRevisions() { + String lengthyDeploymentSpec = + "<deployment version='1.0'>\n" + + " <instance id='alpha'>\n" + + " <test />\n" + + " <staging />\n" + + " <upgrade revision='latest' />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='beta'>\n" + + " <upgrade revision='separate' />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='gamma'>\n" + + " <upgrade revision='separate' />\n" + // TODO: change to new, even stricter policy. + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"; + var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); + var alpha = tester.newDeploymentContext("t", "a", "alpha"); + var beta = tester.newDeploymentContext("t", "a", "beta"); + var gamma = tester.newDeploymentContext("t", "a", "gamma"); + alpha.submit(appPackage).deploy(); + + // revision2 is submitted, and rolls through alpha. + var revision1 = alpha.lastSubmission(); + alpha.submit(appPackage); + var revision2 = alpha.lastSubmission(); + + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(testUsEast3); + assertEquals(Optional.empty(), alpha.instance().change().application()); + + // revision3 is submitted when revision2 is half-way. tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(version), app.instance().change()); + beta.runJob(productionUsEast3); + alpha.submit(appPackage); + var revision3 = alpha.lastSubmission(); + beta.runJob(testUsEast3); + assertEquals(Optional.empty(), beta.instance().change().application()); - app.runJob(productionUsEast3).runJob(productionUsWest1); - tester.triggerJobs(); + // revision3 is the target for alpha, beta is done, version1 is the target for gamma. + tester.outstandingChangeDeployer().run(); + assertEquals(revision3, alpha.instance().change().application()); + assertEquals(Optional.empty(), beta.instance().change().application()); + assertEquals(revision2, gamma.instance().change().application()); + + // revision3 rolls to beta, then a couple of new revisions are submitted to alpha, and the latter is the new target. + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(testUsEast3); + tester.outstandingChangeDeployer().run(); + assertEquals(Optional.empty(), alpha.instance().change().application()); + assertEquals(revision3, beta.instance().change().application()); + + // revision5 supersedes revision4 + alpha.submit(appPackage); + var revision4 = alpha.lastSubmission(); + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3); + alpha.submit(appPackage); + var revision5 = alpha.lastSubmission(); + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); + assertEquals(Optional.empty(), alpha.instance().change().application()); + assertEquals(revision3, beta.instance().change().application()); + + // revision6 rolls through alpha, and becomes the next target for beta + alpha.submit(appPackage); + var revision6 = alpha.lastSubmission(); + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3) + .runJob(testUsEast3); + beta.runJob(productionUsEast3).runJob(testUsEast3); + tester.outstandingChangeDeployer().run(); + assertEquals(Optional.empty(), alpha.instance().change().application()); + assertEquals(revision6, beta.instance().change().application()); + + // revision6 rolls through beta, but revision3 is the next target for the strictest revision policy, in gamma + alpha.jobAborted(stagingTest).runJob(stagingTest); + beta.runJob(productionUsEast3).runJob(testUsEast3); + gamma.runJob(productionUsEast3).runJob(testUsEast3); + tester.outstandingChangeDeployer().run(); + assertEquals(Optional.empty(), alpha.instance().change().application()); + assertEquals(Optional.empty(), beta.instance().change().application()); + // TODO: assertEquals(revision3, gamma.instance().change().application()); + } + + @Test + public void testVeryLengthyPipelineUpgrade() { + String lengthyDeploymentSpec = + "<deployment version='1.0'>\n" + + " <instance id='alpha'>\n" + + " <test />\n" + + " <staging />\n" + + " <upgrade rollout='simultaneous' />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='beta'>\n" + + " <upgrade rollout='simultaneous' />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='gamma'>\n" + + " <upgrade rollout='separate' />\n" + + " <prod>\n" + + " <region>us-east-3</region>\n" + + " <test>us-east-3</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"; + var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); + var alpha = tester.newDeploymentContext("t", "a", "alpha"); + var beta = tester.newDeploymentContext("t", "a", "beta"); + var gamma = tester.newDeploymentContext("t", "a", "gamma"); + alpha.submit(appPackage).deploy(); + + // A version releases, but when the first upgrade has gotten through alpha, beta, and gamma, a newer version has high confidence. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + var version2 = new Version("7.2"); + tester.controllerTester().upgradeSystem(version1); + + tester.upgrader().maintain(); + alpha.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsEast3).runJob(testUsEast3); + assertEquals(Change.empty(), alpha.instance().change()); + + tester.upgrader().maintain(); + beta.runJob(productionUsEast3); + tester.controllerTester().upgradeSystem(version2); + beta.runJob(testUsEast3); + assertEquals(Change.empty(), beta.instance().change()); + + tester.upgrader().maintain(); + assertEquals(Change.of(version2), alpha.instance().change()); + assertEquals(Change.empty(), beta.instance().change()); + assertEquals(Change.of(version1), gamma.instance().change()); + } + + @Test + public void testRevisionJoinsUpgradeWithLeadingRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("leading") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); + + // Revision starts rolling, and catches up. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + + // Upgrade got here first, and has triggered, but is now obsolete. + app.triggerJobs(); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); + + // Once staging tests verify the joint upgrade, the job is replaced with that. + app.runJob(stagingTest); + app.triggerJobs(); + app.jobAborted(productionUsEast3).runJob(productionUsEast3); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + + // Platform and revision now proceed together. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testRevisionPassesUpgradeWithSimultaneousRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("simultaneous") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); + + // Revision starts rolling, and catches up. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + + // Upgrade got here first, and has triggered, but is now obsolete. + app.triggerJobs(); + app.assertRunning(productionUsEast3); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); + + // Once staging tests verify the joint upgrade, the job is replaced with that. + app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest); + app.triggerJobs(); + app.jobAborted(productionUsEast3).runJob(productionUsEast3); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + + // Revision now proceeds alone. + app.triggerJobs(); + assertEquals(new Versions(version0, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + + // Upgrade follows. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision1), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); } @Test @@ -1102,7 +1681,7 @@ public class DeploymentTriggerTest { ZoneId.from("prod.cd-aws-us-east-1a")); tester.controllerTester() .setZones(zones, SystemName.cd) - .setRoutingMethod(zones, RoutingMethod.shared); + .setRoutingMethod(zones, RoutingMethod.sharedLayer4); tester.controllerTester().upgradeSystem(Version.fromString("6.1")); tester.controllerTester().computeVersionStatus(); var app = tester.newDeploymentContext(); @@ -1114,9 +1693,10 @@ public class DeploymentTriggerTest { tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) .abortJob(stagingTest) // Complete failing run. - .runJob(stagingTest) + .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. .runJob(productionCdAwsUsEast1a); + // Manually deploy to east again, then upgrade the system. app.runJob(productionCdUsEast1, cdPackage); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); @@ -1124,16 +1704,16 @@ public class DeploymentTriggerTest { // System and staging tests both require unknown versions, and are broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) - .jobAborted(systemTest) + .abortJob(systemTest) .jobAborted(stagingTest) - .runJob(systemTest) - .runJob(stagingTest) + .runJob(systemTest) // Run test for aws zone again. + .runJob(stagingTest) // Run test for aws zone again. .runJob(productionCdAwsUsEast1a); + // Deploy manually again, then submit new package. app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); - app.jobAborted(systemTest) - .runJob(systemTest); + app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) @@ -1145,7 +1725,7 @@ public class DeploymentTriggerTest { @Test public void testsInSeparateInstance() { String deploymentSpec = - "<deployment version='1.0'>\n" + + "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + " <instance id='canary'>\n" + " <upgrade policy='canary' />\n" + " <test />\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 061cc69fc26..d9b10ec933c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import org.junit.Before; import org.junit.Test; @@ -53,6 +54,9 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -336,6 +340,47 @@ public class InternalStepRunnerTest { } @Test + public void testCanBeReset() { + RunId id = app.startSystemTestTests(); + tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), info, "Not enough data!")); + tester.cloud().set(TesterCloud.Status.INCONCLUSIVE); + + long lastId1 = tester.jobs().details(id).get().lastId().getAsLong(); + Instant instant1 = tester.clock().instant(); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertEquals(running, tester.jobs().run(id).get().status()); + tester.cloud().clearLog(); + + // Test sleeps for a while. + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + tester.clock().advance(Duration.ofSeconds(899)); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + + tester.clock().advance(JobRunner.jobTimeout); + var testZone = JobType.systemTest.zone(tester.controller().system()); + tester.runner().run(); + app.flushDnsUpdates(); + tester.configServer().convergeServices(app.instanceId(), testZone); + tester.configServer().convergeServices(app.testerId().id(), testZone); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertTrue(tester.jobs().run(id).get().steps().get(Step.endTests).startTime().isPresent()); + + tester.cloud().set(TesterCloud.Status.SUCCESS); + long lastId2 = tester.jobs().details(id).get().lastId().getAsLong(); + tester.runner().run(); + assertEquals(success, tester.jobs().run(id).get().status()); + + assertTestLogEntries(id, Step.endTests, + new LogEntry(lastId1 + 1, Instant.ofEpochMilli(123), info, "Not enough data!"), + new LogEntry(lastId1 + 2, instant1, info, "Tests were inconclusive, and will run again in 15 minutes."), + new LogEntry(lastId2 + 1, tester.clock().instant(), info, "Tests completed successfully.")); + } + + @Test public void deployToDev() { ZoneId zone = JobType.devUsEast1.zone(system()); tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); @@ -482,7 +527,7 @@ public class InternalStepRunnerTest { tester.clock().advance(InternalStepRunner.Timeouts.of(system()).testerCertificate().plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(RunStatus.aborted, tester.jobs().run(id).get().status()); + assertEquals(RunStatus.error, tester.jobs().run(id).get().status()); } private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index b81b3ae5d66..74c06d7ca1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -11,15 +11,15 @@ import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClientMock; +import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; @@ -32,10 +32,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.horizon.MockHorizonClie import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; -import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopTenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; @@ -59,7 +57,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); - private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); @@ -116,11 +113,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public GlobalRoutingService globalRoutingService() { - return memoryGlobalRoutingService; - } - - @Override public MockMailer mailer() { return mockMailer; } @@ -279,10 +271,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return configServerMock; } - public MemoryGlobalRoutingService globalRoutingServiceMock() { - return memoryGlobalRoutingService; - } - public MockContactRetriever contactRetrieverMock() { return mockContactRetriever; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 23ab91aaf8c..a4c30cca29e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -52,26 +52,28 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry */ public ZoneRegistryMock(SystemName system) { this.system = system; - this.zones = system.isPublic() ? - List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-eu-west-1a")) : - List.of(ZoneApiMock.fromId("test.us-east-1"), - ZoneApiMock.fromId("staging.us-east-3"), - ZoneApiMock.fromId("dev.us-east-1"), - ZoneApiMock.fromId("dev.aws-us-east-2a"), - ZoneApiMock.fromId("perf.us-east-3"), - ZoneApiMock.fromId("prod.aws-us-east-1a"), - ZoneApiMock.fromId("prod.ap-northeast-1"), - ZoneApiMock.fromId("prod.ap-northeast-2"), - ZoneApiMock.fromId("prod.ap-southeast-1"), - ZoneApiMock.fromId("prod.us-east-3"), - ZoneApiMock.fromId("prod.us-west-1"), - ZoneApiMock.fromId("prod.us-central-1"), - ZoneApiMock.fromId("prod.eu-west-1")); - // All zones use a shared routing method by default - setRoutingMethod(this.zones, system.isPublic() ? RoutingMethod.exclusive : RoutingMethod.shared); + if (system.isPublic()) { + this.zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-eu-west-1a")); + setRoutingMethod(this.zones, RoutingMethod.exclusive); + } else { + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), + ZoneApiMock.fromId("dev.us-east-1"), + ZoneApiMock.fromId("dev.aws-us-east-2a"), + ZoneApiMock.fromId("perf.us-east-3"), + ZoneApiMock.fromId("prod.aws-us-east-1a"), + ZoneApiMock.fromId("prod.ap-northeast-1"), + ZoneApiMock.fromId("prod.ap-northeast-2"), + ZoneApiMock.fromId("prod.ap-southeast-1"), + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1")); + setRoutingMethod(this.zones, RoutingMethod.sharedLayer4); + } } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -117,18 +119,15 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) { - return setRoutingMethod(zone, List.of(routingMethods)); + return setRoutingMethod(zone, Set.of(routingMethods)); } public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) { - zones.forEach(zone -> setRoutingMethod(zone, List.of(routingMethods))); + zones.forEach(zone -> setRoutingMethod(zone, Set.of(routingMethods))); return this; } - public ZoneRegistryMock setRoutingMethod(ZoneApi zone, List<RoutingMethod> routingMethods) { - if (routingMethods.stream().distinct().count() != routingMethods.size()) { - throw new IllegalArgumentException("Routing methods must be distinct"); - } + private ZoneRegistryMock setRoutingMethod(ZoneApi zone, Set<RoutingMethod> routingMethods) { this.zoneRoutingMethods.put(zone, List.copyOf(routingMethods)); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index d6e43c07ec8..df2b462914e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -15,6 +15,7 @@ import org.junit.Test; import java.time.Duration; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -35,14 +36,16 @@ public class ArchiveAccessMaintainerTest { createTenantWithAccessRole(tester, "tenant2", tenant2role); ZoneId testZone = ZoneId.from("prod.aws-us-east-1c"); - tester.controller().archiveBucketDb().archiveUriFor(testZone, tenant1); + tester.controller().archiveBucketDb().archiveUriFor(testZone, tenant1, true); var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); - assertNull(archiveService.authorizedIamRoles.get(testBucket)); + assertNull(archiveService.authorizedIamRolesForBucket.get(testBucket)); + assertNull(archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); MockMetric metric = new MockMetric(); new ArchiveAccessMaintainer(tester.controller(), metric, Duration.ofMinutes(10)).maintain(); - assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRoles.get(testBucket)); + assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRolesForBucket.get(testBucket)); + assertEquals(Set.of(tenant1role), archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); var expected = Map.of("archive.bucketCount", tester.controller().zoneRegistry().zones().all().ids().stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 451991f9604..0a2f5d9a236 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -43,6 +43,9 @@ public class ArchiveUriUpdaterTest { // Initially we should not set any archive URIs as the archive service does not return any updater.maintain(); assertArchiveUris(Map.of(), zone); + // but the controller zone is always present + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), + ZoneId.from("prod", "controller")); // Archive service now has URI for tenant1, but tenant1 is not deployed in zone setBucketNameInService(Map.of(tenant1, "uri-1"), zone); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 0e365ffd135..7c4203d253c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -56,6 +56,22 @@ public class DeploymentIssueReporterTest { } @Test + public void nonProductionAppGetsNoIssues() { + tester.controllerTester().upgradeSystem(Version.fromString("6.2")); + var app = tester.newDeploymentContext("application", "tenant", "default"); + Contact contact = tester.controllerTester().serviceRegistry().contactRetrieverMock().contact(); + tester.controller().tenants().lockOrThrow(app.instanceId().tenant(), LockedTenant.Athenz.class, tenant -> + tester.controller().tenants().store(tenant.with(contact))); + + // app submits a package with no production deployments, and shall not receive issues. + app.submit(new ApplicationPackageBuilder().systemTest().stagingTest().build()).runJob(systemTest).failDeployment(stagingTest); + + // Advance to where deployment issues should be detected. + tester.clock().advance(maxFailureAge.plus(Duration.ofDays(1))); + assertFalse("No issues are produced for app.", issues.isOpenFor(app.application().id())); + } + + @Test public void testDeploymentFailureReporting() { tester.controllerTester().upgradeSystem(Version.fromString("6.2")); @@ -72,6 +88,7 @@ public class DeploymentIssueReporterTest { tester.controller().tenants().lockOrThrow(app3.instanceId().tenant(), LockedTenant.Athenz.class, tenant -> tester.controller().tenants().store(tenant.with(contact))); + // NOTE: All maintenance should be idempotent within a small enough time interval, so maintain is called twice in succession throughout. // app 1 fails staging tests. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index d97f1d58043..455e802e87b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -283,7 +283,7 @@ public class MetricsReporterTest { context.submit(applicationPackage).deploy(); reporter.maintain(); - assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 2, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.flushDnsUpdates(); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 8e90cfc69f3..acaa8b24d9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -30,42 +31,34 @@ public class OutstandingChangeDeployerTest { OutstandingChangeDeployer deployer = new OutstandingChangeDeployer(tester.controller(), Duration.ofMinutes(10)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") + .upgradeRevision("separate") .build(); - var app1 = tester.newDeploymentContext("tenant", "app1", "default").submit(applicationPackage).deploy(); + var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); Version version = new Version(6, 2); - tester.deploymentTrigger().triggerChange(app1.instanceId(), Change.of(version)); - tester.deploymentTrigger().triggerReadyJobs(); + tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version)); + assertEquals(Change.of(version), app.instance().change()); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); - assertEquals(Change.of(version), app1.instance().change()); - assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); + app.submit(applicationPackage); + Optional<ApplicationVersion> revision = app.lastSubmission(); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong()); - app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d"))); + app.submit(applicationPackage); + Optional<ApplicationVersion> outstanding = app.lastSubmission(); + assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); - assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); - app1.assertRunning(JobType.systemTest); - app1.assertRunning(JobType.stagingTest); - assertEquals(2, tester.jobs().active().size()); + tester.outstandingChangeDeployer().run(); + assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - deployer.maintain(); - tester.triggerJobs(); - assertEquals("No effect as job is in progress", 2, tester.jobs().active().size()); - assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); - - app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1) - .runJob(JobType.stagingTest).runJob(JobType.systemTest); - assertEquals("Upgrade done", 0, tester.jobs().active().size()); - - deployer.maintain(); - tester.triggerJobs(); - assertEquals("1.0.2-cafed00d", app1.instance().change().application().get().id()); - List<Run> runs = tester.jobs().active(); - assertEquals(1, runs.size()); - app1.assertRunning(JobType.productionUsWest1); - assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); + app.deploy(); + tester.outstandingChangeDeployer().run(); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(outstanding.get()), app.instance().change()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index f9a976fcadc..265dedec8d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -35,6 +35,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PIN; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -167,7 +168,6 @@ public class UpgraderTest { // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change default0.submit(applicationPackage("default")); - default0.triggerJobs().jobAborted(stagingTest); default0.deploy(); tester.controllerTester().computeVersionStatus(); @@ -534,7 +534,7 @@ public class UpgraderTest { } @Test - public void testBlockVersionChangeHalfwayThoughThenNewRevision() { + public void testBlockVersionChangeHalfwayThroughThenNewRevision() { // Friday, 16:00 tester.at(Instant.parse("2017-09-29T16:00:00.00Z")); @@ -542,7 +542,7 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(version); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - // Block upgrades on weekends and ouside working hours + // Block upgrades on weekends and outside working hours .blockChange(false, true, "mon-fri", "00-09,17-23", "UTC") .blockChange(false, true, "sat-sun", "00-23", "UTC") .region("us-west-1") @@ -570,19 +570,20 @@ public class UpgraderTest { // A new revision is submitted and starts rolling out. app.submit(applicationPackage); - // production-us-central-1 isn't triggered, as the revision + platform is the new change to roll out. + // production-us-central-1 is re-triggered with upgrade until revision catches up. tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(3, tester.jobs().active().size()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); // us-central-1 has an older version, and needs a new staging test to begin. - app.runJob(stagingTest); + app.runJob(stagingTest).triggerJobs().jobAborted(productionUsCentral1); // Retry will include revision. tester.triggerJobs(); // Triggers us-central-1 before platform upgrade is cancelled. - // A new version is also released, cancelling the upgrade, since it is failing on a now outdated version. + // A new version is also released, and someone cancels the upgrade, suspecting it is faulty. tester.clock().advance(Duration.ofHours(17)); version = Version.fromString("6.4"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); + tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); // us-central-1 succeeds upgrade to 6.3, with the revision, but us-east-3 wants to proceed with only the revision change. app.runJob(productionUsCentral1); @@ -799,8 +800,10 @@ public class UpgraderTest { app.instance().change().application().get().id().equals(applicationVersion)); // Deployment completes - app.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1).runJob(productionUsEast3); - assertTrue("All jobs consumed", tester.jobs().active().isEmpty()); + app.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsWest1) + .runJob(productionUsEast3); + assertEquals("All jobs consumed", List.of(), tester.jobs().active()); for (Deployment deployment : app.instance().deployments().values()) { assertEquals(version, deployment.version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index b33f8f6f7e7..e509a199c82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -42,6 +42,8 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import static org.junit.Assert.assertEquals; @@ -89,12 +91,14 @@ public class ApplicationSerializerTest { Optional.of(Instant.ofEpochMilli(666)), Optional.empty(), Optional.of("best commit"), - true); + true, + Optional.of("hash1")); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); ApplicationVersion applicationVersion2 = ApplicationVersion .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b", Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); + SortedSet<ApplicationVersion> versions = new TreeSet<>(Set.of(applicationVersion2)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); @@ -120,13 +124,15 @@ public class ApplicationSerializerTest { Map.of(JobType.systemTest, Instant.ofEpochMilli(333)), List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of("us-west-1"))), rotationStatus, - Change.of(new Version("6.1"))), + Change.of(new Version("6.1")), + Optional.of(applicationVersion2)), new Instance(id3, List.of(), Map.of(), List.of(), RotationStatus.EMPTY, - Change.of(Version.fromString("6.7")).withPin())); + Change.of(Version.fromString("6.7")).withPin(), + Optional.empty())); Application original = new Application(TenantAndApplicationId.from(id1), Instant.now().truncatedTo(ChronoUnit.MILLIS), @@ -140,6 +146,7 @@ public class ApplicationSerializerTest { Set.of(publicKey, otherPublicKey), projectId, Optional.of(applicationVersion1), + versions, instances); Application serialized = APPLICATION_SERIALIZER.fromSlime(SlimeUtils.toJsonBytes(APPLICATION_SERIALIZER.toSlime(original))); @@ -151,6 +158,10 @@ public class ApplicationSerializerTest { assertEquals(original.latestVersion().get().buildTime(), serialized.latestVersion().get().buildTime()); assertEquals(original.latestVersion().get().sourceUrl(), serialized.latestVersion().get().sourceUrl()); assertEquals(original.latestVersion().get().commit(), serialized.latestVersion().get().commit()); + assertEquals(original.latestVersion().get().bundleHash(), serialized.latestVersion().get().bundleHash()); + assertEquals(original.versions(), serialized.versions()); + assertEquals(original.versions(), serialized.versions()); + assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); @@ -182,6 +193,9 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change()); assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change()); + assertEquals(original.require(id1.instance()).latestDeployed(), serialized.require(id1.instance()).latestDeployed()); + assertEquals(original.require(id3.instance()).latestDeployed(), serialized.require(id3.instance()).latestDeployed()); + // Test metrics assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index eca2b17a5f1..f4f52b20325 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -91,7 +91,8 @@ public class RunSerializerTest { Optional.of(Instant.ofEpochMilli(100)), Optional.empty(), Optional.empty(), - true); + true, + Optional.empty()); assertEquals(applicationVersion, run.versions().targetApplication()); assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail()); assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 7b9131a38dd..54cde2bacef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -4,6 +4,7 @@ "type": "production-us-east-3", "number": 112358, "start": 1196676930000, + "sleepUntil": 1196676930100, "status": "running", "lastTestRecord": 3, "lastVespaLogTimestamp": 1196676930000432, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index a6d061e7c80..2fd8026319b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -92,15 +92,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { Request request = new Request("http://localhost:8080" + path, data, method, principal); request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, roles)); if (user != null) { - Map<String, String> userAttributes = new HashMap<>(); - userAttributes.put("email", user.email()); - if (user.name() != null) - userAttributes.put("name", user.name()); - if (user.nickname() != null) - userAttributes.put("nickname", user.nickname()); - if (user.picture() != null) - userAttributes.put("picture", user.picture()); - request.getAttributes().put(User.ATTRIBUTE_NAME, Map.copyOf(userAttributes)); + request.getAttributes().put(User.ATTRIBUTE_NAME, user); } request.getHeaders().put("Content-Type", contentType); return request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index a4de6ab7700..92055c85a53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -238,7 +238,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Test public void create_application_on_deploy() { var application = ApplicationName.from("unique"); - var applicationPackage = new ApplicationPackageBuilder().build(); + var applicationPackage = new ApplicationPackageBuilder().withoutAthenzIdentity().build(); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); @@ -256,6 +256,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var application = ApplicationName.from("unique"); var applicationPackage = new ApplicationPackageBuilder() .trustDefaultCertificate() + .withoutAthenzIdentity() .build(); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); @@ -273,6 +274,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("default") .region("aws-us-east-1c"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 8eaa190e9fa..516911b3c7b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -26,8 +26,6 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; @@ -126,6 +124,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final String accessDenied = "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}"; private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("default") .globalServiceId("foo") .region("us-central-1") @@ -135,6 +134,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); private static final ApplicationPackage applicationPackageInstance1 = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1") .globalServiceId("foo") .region("us-central-1") @@ -338,6 +338,7 @@ public class ApplicationApiTest extends ControllerContainerTest { app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsCentral1); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1") .globalServiceId("foo") .region("us-west-1") @@ -448,7 +449,6 @@ public class ApplicationApiTest extends ControllerContainerTest { deploymentTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); deploymentTester.controllerTester().computeVersionStatus(); setDeploymentMaintainedInfo(); - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-central-1")); // GET tenant application deployments tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -817,6 +817,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Sixth attempt has a multi-instance deployment spec, and is accepted. ApplicationPackage multiInstanceSpec = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1,instance2") .region("us-central-1") .parallel("us-west-1", "us-east-3") @@ -949,7 +950,6 @@ public class ApplicationApiTest extends ControllerContainerTest { 404); // GET global rotation status - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), new File("global-rotation.json")); @@ -1001,10 +1001,6 @@ public class ApplicationApiTest extends ControllerContainerTest { var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); app.submit(applicationPackage).deploy(); - setZoneInRotation("rotation-fqdn-2", ZoneId.from("prod", "us-west-1")); - setZoneInRotation("rotation-fqdn-2", ZoneId.from("prod", "us-east-3")); - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "eu-west-1")); - // GET global rotation status without specifying endpointId fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), @@ -1528,7 +1524,7 @@ public class ApplicationApiTest extends ControllerContainerTest { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), - List.of(RoutingMethod.exclusive, RoutingMethod.shared)); + RoutingMethod.exclusive); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) .instances("instance1") @@ -1547,15 +1543,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("deployment-with-routing-policy.json")); - // GET deployment including legacy endpoints - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/instance1", GET) - .userIdentity(USER_ID) - .properties(Map.of("includeLegacyEndpoints", "true")), - new File("deployment-with-routing-policy-legacy.json")); - - // Hide shared endpoints - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.HIDE_SHARED_ROUTING_ENDPOINT.id(), true); - // GET deployment tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/instance1", GET) .userIdentity(USER_ID), @@ -1566,8 +1553,7 @@ public class ApplicationApiTest extends ControllerContainerTest { public void support_access() { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); - deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), - List.of(RoutingMethod.exclusive, RoutingMethod.shared)); + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), RoutingMethod.exclusive); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) @@ -1817,11 +1803,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } } - private void setZoneInRotation(String rotationName, ZoneId zone) { - tester.serviceRegistry().globalRoutingServiceMock().setStatus(rotationName, zone, com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus.IN); - //new RotationStatusUpdater(tester.controller(), Duration.ofDays(1)).run(); - } - private void updateContactInformation() { Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 2ae755ac8fe..4935ab22586 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -192,7 +192,7 @@ public class JobControllerApiHandlerHelperTest { private void compare(HttpResponse response, String expected) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - JsonTestHelper.assertJsonEquals(expected, baos.toString()); + JsonTestHelper.assertJsonEquals(baos.toString(), expected); } private void assertResponse(HttpResponse response, String fileName) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java index 5bf22ede19d..2c81b1a7fd8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; -import com.google.inject.Key; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.Container; import com.yahoo.jdisc.Request; @@ -72,9 +71,6 @@ public class MultipartParserTest { public RequestHandler resolveHandler(Request request) { return null; } @Override - public <T> T getInstance(Key<T> key) { return null; } - - @Override public <T> T getInstance(Class<T> aClass) { return null; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index a214969485d..fd4093ca332 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -37,7 +37,6 @@ }, "status": "complete", "quota": "(ignore)", - "archiveUri": "s3://bucketName/scoober/", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index b8c48eb3d0c..6223cbbb2b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -1223,5 +1223,40 @@ } ] } + ], + "builds": [ + { + "hash": "1.0.3-commit1", + "build": 3, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + { + "hash": "1.0.2-commit1", + "build": 2, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + { + "hash": "1.0.1-commit1", + "build": 1, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index abe3d4100d9..e3fb8eec9c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -21,7 +21,7 @@ "platform": { "platform": "6.1.0", "at": "(ignore)", - "upgrade": false, + "upgrade": true, "blockers": [] }, "application": { @@ -536,7 +536,7 @@ "platform": { "platform": "6.1.0", "at": "(ignore)", - "upgrade": false, + "upgrade": true, "blockers": [] }, "application": { @@ -547,7 +547,7 @@ "commit": "commit1" }, "at": 1000, - "upgrade": false, + "upgrade": true, "blockers": [] } } @@ -630,6 +630,52 @@ ], "runs": [] } + ], + "builds": [ + { + "hash": "1.0.4-commit1", + "build": 4, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + { + "hash": "1.0.3-commit1", + "build": 3, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + { + "hash": "1.0.2-commit1", + "build": 2, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + { + "hash": "1.0.1-commit1", + "build": 1, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json deleted file mode 100644 index eb508b2459e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "environment": "prod", - "region": "us-west-1", - "endpoints": [ - { - "cluster": "default", - "tls": true, - "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", - "scope": "zone", - "routingMethod": "exclusive", - "legacy": false - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false - }, - { - "cluster": "default", - "tls": false, - "url": "http://instance1.application1.tenant1.us-west-1.prod.vespa.yahooapis.com:4080/", - "scope": "zone", - "routingMethod": "shared", - "legacy": true - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa.yahooapis.com:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": true - } - ], - "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", - "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", - "version": "6.1.0", - "revision": "1.0.1-commit1", - "deployTimeEpochMs": "(ignore)", - "screwdriverId": "1000", - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1", - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "quota": "(ignore)", - "activity": {}, - "metrics": { - "queriesPerSecond": 0.0, - "writesPerSecond": 0.0, - "documentCount": 0.0, - "queryLatencyMillis": 0.0, - "writeLatencyMillis": 0.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 97ac87fb5a0..4457bede34e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -12,14 +12,6 @@ "scope": "zone", "routingMethod": "exclusive", "legacy": false - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false } ], "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index ab2a3bf945c..621617f1b1c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -8,17 +8,17 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.us-central-1.vespa.oath.cloud/", "scope": "zone", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false }, { "cluster": "foo", "tls": true, - "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.global.vespa.oath.cloud/", "scope": "global", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index 45df6aad67c..175c45eb2cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -11,7 +11,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.dev.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.dev.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index f2f8e14f093..2ca520c0122 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -8,9 +8,9 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.us-east-1.dev.vespa.oath.cloud/", "scope": "zone", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 62ad3a2db7e..d9ec8e4dfef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -11,17 +11,17 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.us-central-1.vespa.oath.cloud/", "scope": "zone", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false }, { "cluster": "foo", "tls": true, - "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.global.vespa.oath.cloud/", "scope": "global", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false }, { @@ -33,7 +33,7 @@ "legacy": false } ], - "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 0525f059dd0..6b1d48f4a08 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -117,16 +117,26 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 14503000, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 14503000, + "type": "debug", + "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 14503000, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -150,7 +160,7 @@ } ] }, - "lastId": 27, + "lastId": 29, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 7ee3952a8b5..377b8c6ed69 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -137,91 +137,151 @@ { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -237,7 +297,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://instance1.application1.tenant1.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": "(ignore)", @@ -264,7 +324,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://instance1.application1.tenant1.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": "(ignore)", @@ -294,7 +354,7 @@ } ] }, - "lastId": 54, + "lastId": 66, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 7f59eaf75c2..66173ec4976 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -132,91 +132,151 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -232,7 +292,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, @@ -259,7 +319,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, @@ -289,7 +349,7 @@ } ] }, - "lastId": 54, + "lastId": 66, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json index 0632ab7a67b..3a5e6dc5dc3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json @@ -5,18 +5,18 @@ "isCI": false, "endpoints": { "dev.us-east-1": [ - "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" + "https://my-user.application1.tenant1.us-east-1.dev.vespa.oath.cloud/" ], "prod.us-central-1": [ - "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" + "https://application1.tenant1.us-central-1.vespa.oath.cloud/" ] }, "zoneEndpoints": { "dev.us-east-1": { - "default": "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" + "default": "https://my-user.application1.tenant1.us-east-1.dev.vespa.oath.cloud/" }, "prod.us-central-1": { - "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" + "default": "https://application1.tenant1.us-central-1.vespa.oath.cloud/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json index c81ed767239..0a9236655ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -5,12 +5,12 @@ "isCI": false, "endpoints": { "prod.us-central-1": [ - "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" + "https://application1.tenant1.us-central-1.vespa.oath.cloud/" ] }, "zoneEndpoints": { "prod.us-central-1": { - "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" + "default": "https://application1.tenant1.us-central-1.vespa.oath.cloud/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 2edf1867fd3..0dad88e645b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -13,6 +13,9 @@ "name": "ArchiveUriUpdater" }, { + "name": "BillingDatabaseMaintainer" + }, + { "name": "ChangeRequestMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index 9e17b44c9a6..eab3a37a9c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -75,6 +76,18 @@ public class ControllerAuthorizationFilterTest { assertIsAllowed(invokeFilter(filter, createRequest(Method.GET, "/zone/v1/path", securityContext))); } + @Test + public void hostedDeveloper() { + ControllerTester tester = new ControllerTester(); + TenantName tenantName = TenantName.defaultName(); + SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.hostedDeveloper(tenantName))); + + ControllerAuthorizationFilter filter = createFilter(tester); + assertIsAllowed(invokeFilter(filter, createRequest(Method.POST, "/application/v4/tenant/" + tenantName.value() + "/application/app/instance/default/environment/dev/region/region/deploy", securityContext))); + assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/application/v4/tenant/" + tenantName.value() + "/application/app/instance/default/environment/prod/region/region/deploy", securityContext))); + assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/application/v4/tenant/" + tenantName.value() + "/application/app/submit", securityContext))); + } + private static void assertIsAllowed(Optional<AuthorizationResponse> response) { assertFalse("Expected no response from filter, but got \"" + response.map(r -> r.message + "\" (" + r.statusCode + ")").orElse(""), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 5b2fabcaff8..0b128ebf7a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -258,7 +258,7 @@ public class RoutingApiTest extends ControllerContainerTest { // One shared and one exclusive zone deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), - RoutingMethod.shared); + RoutingMethod.sharedLayer4); deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(eastZone), RoutingMethod.exclusive); @@ -273,7 +273,7 @@ public class RoutingApiTest extends ControllerContainerTest { .build(); context.submit(applicationPackage).deploy(); - // GET status for zone using shared routing + // GET status for zone using sharedLayer4 routing tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-initial.json")); @@ -331,4 +331,5 @@ public class RoutingApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/endpoint", "", Request.Method.GET), new File("endpoint/endpoints.json")); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json index f78f913cb7e..75369a19ea7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json @@ -2,13 +2,13 @@ "endpoints": [ { "name": "default", - "dnsName": "a1--t1.global.vespa.oath.cloud", - "routingMethod": "shared", + "dnsName": "a1.t1.global.vespa.oath.cloud", + "routingMethod": "sharedLayer4", "cluster": "default", "scope": "global", "zones": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-east-3", @@ -17,7 +17,7 @@ "changedAt": 1497618757000 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-west-1", @@ -28,4 +28,4 @@ ] } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json index e0b0e5e9b7a..06fb2b92c53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-east-3", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json index f0dd0b7310d..1711bb1f856 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json @@ -1,7 +1,7 @@ { "zones": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "test", "region": "us-east-1", "status": "in", @@ -9,7 +9,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "staging", "region": "us-east-3", "status": "in", @@ -17,7 +17,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "dev", "region": "us-east-1", "status": "in", @@ -25,7 +25,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "dev", "region": "aws-us-east-2a", "status": "in", @@ -33,7 +33,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "perf", "region": "us-east-3", "status": "in", @@ -41,7 +41,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "aws-us-east-1a", "status": "in", @@ -49,7 +49,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "ap-northeast-1", "status": "in", @@ -57,7 +57,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "ap-northeast-2", "status": "in", @@ -65,7 +65,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "ap-southeast-1", "status": "in", @@ -73,7 +73,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-east-3", "status": "in", @@ -81,7 +81,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-west-1", "status": "in", @@ -89,7 +89,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-central-1", "status": "in", @@ -97,7 +97,7 @@ "changedAt": 0 }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "eu-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json index 1ee4e1b82ba..5de12d9b1ec 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-east-3", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a1:default", "environment": "prod", "region": "us-west-1", @@ -19,7 +19,7 @@ "changedAt": "(ignore)" }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a2:default", "environment": "prod", "region": "us-east-3", @@ -28,7 +28,7 @@ "changedAt": "(ignore)" }, { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "t1:a2:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json index 5b15b72752c..4eb51c1e907 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json index 90b2317c1b3..615ce4b4a6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json index 85e345c01d0..816bc810048 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json index eb06e9ee11d..8460cc5ec8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json @@ -1,5 +1,5 @@ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json index eb06e9ee11d..8460cc5ec8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json @@ -1,5 +1,5 @@ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json index 440b80bc4d0..88fddcbd955 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json @@ -1,5 +1,5 @@ { - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "environment": "prod", "region": "us-west-1", "status": "out", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 537f6c48bdf..a93e9f55e30 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -67,9 +67,16 @@ public class UserApiTest extends ControllerContainerCloudTest { tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .roles(operator) .principal("administrator@tenant") + .user(new User("administrator@tenant", "administrator", "admin", "picture")) .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); + // GET at tenant/info with contact information. + tester.assertResponse(request("/application/v4/tenant/my-tenant/info") + .roles(operator) + .principal("administrator@tenant"), + new File("tenant-info-after-created.json")); + // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), accessDenied, 403); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json new file mode 100644 index 00000000000..942b5c1db45 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json @@ -0,0 +1,8 @@ +{ + "name": "", + "email": "", + "website":"", + "invoiceEmail":"", + "contactName": "administrator", + "contactEmail": "administrator@tenant" +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 7cc1a51a114..54585767d51 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -1,6 +1,7 @@ { "tenant": "my-tenant", "type": "CLOUD", + "creator": "administrator@tenant", "pemDeveloperKeys": [ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 1662484ade8..1cd2fb41263 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -1,6 +1,7 @@ { "tenant": "my-tenant", "type": "CLOUD", + "creator": "administrator@tenant", "pemDeveloperKeys": [ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 9a56123e8e3..d7847da2404 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -60,8 +60,8 @@ public class RotationRepositoryTest { Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); + assertEquals(URI.create("https://app1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).direct().first().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), @@ -151,13 +151,13 @@ public class RotationRepositoryTest { ZoneApiMock.fromId("prod.cd-us-west-1")); tester.controllerTester().zoneRegistry() .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.shared) + .setRoutingMethod(zones, RoutingMethod.sharedLayer4) .setSystemName(SystemName.cd); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage).deploy(); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); - assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", + assertEquals("https://cd.app2.tenant2.global.vespa.oath.cloud/", tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); } @@ -175,10 +175,10 @@ public class RotationRepositoryTest { var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance1.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).direct().first().get().url()); + assertEquals(URI.create("https://instance2.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).direct().first().get().url()); } @Test @@ -197,10 +197,10 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance1.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).direct().first().get().url()); + assertEquals(URI.create("https://instance2.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).direct().first().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip Binary files differnew file mode 100644 index 00000000000..90d52524fb3 --- /dev/null +++ b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip diff --git a/controller-server/src/test/resources/application-packages/changed-services-xml.zip b/controller-server/src/test/resources/application-packages/changed-services-xml.zip Binary files differnew file mode 100644 index 00000000000..3051d27836a --- /dev/null +++ b/controller-server/src/test/resources/application-packages/changed-services-xml.zip diff --git a/controller-server/src/test/resources/application-packages/original.zip b/controller-server/src/test/resources/application-packages/original.zip Binary files differnew file mode 100644 index 00000000000..4cf2ffa7c46 --- /dev/null +++ b/controller-server/src/test/resources/application-packages/original.zip diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json index 48bf4792176..7b91b4930a1 100644 --- a/controller-server/src/test/resources/testConfig.json +++ b/controller-server/src/test/resources/testConfig.json @@ -5,12 +5,12 @@ "isCI": true, "endpoints": { "test.aws-us-east-1c": [ - "https://ai--default--default.global.vespa.oath.cloud/" + "https://ai.default.default.global.vespa.oath.cloud/" ] }, "zoneEndpoints": { "test.aws-us-east-1c": { - "ai": "https://ai--default--default.global.vespa.oath.cloud/" + "ai": "https://ai.default.default.global.vespa.oath.cloud/" } }, "clusters": { diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 0bada8160be..1432ea7164c 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -18,7 +18,7 @@ endfunction() function(setup_vespa_default_build_settings_rhel_8) message("-- Setting up default build settings for rhel 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) @@ -68,18 +68,6 @@ function(setup_vespa_default_build_settings_darwin) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_fedora_32) - message("-- Setting up default build settings for fedora 32") - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) -endfunction() - -function(setup_vespa_default_build_settings_fedora_33) - message("-- Setting up default build settings for fedora 33") - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) -endfunction() - function(setup_vespa_default_build_settings_fedora_34) message("-- Setting up default build settings for fedora 34") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) @@ -212,10 +200,6 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_almalinux_8_5() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") - setup_vespa_default_build_settings_fedora_32() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 33") - setup_vespa_default_build_settings_fedora_33() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 34") setup_vespa_default_build_settings_fedora_34() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 35") diff --git a/dist/vespa.spec b/dist/vespa.spec index 724cffcc7f1..74ef6a6f8fc 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -51,6 +51,7 @@ BuildRequires: maven BuildRequires: python3-pytest %else BuildRequires: devtoolset-11-gcc-c++ +BuildRequires: devtoolset-11-libasan-devel BuildRequires: devtoolset-11-libatomic-devel BuildRequires: devtoolset-11-binutils BuildRequires: rh-maven35 @@ -66,11 +67,13 @@ BuildRequires: python3-devel %if 0%{?_centos_stream} BuildRequires: gcc-toolset-11-gcc-c++ BuildRequires: gcc-toolset-11-binutils +BuildRequires: gcc-toolset-11-libasan-devel BuildRequires: gcc-toolset-11-libatomic-devel %define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %else BuildRequires: gcc-toolset-11-gcc-c++ BuildRequires: gcc-toolset-11-binutils +BuildRequires: gcc-toolset-11-libasan-devel BuildRequires: gcc-toolset-11-libatomic-devel %define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %endif @@ -90,6 +93,8 @@ BuildRequires: glibc-langpack-en %endif %if 0%{?fedora} BuildRequires: gcc-c++ +BuildRequires: libasan +BuildRequires: libasan-static BuildRequires: libatomic BuildRequires: pybind11-devel BuildRequires: python3-pytest @@ -124,7 +129,7 @@ BuildRequires: (llvm-devel >= 13.0.0 and llvm-devel < 14) BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) %endif %else -BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) +BuildRequires: (llvm-devel >= 12.0.1 and llvm-devel < 13) %endif BuildRequires: vespa-boost-devel >= 1.76.0-1 BuildRequires: vespa-openssl-devel >= 1.1.1l-1 @@ -156,20 +161,6 @@ BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 -%if 0%{?fc32} -BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 10.0.0 -BuildRequires: boost-devel >= 1.69 -BuildRequires: gtest-devel -BuildRequires: gmock-devel -%endif -%if 0%{?fc33} -BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 11.0.0 -BuildRequires: boost-devel >= 1.73 -BuildRequires: gtest-devel -BuildRequires: gmock-devel -%endif %if 0%{?fc34} BuildRequires: protobuf-devel BuildRequires: llvm-devel >= 12.0.0 @@ -302,7 +293,7 @@ Requires: vespa-gtest = 1.11.0 %define _vespa_llvm_version 12 %endif %else -%define _vespa_llvm_version 10 +%define _vespa_llvm_version 12 %endif Requires: vespa-gtest = 1.11.0 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 @@ -316,12 +307,6 @@ Requires: gtest %endif %if 0%{?fedora} Requires: gtest -%if 0%{?fc32} -%define _vespa_llvm_version 10 -%endif -%if 0%{?fc33} -%define _vespa_llvm_version 11 -%endif %if 0%{?fc34} %define _vespa_llvm_version 12 %endif @@ -432,7 +417,7 @@ Requires: (llvm-libs >= 13.0.0 and llvm-libs < 14) Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) %endif %else -Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) +Requires: (llvm-libs >= 12.0.1 and llvm-libs < 13) %endif Requires: vespa-protobuf = 3.19.1 %endif @@ -442,12 +427,6 @@ Requires: protobuf %endif %if 0%{?fedora} Requires: protobuf -%if 0%{?fc32} -Requires: llvm-libs >= 10.0.0 -%endif -%if 0%{?fc33} -Requires: llvm-libs >= 11.0.0 -%endif %if 0%{?fc34} Requires: llvm-libs >= 12.0.0 %endif diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java index 32f28ba1294..8cb78bd718f 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java @@ -30,10 +30,10 @@ import java.util.logging.Logger; * * @author bratseth */ -//TODO Vespa 8 This class and a lot of other in this package should not be part of PublicAPI +//TODO: Vespa 8: This class and a lot of other in this package should not be part of PublicAPI public class DocprocService extends AbstractComponent { - private static Logger log = Logger.getLogger(DocprocService.class.getName()); + private static final Logger log = Logger.getLogger(DocprocService.class.getName()); private volatile DocprocExecutor executor; /** The processings currently in progress at this service */ diff --git a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java index 934e1a281e5..877ecf52789 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java @@ -48,10 +48,10 @@ public abstract class DocumentProcessor extends ChainedComponent { private Map<Pair<String,String>, String> fieldMap = new HashMap<>(); /** For a doc type, the actual field name mapping to do */ - // TODO how to flush this when reconfig of schemamapping? Must solve - private Map<String, Map<String, String>> docMapCache = new HashMap<>(); + // TODO: How to flush this when reconfig of schemamapping? + private final Map<String, Map<String, String>> docMapCache = new HashMap<>(); - final boolean hasAnnotations; + private final boolean hasAnnotations; public DocumentProcessor() { hasAnnotations = getClass().getAnnotation(Accesses.class) != null; @@ -70,6 +70,34 @@ public abstract class DocumentProcessor extends ChainedComponent { */ public abstract Progress process(Processing processing); + /** Sets the schema map for field names */ + public void setFieldMap(Map<Pair<String, String>, String> fieldMap) { + this.fieldMap = fieldMap; + + } + + /** Schema map for field names (doctype,from)→to */ + public Map<Pair<String, String>, String> getFieldMap() { + return fieldMap; + } + + public Map<String, String> getDocMap(String docType) { + Map<String, String> cached = docMapCache.get(docType); + if (cached!=null) { + return cached; + } + Map<String, String> ret = new HashMap<>(); + for (Entry<Pair<String, String>, String> e : fieldMap.entrySet()) { + // Remember to include tuple if doctype is unset in mapping + if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst()==null || "".equals(e.getKey().getFirst())) { + ret.put(e.getKey().getSecond(), e.getValue()); + } + } + docMapCache.put(docType, ret); + return ret; + } + + @Override public String toString() { return "processor " + getId().stringValue(); } @@ -99,7 +127,7 @@ public abstract class DocumentProcessor extends ChainedComponent { */ public static final Progress PERMANENT_FAILURE = new Progress("permanent_failure"); - private String name; + private final String name; private Optional<String> reason = Optional.empty(); @@ -120,6 +148,7 @@ public abstract class DocumentProcessor extends ChainedComponent { return new Progress(this.name, reason); } + @Override public String toString() { return name; } @@ -128,16 +157,20 @@ public abstract class DocumentProcessor extends ChainedComponent { return reason; } + @Override public boolean equals(Object object) { return object instanceof Progress && ((Progress) object).name.equals(this.name); } + @Override public int hashCode() { return name.hashCode(); } + } public static final class LaterProgress extends Progress { + private final long delay; public static final long DEFAULT_LATER_DELAY = 20; //ms @@ -153,33 +186,7 @@ public abstract class DocumentProcessor extends ChainedComponent { public long getDelay() { return delay; } - } - - /** Sets the schema map for field names */ - public void setFieldMap(Map<Pair<String, String>, String> fieldMap) { - this.fieldMap = fieldMap; - - } - /** Schema map for field names (doctype,from)→to */ - public Map<Pair<String, String>, String> getFieldMap() { - return fieldMap; - } - - public Map<String, String> getDocMap(String docType) { - Map<String, String> cached = docMapCache.get(docType); - if (cached!=null) { - return cached; - } - Map<String, String> ret = new HashMap<>(); - for (Entry<Pair<String, String>, String> e : fieldMap.entrySet()) { - // Remember to include tuple if doctype is unset in mapping - if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst()==null || "".equals(e.getKey().getFirst())) { - ret.put(e.getKey().getSecond(), e.getValue()); - } - } - docMapCache.put(docType, ret); - return ret; } } diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index d2583fc5a6d..616ad3c9241 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -20,23 +20,23 @@ import java.util.Map; */ public class Processing { - /** The name of the service which owns this processing. Null is the same as "default" */ + /** The name of the service which owns this processing. Null is the same as "default". */ private String service = null; - /** The processors to call the next work is done on this processing */ + /** The processors to call the next work is done on this processing. */ private CallStack callStack = null; - /** The collection of documents or document updates processed by this. This is never null */ + /** The collection of documents or document updates processed by this. This is never null. */ private final List<DocumentOperation> documentOperations; /** * Documents or document updates which should be added to <code>documents</code> before * the next access, or null if documents or document updates have never been added to - * this processing + * this processing. */ private List<DocumentOperation> documentsToAdd = null; - /** The processing context variables */ + /** The processing context variables. */ private Map<String, Object> context = null; /** The endpoint of this processing. */ @@ -113,6 +113,7 @@ public class Processing { } /** + * Creates a Processing from a list of operations. * * @param service the unique name of the service processing this * @param documentsAndUpdates the document operation list. This <b>transfers ownership</b> of this list @@ -121,7 +122,9 @@ public class Processing { * This <b>transfers ownership</b> of this structure * to this class. The caller <i>must not</i> modify it */ - public static Processing createProcessingFromDocumentOperations(String service, List<DocumentOperation> documentsAndUpdates, CallStack callStack) { + public static Processing createProcessingFromDocumentOperations(String service, + List<DocumentOperation> documentsAndUpdates, + CallStack callStack) { return new Processing(service, documentsAndUpdates, callStack, null, false); } @@ -245,6 +248,15 @@ public class Processing { this.callStack = callStack; } + List<DocumentOperation> getOnceOperationsToBeProcessed() { + if (operationsGotten) + return Collections.emptyList(); + + operationsGotten = true; + return getDocumentOperations(); + } + + @Override public String toString() { String previousCall = ""; if (callStack != null) { @@ -266,11 +278,4 @@ public class Processing { } } - List<DocumentOperation> getOnceOperationsToBeProcessed() { - if (operationsGotten) - return Collections.emptyList(); - - operationsGotten = true; - return getDocumentOperations(); - } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index acbedaf1156..2affcf12809 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -161,7 +161,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { String serviceName = requestContext.getServiceName(); DocprocService service = docprocServiceRegistry.getComponent(serviceName); - //No need to enqueue a task if the docproc chain is empty, just forward requestContext + // No need to enqueue a task if the docproc chain is empty, just forward requestContext if (service == null) { log.log(Level.SEVERE, "DocprocService for session '" + serviceName + "' not found, returning request '" + requestContext + "'."); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 146cd9cb988..655703d10ee 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -32,7 +32,7 @@ public class DocumentProcessingTask implements Runnable { private final List<Processing> processingsDone = new ArrayList<>(); private final DocumentProcessingHandler docprocHandler; - private RequestContext requestContext; + private final RequestContext requestContext; private final DocprocService service; private final ThreadPoolExecutor executor; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index 910d4b7961b..a17be4de9a5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -107,4 +107,5 @@ class ProcessingFactory { processing.setVariable("timeout", message.getTimeRemaining()); return processing; } + } diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java b/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java index 8a544dea221..ffe89e5a405 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java @@ -16,6 +16,7 @@ import java.util.Map.Entry; * * @author vegardh */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class SchemaMap implements ConfigSubscriber.SingleSubscriber<SchemamappingConfig> { /** Map key. Doctype can be null, not the others. */ diff --git a/document/abi-spec.json b/document/abi-spec.json index d5ad686cd1f..83a56d4fb5e 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -2576,6 +2576,24 @@ "public static final java.lang.String NAME" ] }, + "com.yahoo.document.fieldset.DocumentOnly": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.document.fieldset.FieldSet" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public boolean contains(com.yahoo.document.fieldset.FieldSet)", + "public com.yahoo.document.fieldset.FieldSet clone()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields": [ + "public static final java.lang.String NAME" + ] + }, "com.yahoo.document.fieldset.FieldCollection": { "superClass": "java.util.ArrayList", "interfaces": [ diff --git a/document/pom.xml b/document/pom.xml index 3faada08553..2b050a4904e 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -81,17 +81,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>com.yahoo.document.foo</mainClass> diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index ff6a7194e7d..7cbda1410c2 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -3,7 +3,6 @@ package com.yahoo.document; import com.google.inject.Inject; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.annotation.AnnotationTypes; @@ -14,7 +13,14 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.TensorType; import java.lang.reflect.Modifier; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Logger; /** @@ -33,6 +39,8 @@ import java.util.logging.Logger; public class DocumentTypeManager { private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); + + @SuppressWarnings("removal") // TODO Vespa 8: remove private ConfigSubscriber subscriber; // *Configured data types* (not built-in/primitive) indexed by their id diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index e43ff26272a..be2c182426e 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -3,9 +3,10 @@ package com.yahoo.document; import com.yahoo.compress.CompressionType; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.internal.GeoPosType; import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; @@ -23,6 +24,7 @@ import com.yahoo.tensor.TensorType; * * @author Einar M R Rosenvinge */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig> { private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); @@ -97,11 +99,19 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } + boolean looksLikePosition(StructDataType type) { + var pos = PositionDataType.INSTANCE; + return type.getName().equals(pos.getName()) && type.getId() == pos.getId(); + } + private void startStructsAndDocs(DocumentmanagerConfig config) { for (var thisDataType : config.datatype()) { for (var o : thisDataType.structtype()) { int id = thisDataType.id(); StructDataType type = new StructDataType(id, o.name()); + if (usev8geopositions && looksLikePosition(type)) { + type = new GeoPosType(8); + } inProgress(type); configMap.remove(id); } @@ -198,6 +208,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (var struct : thisDataType.structtype()) { int id = thisDataType.id(); StructDataType type = (StructDataType) typesById.get(id); + if (type instanceof GeoPosType) { + continue; + } for (var parent : struct.inherits()) { var parentStruct = (StructDataType) typesByName.get(parent.name()); type.inherit(parentStruct); @@ -319,9 +332,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private final DocumentTypeManager manager; } - private static class ApplyNewDoctypeConfig { + public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) { this.manager = manager; this.usev8geopositions = config.usev8geopositions(); @@ -379,7 +392,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (var typeconf : docTypeConfig.primitivetype()) { DataType type = manager.getDataType(typeconf.name()); if (! (type instanceof PrimitiveDataType)) { - throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type); + throw new IllegalArgumentException("Needed primitive type for '"+typeconf.name()+"' [idx "+typeconf.idx()+"] but got: "+type); } addNewType(typeconf.idx(), type); } @@ -411,10 +424,32 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } + private final Field POS_X = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_X); + private final Field POS_Y = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_Y); + + boolean isPositionStruct(DocumentmanagerConfig.Doctype.Structtype cfg) { + if (! cfg.name().equals(PositionDataType.STRUCT_NAME)) return false; + if (! cfg.inherits().isEmpty()) return false; + if (cfg.field().size() != 2) return false; + var f0 = cfg.field(0); + var f1 = cfg.field(1); + if (! f0.name().equals(POS_X.getName())) return false; + if (! f1.name().equals(POS_Y.getName())) return false; + if (f0.internalid() != POS_X.getId()) return false; + if (f1.internalid() != POS_Y.getId()) return false; + if (typesByIdx.get(f0.type()) != POS_X.getDataType()) return false; + if (typesByIdx.get(f1.type()) != POS_Y.getDataType()) return false; + return true; + } + void createEmptyStructs() { String docName = docTypeConfig.name(); for (var typeconf : docTypeConfig.structtype()) { - addNewType(typeconf.idx(), new StructDataType(typeconf.name())); + if (usev8geopositions && isPositionStruct(typeconf)) { + addNewType(typeconf.idx(), new GeoPosType(8)); + } else { + addNewType(typeconf.idx(), new StructDataType(typeconf.name())); + } } } @@ -486,6 +521,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } void fillStructs() { for (var structCfg : docTypeConfig.structtype()) { + if (usev8geopositions && isPositionStruct(structCfg)) { + continue; + } int idx = structCfg.idx(); StructDataType type = (StructDataType) typesByIdx.get(idx); for (var parent : structCfg.inherits()) { @@ -541,11 +579,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } for (var docType : config.doctype()) { var docTypeData = inProgressById.get(docType.idx()); + docTypeData.createSimpleTypes(); docTypeData.createEmptyStructs(); docTypeData.initializeDocType(); docTypeData.createEmptyAnnotationTypes(); docTypeData.createFactories(); - docTypeData.createSimpleTypes(); } createComplexTypes(); for (var docType : config.doctype()) { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 11a8eb7a350..672690bafad 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -21,7 +21,7 @@ import java.util.ListIterator; import java.util.RandomAccess; /** - * FieldValue which encapsulates a Array value + * FieldValue which encapsulates an Array value * * @author Einar M R Rosenvinge */ @@ -42,8 +42,7 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i this(type); for (T v : values) { if (!((ArrayDataType)type).getNestedType().isValueCompatible(v)) { - throw new IllegalArgumentException("FieldValue " + v + - " is not compatible with " + type + "."); + throw new IllegalArgumentException("FieldValue " + v + " is not compatible with " + type + "."); } } this.values.addAll(values); diff --git a/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java b/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java new file mode 100644 index 00000000000..7ddfad13dfc --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +/** + * @author arnej27959 + */ +public class DocumentOnly implements FieldSet { + public static final String NAME = "[document]"; + @Override + public boolean contains(FieldSet o) { + return (o instanceof DocumentOnly || o instanceof DocIdOnly || o instanceof NoFields); + } + + @Override + public FieldSet clone() { + return new DocumentOnly(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java index 41a1b898867..9b6507d171a 100644 --- a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java +++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java @@ -21,13 +21,14 @@ public class FieldSetRepo { FieldSet parseSpecialValues(String name) { if (name.equals(DocIdOnly.NAME)) { return new DocIdOnly(); } + else if (name.equals(DocumentOnly.NAME)) { return (new DocumentOnly()); } else if (name.equals(AllFields.NAME)) { return (new AllFields()); } else if (name.equals(NoFields.NAME)) { return (new NoFields()); } else if (name.equals("[docid]")) { return (new DocIdOnly()); } else { throw new IllegalArgumentException( "The only special names (enclosed in '[]') allowed are " + - "id, all, none"); + "id, all, document, none"); } } @@ -101,6 +102,8 @@ public class FieldSetRepo { return NoFields.NAME; } else if (fieldSet instanceof DocIdOnly) { return DocIdOnly.NAME; + } else if (fieldSet instanceof DocumentOnly) { + return DocumentOnly.NAME; } else { throw new IllegalArgumentException("Unknown field set type " + fieldSet); } @@ -112,6 +115,18 @@ public class FieldSetRepo { * fieldset. */ public void copyFields(Document source, Document target, FieldSet fieldSet) { + if (fieldSet instanceof DocumentOnly) { + var actual = source.getDataType().fieldSet(DocumentOnly.NAME); + if (actual != null) { + for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + if (actual.contains(v.getKey())) { + target.setFieldValue(v.getKey(), v.getValue()); + } + } + return; + } + } for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) { Map.Entry<Field, FieldValue> v = i.next(); @@ -126,6 +141,21 @@ public class FieldSetRepo { */ public void stripFields(Document target, FieldSet fieldSet) { List<Field> toStrip = new ArrayList<>(); + if (fieldSet instanceof DocumentOnly) { + var actual = target.getDataType().fieldSet(DocumentOnly.NAME); + if (actual != null) { + for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + if (! actual.contains(v.getKey())) { + toStrip.add(v.getKey()); + } + } + for (Field f : toStrip) { + target.removeFieldValue(f); + } + return; + } + } for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) { Map.Entry<Field, FieldValue> v = i.next(); diff --git a/document/src/main/java/com/yahoo/document/internal/GeoPosType.java b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java new file mode 100644 index 00000000000..2999f7506ee --- /dev/null +++ b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.document.internal; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.Struct; + +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; + +/** + * @author arnej + **/ +public final class GeoPosType extends StructDataType { + + private final boolean useV8json; + private static final Field F_X = new Field("x", DataType.INT); + private static final Field F_Y = new Field("y", DataType.INT); + + public GeoPosType(int vespaVersion) { + super("position"); + this.useV8json = (vespaVersion == 8); + assert(vespaVersion > 6); + assert(vespaVersion < 9); + addField(F_X); + addField(F_Y); + } + + public boolean renderJsonAsVespa8() { + return this.useV8json; + } + + public double getLatitude(Struct pos) { + assert(pos.getDataType() == this); + double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6; + return ns; + } + + public double getLongitude(Struct pos) { + assert(pos.getDataType() == this); + double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6; + return ew; + } + + private static final DecimalFormat degreeFmt; + + static { + degreeFmt = new DecimalFormat("0.0#####", DecimalFormatSymbols.getInstance(Locale.ENGLISH)); + degreeFmt.setMinimumIntegerDigits(1); + degreeFmt.setMinimumFractionDigits(1); + degreeFmt.setMaximumFractionDigits(6); + } + + static String fmtD(double degrees) { + return degreeFmt.format(degrees); + } + + public String fmtLatitude(Struct pos) { + assert(pos.getDataType() == this); + double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6; + return fmtD(ns); + } + + public String fmtLongitude(Struct pos) { + assert(pos.getDataType() == this); + double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6; + return fmtD(ew); + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 1d9fd3aa1ec..340bd542885 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -25,6 +25,7 @@ import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.TensorReader; import com.yahoo.document.json.readers.TensorRemoveUpdateReader; import com.yahoo.document.serialization.FieldWriter; @@ -153,12 +154,32 @@ public class JsonSerializationHelper { }); } + private static void serializeGeoPos(JsonGenerator generator, FieldBase field, Struct value, GeoPosType dataType) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> { + generator.writeStartObject(); + generator.writeFieldName("lat"); + generator.writeRawValue(dataType.fmtLatitude(value)); + generator.writeFieldName("lng"); + generator.writeRawValue(dataType.fmtLongitude(value)); + generator.writeEndObject(); + }); + } + public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { - if (value.getDataType() == PositionDataType.INSTANCE) { + DataType dt = value.getDataType(); + // TODO remove in Vespa 8: + if (dt == PositionDataType.INSTANCE) { serializeString(generator, field, PositionDataType.renderAsString(value)); return; } - + if (dt instanceof GeoPosType) { + var gpt = (GeoPosType)dt; + if (gpt.renderJsonAsVespa8()) { + serializeGeoPos(generator, field, value, gpt); + return; + } + } serializeStructuredField(fieldWriter, generator, field, value); } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 757356e3096..4afab9233c6 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -116,13 +116,11 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu doc.setDataType(readDocumentType()); doc.setId(documentId); - Struct h = doc.getHeader(); - h.clear(); if ((content & 0x2) != 0) { - readHeaderBody(h); + readHeaderBody(doc); } if ((content & 0x4) != 0) { - readHeaderBody(h); + readHeaderBody(doc); } if (dataLength != (position() - dataPos)) { @@ -322,15 +320,12 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu buf = bigBuf; } - private void readHeaderBody(Struct primary) { - primary.setVersion(version); - + private void readHeaderBody(Document target) { if (version < 8) { throw new DeserializationException("Illegal document serialization version " + version); } int dataSize = getInt(null); - byte comprCode = getByte(null); CompressionType compression = CompressionType.valueOf(comprCode); @@ -366,7 +361,8 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu // for a while: deserialize from this buffer instead: buf = GrowableByteBuffer.wrap(destination); - StructDataType priType = primary.getDataType(); + StructDataType priType = target.getDataType().contentStruct(); + for (int i=0; i<numberOfFields; ++i) { int posBefore = position(); Integer f_id = fieldIdsAndLengths.get(i).first; @@ -374,7 +370,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu if (structField != null) { FieldValue value = structField.getDataType().createFieldValue(); value.deserialize(structField, this); - primary.setFieldValue(structField, value); + target.setFieldValue(structField, value); } //jump to beginning of next field: position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 09e41a0e8bf..8f5a0f6acd5 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -106,7 +106,8 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume doc.getDataType().serialize(this); if (hasHead) { - doc.getHeader().serialize(null, this); + StructuredFieldValue asStructured = doc; + write(null, asStructured); } int finalPos = buf.position(); @@ -311,12 +312,12 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } /** - * Write out the value of struct field + * Write out the value of structured field * * @param field - field description (name and data type) * @param s - field value */ - public void write(FieldBase field, Struct s) { + public void write(FieldBase field, StructuredFieldValue s) { // Serialize all parts first.. As we need to know length before starting // Serialize all the fields. @@ -330,7 +331,9 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume List<Integer> fieldIds = new LinkedList<>(); List<java.lang.Integer> fieldLengths = new LinkedList<>(); - for (Map.Entry<Field, FieldValue> value : s.getFields()) { + var iter = s.iterator(); + while (iter.hasNext()) { + Map.Entry<Field, FieldValue> value = iter.next(); int startPos = buffer.position(); value.getValue().serialize(value.getKey(), this); @@ -343,13 +346,14 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume buffer.flip(); buf = bigBuffer; + int sz = fieldIds.size(); // Actual serialization starts here. int lenPos = buf.position(); putInt(null, 0); // Move back to this after compression is done. buf.put(CompressionType.NONE.getCode()); - buf.putInt1_4Bytes(s.getFieldCount()); + buf.putInt1_4Bytes(sz); - for (int i = 0; i < s.getFieldCount(); ++i) { + for (int i = 0; i < sz; ++i) { putInt1_4Bytes(null, fieldIds.get(i)); putInt2_4_8Bytes(null, fieldLengths.get(i)); } @@ -365,13 +369,14 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } /** - * Write out the value of structured field + * Write out the value of struct field * * @param field - field description (name and data type) * @param value - field value */ - public void write(FieldBase field, StructuredFieldValue value) { - throw new IllegalArgumentException("Not Implemented"); + public void write(FieldBase field, Struct value) { + StructuredFieldValue asStructured = value; + write(field, asStructured); } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index c610af9778d..082a88bb000 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -105,7 +105,8 @@ public final class XmlDocumentWriter implements DocumentWriter { if (lastModified != null) { buffer.addAttribute("lastmodifiedtime", lastModified); } - write(null, value.getHeader()); + StructuredFieldValue asStructured = value; + write(null, asStructured); buffer.endTag(); } @@ -228,14 +229,13 @@ public final class XmlDocumentWriter implements DocumentWriter { @Override public void write(FieldBase field, Struct value) { - optionalWrapperStart(field); - XmlSerializationHelper.printStructXml(value, buffer); - optionalWrapperEnd(field); + StructuredFieldValue asStructured = value; + write(field, asStructured); } @Override public void write(FieldBase field, StructuredFieldValue value) { - buffer.beginTag(field.getName()); + optionalWrapperStart(field); Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); while (i.hasNext()) { Map.Entry<Field, FieldValue> v = i.next(); @@ -243,7 +243,7 @@ public final class XmlDocumentWriter implements DocumentWriter { v.getValue().printXml(buffer); buffer.endTag(); } - buffer.endTag(); + optionalWrapperEnd(field); } @Override diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 9b99c0c9bbd..1bb91b91762 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -54,7 +54,7 @@ public class XmlSerializationHelper { if (lastModified != null) { xml.addAttribute("lastmodifiedtime", lastModified); } - doc.getHeader().printXml(xml); + printStructured(doc, xml); } public static void printDoubleXml(DoubleFieldValue d, XmlStream xml) { @@ -97,7 +97,7 @@ public class XmlSerializationHelper { } } - public static void printStructXml(Struct s, XmlStream xml) { + private static void printStructured(StructuredFieldValue s, XmlStream xml) { Iterator<Map.Entry<Field, FieldValue>> it = s.iterator(); while (it.hasNext()) { Map.Entry<Field, FieldValue> val = it.next(); @@ -106,6 +106,9 @@ public class XmlSerializationHelper { xml.endTag(); } } + public static void printStructXml(Struct s, XmlStream xml) { + printStructured(s, xml); + } public static void printWeightedSetXml(WeightedSet ws, XmlStream xml) { Iterator<FieldValue> it = ws.fieldValueIterator(); diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java index 45a87cf3568..159eed8c2ba 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: Remove this package on Vespa 8 +// TODO: Remove this package on Vespa 9 @ExportPackage package com.yahoo.vespaxmlparser; diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg index 6ceda63e606..a4cf62db0c7 100644 --- a/document/src/test/document/documentmanager.cfg +++ b/document/src/test/document/documentmanager.cfg @@ -1,3 +1,4 @@ +usev8geopositions true doctype[4] doctype[0].name "document" doctype[0].idx 1000 @@ -45,6 +46,14 @@ doctype[0].annotationtype[8].internalid 6 doctype[0].annotationtype[8].datatype 1004 doctype[0].structtype[0].idx 1001 doctype[0].structtype[0].name document.header +doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 1002 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 1002 doctype[1].name "foobar" doctype[1].idx 1014 doctype[1].inherits[0].idx 1000 diff --git a/document/src/test/document/documentmanager.testv8pos.cfg b/document/src/test/document/documentmanager.testv8pos.cfg new file mode 100644 index 00000000000..3f776748b79 --- /dev/null +++ b/document/src/test/document/documentmanager.testv8pos.cfg @@ -0,0 +1,31 @@ +usev8geopositions true +doctype[2] +doctype[0].name "document" +doctype[0].idx 1000 +doctype[0].contentstruct 1001 +doctype[0].primitivetype[0].idx 1002 +doctype[0].primitivetype[0].name "int" +doctype[0].structtype[0].idx 1001 +doctype[0].structtype[0].name document.header +doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 1002 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 1002 +doctype[1].name "foobar" +doctype[1].idx 1014 +doctype[1].contentstruct 1015 +doctype[1].inherits[0].idx 1000 +doctype[1].arraytype[0].idx 1017 +doctype[1].arraytype[0].elementtype 10010 +doctype[1].structtype[0].idx 1015 +doctype[1].structtype[0].name foobar.header +doctype[1].structtype[0].field[0].name "simplepos" +doctype[1].structtype[0].field[0].internalid 1707020592 +doctype[1].structtype[0].field[0].type 10010 +doctype[1].structtype[0].field[1].name "arraypos" +doctype[1].structtype[0].field[1].internalid 1055920092 +doctype[1].structtype[0].field[1].type 1017 diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java index bf9960c3ca3..871e54ca46c 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java @@ -5,6 +5,8 @@ import com.yahoo.document.datatypes.FloatFieldValue; import com.yahoo.document.datatypes.Raw; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertNotNull; @@ -61,6 +63,8 @@ public class DocumentTestCaseBase { stringField = testDocType.getField("stringattr"); minField = testDocType.getField("Minattr"); + testDocType.addFieldSets(Map.of("[document]", List.of("stringattr", "intattr"))); + docMan.registerDocumentType(testDocType); } diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index 0aa5aec4b85..b89ed2b6b08 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -114,7 +114,7 @@ public class DocumentTypeManagerTestCase { assertSame(docType2, manager.getDocumentType(new DataTypeName("foo1"))); assertSame(docType3, manager.getDocumentType(new DataTypeName("foo2"))); assertSame(docType4, manager.getDocumentType(new DataTypeName("foo3"))); - + assertEquals(manager.getDocumentTypes().size(), 5); assertNotNull(manager.getDocumentTypes().get(new DataTypeName("document"))); assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), docType1); @@ -587,6 +587,18 @@ search annotationsimplicitstruct { assertFalse(docType.hasImportedField("a_missing_imported_field")); } + @Test + public void position_type_is_recognized_as_v8() { + var manager = DocumentTypeManager.fromFile("src/test/document/documentmanager.testv8pos.cfg"); + var docType = manager.getDocumentType("foobar"); + var simplepos = docType.getField("simplepos").getDataType(); + assertTrue(simplepos instanceof StructDataType); + var arraypos = docType.getField("arraypos").getDataType(); + assertTrue(arraypos instanceof ArrayDataType); + var array = (ArrayDataType) arraypos; + assertTrue(array.getNestedType() instanceof StructDataType); + } + // TODO test clone(). Also fieldSets not part of clone()..! // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this... diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java index e1e93adfb6d..b6dff63ac7b 100644 --- a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java +++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java @@ -23,6 +23,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { @Test public void testClone() throws Exception { assertTrue(new AllFields().clone() instanceof AllFields); + assertTrue(new DocumentOnly().clone() instanceof DocumentOnly); assertTrue(new NoFields().clone() instanceof NoFields); assertTrue(new DocIdOnly().clone() instanceof DocIdOnly); } @@ -32,6 +33,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { FieldSetRepo repo = new FieldSetRepo(); assertTrue(repo.parse(docMan, AllFields.NAME) instanceof AllFields); + assertTrue(repo.parse(docMan, DocumentOnly.NAME) instanceof DocumentOnly); assertTrue(repo.parse(docMan, NoFields.NAME) instanceof NoFields); assertTrue(repo.parse(docMan, DocIdOnly.NAME) instanceof DocIdOnly); @@ -72,21 +74,30 @@ public class FieldSetTestCase extends DocumentTestCaseBase { assertTrue(intAttr.contains(new DocIdOnly())); assertTrue(intAttr.contains(new NoFields())); assertFalse(intAttr.contains(new AllFields())); + assertFalse(intAttr.contains(new DocumentOnly())); assertFalse(new NoFields().contains(intAttr)); assertFalse(new NoFields().contains(new AllFields())); assertFalse(new NoFields().contains(new DocIdOnly())); + assertFalse(new NoFields().contains(new DocumentOnly())); assertTrue(new AllFields().contains(intAttr)); assertTrue(new AllFields().contains(rawAttr)); assertTrue(new AllFields().contains(new DocIdOnly())); + assertTrue(new AllFields().contains(new DocumentOnly())); assertTrue(new AllFields().contains(new NoFields())); assertTrue(new AllFields().contains(new AllFields())); assertTrue(new DocIdOnly().contains(new NoFields())); assertTrue(new DocIdOnly().contains(new DocIdOnly())); + assertFalse(new DocIdOnly().contains(new DocumentOnly())); assertFalse(new DocIdOnly().contains(intAttr)); + assertTrue(new DocumentOnly().contains(new NoFields())); + assertTrue(new DocumentOnly().contains(new DocIdOnly())); + assertTrue(new DocumentOnly().contains(new DocumentOnly())); + assertFalse(new DocumentOnly().contains(intAttr)); + assertContains("testdoc:rawattr,intattr", "testdoc:intattr"); assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr"); assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr"); @@ -124,6 +135,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { doc.removeFieldValue("rawattr"); assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, AllFields.NAME)); + assertEquals("stringattr:tjohei,intattr:50", doCopyFields(doc, DocumentOnly.NAME)); assertEquals("floatattr:3.56,byteattr:30", doCopyFields(doc, "testdoc:floatattr,byteattr")); } @@ -140,6 +152,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { doc.removeFieldValue("rawattr"); assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, AllFields.NAME)); + assertEquals("stringattr:tjohei,intattr:50", doStripFields(doc, DocumentOnly.NAME)); assertEquals("floatattr:3.56,byteattr:30", doStripFields(doc, "testdoc:floatattr,byteattr")); } @@ -150,6 +163,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { AllFields.NAME, NoFields.NAME, DocIdOnly.NAME, + DocumentOnly.NAME, "testdoc:rawattr", "testdoc:rawattr,intattr" }; diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 66ff7a7d4cd..ab4af5e722e 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -31,6 +31,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.document.serialization.DocumentSerializer; @@ -149,6 +150,7 @@ public class JsonReaderTestCase { DocumentType x = new DocumentType("testsinglepos"); DataType d = PositionDataType.INSTANCE; x.addField(new Field("singlepos", d)); + x.addField(new Field("geopos", new GeoPosType(8))); types.registerDocumentType(x); } { @@ -612,6 +614,43 @@ public class JsonReaderTestCase { } @Test + public void testPositionGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': 'N63.429722;E10.393333' }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + } + + @Test + public void testPositionOldGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': {'x':10393333,'y':63429722} }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + } + + @Test + public void testGeoPositionGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': {'lat':63.429722,'lng':10.393333} }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + assertEquals(PositionDataType.INSTANCE, f.getDataType()); + } + + @Test public void testPositionNegative() throws IOException { Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", " 'fields': {", diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 29703eadfce..7573aba519f 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.ReferenceFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.tensor.TensorType; @@ -93,6 +94,7 @@ public class JsonWriterTestCase { DocumentType x = new DocumentType("testmultipos"); DataType d = new ArrayDataType(PositionDataType.INSTANCE); x.addField(new Field("multipos", d)); + x.addField(new Field("geopos", new ArrayDataType(new GeoPosType(8)))); types.registerDocumentType(x); } @@ -100,6 +102,7 @@ public class JsonWriterTestCase { DocumentType x = new DocumentType("testsinglepos"); DataType d = PositionDataType.INSTANCE; x.addField(new Field("singlepos", d)); + x.addField(new Field("geopos", new GeoPosType(8))); types.registerDocumentType(x); } @@ -202,11 +205,13 @@ public class JsonWriterTestCase { @Test public void singlePosTest() throws IOException { roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }"); + roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"geopos\": { \"lat\": 60.222333, \"lng\": 10.12 } }"); } @Test public void multiPosTest() throws IOException { roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }"); + roundTripEquality("id:unittest:testmultipos::bamf", "{ \"geopos\": [ { \"lat\": -1.5, \"lng\": -1.5 }, { \"lat\": 63.4, \"lng\": 10.4 }, { \"lat\": 0.0, \"lng\": 0.0 } ] }"); } @Test diff --git a/document/src/tests/fieldsettest.cpp b/document/src/tests/fieldsettest.cpp index b4ee7713613..9f00fdd8c0d 100644 --- a/document/src/tests/fieldsettest.cpp +++ b/document/src/tests/fieldsettest.cpp @@ -28,6 +28,7 @@ TEST_F(FieldSetTest, testParsing) const DocumentTypeRepo& docRepo = testDocMan.getTypeRepo(); (void) dynamic_cast<AllFields&>(*FieldSetRepo::parse(docRepo, AllFields::NAME)); + (void) dynamic_cast<DocumentOnly&>(*FieldSetRepo::parse(docRepo, DocumentOnly::NAME)); (void) dynamic_cast<NoFields&>(*FieldSetRepo::parse(docRepo, NoFields::NAME)); (void) dynamic_cast<DocIdOnly&>(*FieldSetRepo::parse(docRepo, DocIdOnly::NAME)); @@ -73,6 +74,7 @@ TEST_F(FieldSetTest, testContains) NoFields none; AllFields all; + DocumentOnly doconly; DocIdOnly id; EXPECT_EQ(false, headerField.contains(type.getField("headerlongval"))); @@ -87,6 +89,8 @@ TEST_F(FieldSetTest, testContains) EXPECT_EQ(true, all.contains(id)); EXPECT_EQ(false, none.contains(id)); EXPECT_EQ(true, id.contains(none)); + EXPECT_EQ(true, doconly.contains(none)); + EXPECT_EQ(true, doconly.contains(id)); EXPECT_EQ(true, checkContains(repo, "testdoctype1:content,headerval", @@ -181,6 +185,9 @@ TEST_F(FieldSetTest, testCopyDocumentFields) "headerval: 5678\n" "hstringval: hello fantastic world\n"), doCopyFields(*src, repo, AllFields::NAME)); + EXPECT_EQ(std::string("headerval: 5678\n" + "hstringval: hello fantastic world\n"), + doCopyFields(*src, repo, DocumentOnly::NAME)); EXPECT_EQ(std::string("content: megafoo megabar\n" "hstringval: hello fantastic world\n"), doCopyFields(*src, repo, "testdoctype1:hstringval,content")); @@ -219,9 +226,19 @@ TEST_F(FieldSetTest, testDocumentSubsetCopy) EXPECT_EQ(doCopyFields(*src, repo, AllFields::NAME), stringifyFields(*doc)); } + { + Document::UP doc(FieldSet::createDocumentSubsetCopy(*src, DocumentOnly())); + // Test that document id and type are copied correctly. + EXPECT_TRUE(doc.get()); + EXPECT_EQ(src->getId(), doc->getId()); + EXPECT_EQ(src->getType(), doc->getType()); + EXPECT_EQ(doCopyFields(*src, repo, DocumentOnly::NAME), + stringifyFields(*doc)); + } const char* fieldSets[] = { AllFields::NAME, + DocumentOnly::NAME, NoFields::NAME, "testdoctype1:hstringval,content" }; @@ -239,6 +256,7 @@ TEST_F(FieldSetTest, testSerialize) const char* fieldSets[] = { AllFields::NAME, NoFields::NAME, + DocumentOnly::NAME, DocIdOnly::NAME, "testdoctype1:content", "testdoctype1:content,hstringval" @@ -264,6 +282,9 @@ TEST_F(FieldSetTest, testStripFields) "headerval: 5678\n" "hstringval: hello fantastic world\n"), doStripFields(*src, repo, AllFields::NAME)); + EXPECT_EQ(std::string("headerval: 5678\n" + "hstringval: hello fantastic world\n"), + doStripFields(*src, repo, DocumentOnly::NAME)); EXPECT_EQ(std::string("content: megafoo megabar\n" "hstringval: hello fantastic world\n"), doStripFields(*src, repo, "testdoctype1:hstringval,content")); diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp index 5c696d80fe7..86b1d93eacf 100644 --- a/document/src/vespa/document/base/field.cpp +++ b/document/src/vespa/document/base/field.cpp @@ -79,6 +79,7 @@ Field::contains(const FieldSet& fields) const case Type::NONE: case Type::DOCID: return true; + case Type::DOCUMENT_ONLY: case Type::ALL: return false; } diff --git a/document/src/vespa/document/base/testdocrepo.cpp b/document/src/vespa/document/base/testdocrepo.cpp index d48bac0ff74..c1930de8551 100644 --- a/document/src/vespa/document/base/testdocrepo.cpp +++ b/document/src/vespa/document/base/testdocrepo.cpp @@ -4,7 +4,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/config/print/fileconfigreader.h> +#include <vespa/config/print/fileconfigreader.hpp> using document::config_builder::Struct; using document::config_builder::Wset; @@ -18,7 +18,7 @@ TestDocRepo::TestDocRepo() _repo(new DocumentTypeRepo(_cfg)) { } - TestDocRepo::~TestDocRepo() {} +TestDocRepo::~TestDocRepo() = default; DocumenttypesConfig TestDocRepo::getDefaultConfig() { const int type1_id = 238423572; @@ -27,6 +27,7 @@ DocumenttypesConfig TestDocRepo::getDefaultConfig() { const int mystruct_id = -2092985851; const int structarray_id = 759956026; config_builder::DocumenttypesConfigBuilderHelper builder; + ::config::StringVector documentfields = { "headerval", "hstringval", "title" }; builder.document(type1_id, "testdoctype1", Struct("testdoctype1.header") .addField("headerval", DataType::T_INT) @@ -55,7 +56,9 @@ DocumenttypesConfig TestDocRepo::getDefaultConfig() { .addTensorField("sparse_tensor", "tensor(x{})") .addTensorField("sparse_xy_tensor", "tensor(x{},y{})") .addTensorField("sparse_float_tensor", "tensor<float>(x{})") - .addTensorField("dense_tensor", "tensor(x[2])")); + .addTensorField("dense_tensor", "tensor(x[2])")) + .doc_type.fieldsets["[document]"].fields.swap(documentfields); + builder.document(type2_id, "testdoctype2", Struct("testdoctype2.header") .addField("onlyinchild", DataType::T_INT), diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp index 81389d25bd7..1c43e2dfc64 100644 --- a/document/src/vespa/document/datatype/documenttype.cpp +++ b/document/src/vespa/document/datatype/documenttype.cpp @@ -14,8 +14,30 @@ using vespalib::IllegalArgumentException; using vespalib::make_string; using vespalib::stringref; + namespace document { +namespace { +FieldCollection build_field_collection(const std::set<vespalib::string> &fields, + const DocumentType &doc_type) +{ + Field::Set::Builder builder; + for (const auto & field_name : fields) { + if (doc_type.hasField(field_name)) { + builder.add(&doc_type.getField(field_name)); + } + } + return FieldCollection(doc_type, builder.build()); +} +} // namespace <unnamed> + +DocumentType::FieldSet::FieldSet(const vespalib::string & name, Fields fields, + const DocumentType & doc_type) + : _name(name), + _fields(fields), + _field_collection(build_field_collection(fields, doc_type)) +{} + IMPLEMENT_IDENTIFIABLE(DocumentType, StructuredDataType); DocumentType::DocumentType() = default; @@ -75,7 +97,7 @@ DocumentType::~DocumentType() = default; DocumentType & DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields) { - _fieldSets[name] = FieldSet(name, std::move(fields)); + _fieldSets.emplace(name, FieldSet(name, std::move(fields), *this)); return *this; } diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h index 28a0f0c9d55..bddc0a26e99 100644 --- a/document/src/vespa/document/datatype/documenttype.h +++ b/document/src/vespa/document/datatype/documenttype.h @@ -11,6 +11,7 @@ #pragma once +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/datatype/structdatatype.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/string.h> @@ -28,23 +29,19 @@ public: class FieldSet { public: using Fields = std::set<vespalib::string>; - FieldSet() = default; - explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {} - FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {} + FieldSet(const vespalib::string & name, Fields fields, + const DocumentType & doc_type); + FieldSet(const FieldSet&) = default; - FieldSet& operator=(const FieldSet&) = default; FieldSet(FieldSet&&) noexcept = default; - FieldSet& operator=(FieldSet&&) noexcept = default; const vespalib::string & getName() const noexcept { return _name; } const Fields & getFields() const noexcept { return _fields; } - FieldSet & add(vespalib::string & field) { - _fields.insert(field); - return *this; - } + const FieldCollection & asCollection() const { return _field_collection; } private: vespalib::string _name; Fields _fields; + FieldCollection _field_collection; }; using FieldSetMap = std::map<vespalib::string, FieldSet>; using ImportedFieldNames = vespalib::hash_set<vespalib::string>; diff --git a/document/src/vespa/document/fieldset/fieldset.h b/document/src/vespa/document/fieldset/fieldset.h index fd604a19e08..1f53f7a3456 100644 --- a/document/src/vespa/document/fieldset/fieldset.h +++ b/document/src/vespa/document/fieldset/fieldset.h @@ -20,7 +20,8 @@ public: SET, ALL, NONE, - DOCID + DOCID, + DOCUMENT_ONLY }; using SP = std::shared_ptr<FieldSet>; diff --git a/document/src/vespa/document/fieldset/fieldsetrepo.cpp b/document/src/vespa/document/fieldset/fieldsetrepo.cpp index 97b7fa09813..851eb8f4ecf 100644 --- a/document/src/vespa/document/fieldset/fieldsetrepo.cpp +++ b/document/src/vespa/document/fieldset/fieldsetrepo.cpp @@ -28,10 +28,12 @@ parseSpecialValues(vespalib::stringref name) return std::make_shared<NoFields>(); } else if ((name.size() == 7) && (name[1] == 'd') && (name[2] == 'o') && (name[3] == 'c') && (name[4] == 'i') && (name[5] == 'd') && (name[6] == ']')) { return std::make_shared<DocIdOnly>(); + } else if (name.size() == 10 && name == DocumentOnly::NAME) { + return std::make_shared<DocumentOnly>(); } else { throw vespalib::IllegalArgumentException( "The only special names (enclosed in '[]') allowed are " - "id, all, none, not '" + name + "'."); + "id, all, none, docid, document; but not '" + name + "'."); } } @@ -109,6 +111,8 @@ FieldSetRepo::serialize(const FieldSet& fieldSet) return NoFields::NAME; case FieldSet::Type::DOCID: return DocIdOnly::NAME; + case FieldSet::Type::DOCUMENT_ONLY: + return DocumentOnly::NAME; default: return ""; } diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp index 403e220466c..74dac80879a 100644 --- a/document/src/vespa/document/fieldset/fieldsets.cpp +++ b/document/src/vespa/document/fieldset/fieldsets.cpp @@ -47,6 +47,7 @@ FieldCollection::contains(const FieldSet& fields) const case Type::NONE: case Type::DOCID: return true; + case Type::DOCUMENT_ONLY: case Type::ALL: return false; } @@ -60,6 +61,12 @@ FieldSet::copyFields(Document& dest, const Document& src, const FieldSet& fields if (fields.getType() == Type::ALL) { dest.getFields() = src.getFields(); return; + } else if (fields.getType() == Type::DOCUMENT_ONLY) { + const auto * actual = src.getType().getFieldSet(DocumentOnly::NAME); + if (actual != nullptr) { + copyFields(dest, src, actual->asCollection()); + } + return; } for (Document::const_iterator it(src.begin()), e(src.end()); it != e; ++it) @@ -90,6 +97,15 @@ FieldSet::stripFields(Document& doc, const FieldSet& fieldsToKeep) { doc.clear(); return; + } else if (fieldsToKeep.getType() == Type::DOCUMENT_ONLY) { + const auto * actual = doc.getType().getFieldSet(DocumentOnly::NAME); + if (actual != nullptr) { + return stripFields(doc, actual->asCollection()); + } else { + // XXX - should not happen + doc.clear(); + return; + } } std::vector<const Field*> fieldsToRemove; for (Document::const_iterator it(doc.begin()), e(doc.end()); diff --git a/document/src/vespa/document/fieldset/fieldsets.h b/document/src/vespa/document/fieldset/fieldsets.h index 59cda189018..c92453cc67e 100644 --- a/document/src/vespa/document/fieldset/fieldsets.h +++ b/document/src/vespa/document/fieldset/fieldsets.h @@ -33,6 +33,18 @@ public: Type getType() const override { return Type::DOCID; } }; +class DocumentOnly final : public FieldSet +{ +public: + static constexpr const char * NAME = "[document]"; + bool contains(const FieldSet& fields) const override { + return fields.getType() == Type::DOCUMENT_ONLY + || fields.getType() == Type::DOCID + || fields.getType() == Type::NONE; + } + Type getType() const override { return Type::DOCUMENT_ONLY; } +}; + class FieldCollection : public FieldSet { public: diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 15198013c5f..36526f09dd6 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -10,7 +10,6 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/fieldset/fieldsets.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/document/src/vespa/document/util/queue.h b/document/src/vespa/document/util/queue.h index 51daa8f9934..bad117de49d 100644 --- a/document/src/vespa/document/util/queue.h +++ b/document/src/vespa/document/util/queue.h @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <queue> +#include <cassert> +#include <condition_variable> #include <mutex> +#include <queue> #define UNUSED_PARAM(p) namespace document { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index d40aaaf46d1..29fc470fe2a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -47,6 +47,7 @@ import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; public abstract class DocumentAccess { private final DocumentTypeManager documentTypeManager; + @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSubscriber documentTypeConfigSubscriber; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index b76dee5efa9..1d2d8cfd309 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -81,6 +81,8 @@ public class VisitorIterator { protected static class DistributionRangeBucketSource implements BucketSource { private boolean flushActive = false; private int distributionBitCount; + private long totalBucketsSplit; + private long totalBucketsMerged; private final int slices; private final int sliceId; // Wouldn't need this if this were a non-static class, but do it for @@ -99,7 +101,9 @@ public class VisitorIterator { this.slices = slices; this.sliceId = sliceId; - progressToken = progress; + this.totalBucketsSplit = 0; + this.totalBucketsMerged = 0; + this.progressToken = progress; // New progress token (could also be empty, in which this is a // no-op anyway) @@ -281,6 +285,8 @@ public class VisitorIterator { bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + p.getPendingBucketCount()); } + totalBucketsSplit += bucketsSplit; + totalBucketsMerged += bucketsMerged; } private void correctTruncatedBucketCursor() { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index 44675d8d2ac..a9dfe0128e0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -4,6 +4,7 @@ package com.yahoo.documentapi; import com.yahoo.document.BucketId; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.ThrottlePolicy; @@ -30,6 +31,7 @@ public class VisitorParameters extends Parameters { private long fromTimestamp = 0; private long toTimestamp = 0; boolean visitRemoves = false; + // TODO Vespa 8: change to DocumentOnly.NAME; private String fieldSet = AllFields.NAME; boolean visitInconsistentBuckets = false; private ProgressToken resumeToken = null; @@ -286,10 +288,18 @@ public class VisitorParameters extends Parameters { // TODO: Document: Where is the default - does this ever return null, or does it return "storage" if input is null? public Route getRoute() { return visitRoute; } - /** Set the maximum number of documents to visit (max documents returned by the visitor) */ + /** Set the maximum number of documents to visit (max documents returned by the visitor) + * + * @deprecated use setMaxTotalHits instead + */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public void setMaxFirstPassHits(long max) { maxFirstPassHits = max; } - /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) */ + /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) + * + * @deprecated Use getMaxTotalHits() instead + */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public long getMaxFirstPassHits() { return maxFirstPassHits; } /** Set the maximum number of documents to visit (max documents returned by the visitor) */ @@ -349,7 +359,10 @@ public class VisitorParameters extends Parameters { * the desired amount of documents * * @param dynamicallyIncreaseMaxBucketsPerVisitor whether or not to increase + * + * @deprecated deprecated feature, will be removed on Vespa 8 */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public void setDynamicallyIncreaseMaxBucketsPerVisitor(boolean dynamicallyIncreaseMaxBucketsPerVisitor) { this.dynamicallyIncreaseMaxBucketsPerVisitor = dynamicallyIncreaseMaxBucketsPerVisitor; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 259b598dbf5..a39f8e81757 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -7,6 +7,7 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentIdResponse; @@ -127,6 +128,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result get(DocumentId id, DocumentOperationParameters parameters) { + // TODO Vespa 8: change to DocumentOnly.NAME GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); return send(msg, parameters); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 4bc96ed1d22..5537d122e16 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -7,6 +7,7 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.DocumentAccessException; import com.yahoo.documentapi.DocumentOperationParameters; @@ -152,6 +153,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { + // TODO Vespa 8: change to DocumentOnly.NAME GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index e99028684c7..0da9d0ea621 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -327,9 +327,11 @@ public class MessageBusVisitorSession implements VisitorSession { private final String sessionName = createSessionName(); private final String dataDestination; private final Clock clock; + private final Object replyTrackingMonitor = new Object(); private StateDescription state; private long visitorCounter = 0; private long startTimeNanos = 0; + private long scheduledHandleReplyTasks = 0; // Must be protected by replyTrackingMonitor private boolean scheduledSendCreateVisitors = false; private boolean done = false; private boolean destroying = false; // For testing and sanity checking @@ -454,14 +456,40 @@ public class MessageBusVisitorSession implements VisitorSession { return state; } + private boolean hasScheduledHandleReplyTask() { + // This is synchronized instead of an AtomicLong simply because it makes it considerably + // easier to reason about happens-before relationships, memory visibility and sequencing + // of events across threads when an actual critical section is involved. + synchronized (replyTrackingMonitor) { + return scheduledHandleReplyTasks != 0; + } + } + + private void incrementScheduledHandleReplyTasks() { + synchronized (replyTrackingMonitor) { + ++scheduledHandleReplyTasks; + } + } + + private void decrementScheduleHandleReplyTasks() { + synchronized (replyTrackingMonitor) { + assert(scheduledHandleReplyTasks > 0); + --scheduledHandleReplyTasks; + } + } + private ReplyHandler createReplyHandler() { return (reply) -> { // Generally, handleReply will run in the context of the // underlying transport layer's processing thread(s), so we // schedule our own reply handling task to avoid blocking it. try { + // Make concurrent reply handling visible in sender thread, if it's active. + // See SendCreateVisitorsTask.run() for a rationale. + incrementScheduledHandleReplyTasks(); taskExecutor.submitTask(new HandleReplyTask(reply)); } catch (RejectedExecutionException e) { + decrementScheduleHandleReplyTasks(); // We cannot reliably handle reply tasks failing to be submitted, since // the reply task performs all our internal state handling logic. As such, // we just immediately go into a failure destruction mode as soon as this @@ -648,7 +676,39 @@ public class MessageBusVisitorSession implements VisitorSession { if (done) { return; // Session already closed; we must not touch anything else. } - while (progress.getIterator().hasNext()) { + // We both send requests and process replies in the context of a dedicated task executor pool. + // However, MessageBus sending and reply receiving happens in the context of entirely + // separate threads. If the backend responds very quickly to visitor requests (such as + // if buckets are empty), this can leave us in the following awkward position: + // + // 1. Replies arrive from backend, open up the throttle window, reply handling + // task gets pushed onto executor queue (but not yet executed). + // 2. Send loop below continuously get a free send slot, keeps sending visitors + // and filling up the set of pending buckets in the progress token. + // 3. Since visitor session is busy-looping in the send task, reply processing is + // consequently entirely starved until the MessageBus throttle window is bursting + // at the seams. This can effectively nullify the effects of the throttling policy, + // especially if it's dynamic. But a static throttle policy with a sufficiently + // high max window size will also potentially cause a runaway visitor train since + // the active window size keeps getting decreased by backend replies. + // + // To get around this, we explicitly check for concurrently scheduled message handling + // tasks from the transport layer, breaking the loop if at least one handler has been + // scheduled. This also has the (positive) effect of draining all reply tasks before we + // start sending more work downstream. + // + // Since visitor session progress is edge-triggered and progresses exclusively by sending + // new visitors in reply handling tasks, it's critical that we never end up in a situation + // where we have no pending CreateVisitors (or scheduled tasks), or we risk effectively + // hanging the session. We must therefore be very careful that we only exit the send loop + // if we _know_ we have at least one pending task enqueued that will ensure session progress. + // + // We're holding the session (token) lock around checking the pending reply tasks count, so + // if we observe a change we know that a reply task must have been scheduled and that its + // processing must take place sequenced after we have exited the loop, as the reply handling + // also takes the session (token) lock. I.e. it should not be possible to end up in a + // situation where we stall session progress due to not having any further event edges. + while (progress.getIterator().hasNext() && !hasScheduledHandleReplyTask()) { VisitorIterator.BucketProgress bucket = progress.getIterator().getNext(); Result result = sender.send(createMessage(bucket)); if (result.isAccepted()) { @@ -710,7 +770,7 @@ public class MessageBusVisitorSession implements VisitorSession { } private class HandleReplyTask implements Runnable { - private Reply reply; + private final Reply reply; HandleReplyTask(Reply reply) { this.reply = reply; } @@ -718,6 +778,10 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void run() { synchronized (progress.getToken()) { + // Decrement pending replies inside same lock as sender task to ensure that if the sender + // observes a non-zero number of reply tasks, it's guaranteed that this actually means a + // task _will_ be run later at some point. + decrementScheduleHandleReplyTasks(); try { assert(pendingMessageCount > 0); --pendingMessageCount; @@ -748,7 +812,7 @@ public class MessageBusVisitorSession implements VisitorSession { } private class HandleMessageTask implements Runnable { - private Message message; + private final Message message; private HandleMessageTask(Message message) { this.message = message; @@ -933,7 +997,9 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().update(bucket, ProgressToken.FINISHED_BUCKET); } + @SuppressWarnings("removal") // TODO: Vespa 8: remove private boolean enoughHitsReceived() { + // TODO: Vespa 8: remove "Nth pass" concept entirely from API and internals if (params.getMaxFirstPassHits() != -1 && statistics.getDocumentsReturned() >= params.getMaxFirstPassHits()) { @@ -941,7 +1007,7 @@ public class MessageBusVisitorSession implements VisitorSession { } if (params.getMaxTotalHits() != -1 && ((statistics.getDocumentsReturned() - + statistics.getSecondPassDocumentsReturned()) + + statistics.getSecondPassDocumentsReturned()) // TODO: Vespa 8: remove >= params.getMaxTotalHits())) { return true; @@ -1021,6 +1087,7 @@ public class MessageBusVisitorSession implements VisitorSession { return scheduleSendCreateVisitorsIfApplicable(0, TimeUnit.MILLISECONDS); } + @SuppressWarnings("removal")// TODO: Vespa 8: remove private void handleCreateVisitorReply(CreateVisitorReply reply) { CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); @@ -1040,6 +1107,7 @@ public class MessageBusVisitorSession implements VisitorSession { trace.getRoot().addChild(reply.getTrace().getRoot()); } + // TODO: Vespa 8 remove this unused functionality if (params.getDynamicallyIncreaseMaxBucketsPerVisitor() && (reply.getVisitorStatistics().getDocumentsReturned() < params.getMaxFirstPassHits() / 2.0)) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java index cae0587ae39..097632f609f 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java @@ -4,6 +4,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.document.BucketId; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import java.util.ArrayList; import java.util.Iterator; @@ -24,6 +25,7 @@ public class CreateVisitorMessage extends DocumentMessage { private long fromTime = 0; private long toTime = 0; private boolean visitRemoves = false; + // TODO Vespa 8: change to DocumentOnly.NAME private String fieldSet = AllFields.NAME; private boolean visitInconsistentBuckets = false; private Map<String, byte[]> params = new TreeMap<>(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index 67ebe974e4e..a785438da80 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -23,6 +23,7 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class DocumentRouteSelectorPolicy implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<DocumentrouteselectorpolicyConfig> { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java index 669c355b2d8..0f6e738cb86 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java @@ -3,6 +3,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.document.DocumentId; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import java.util.Arrays; @@ -11,6 +12,7 @@ import java.util.Arrays; */ public class GetDocumentMessage extends DocumentMessage { + // TODO Vespa 8: change to DocumentOnly.NAME final static String DEFAULT_FIELD_SET = AllFields.NAME; private DocumentId documentId = null; private String fieldSet = DEFAULT_FIELD_SET; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java index 57cd70cf4b4..095d1f433d2 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -15,6 +15,7 @@ import static java.util.stream.Collectors.toUnmodifiableMap; /** * @author baldersheim */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class MessageTypePolicy implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<MessagetyperouteselectorpolicyConfig> { private final AtomicReference<Map<Integer, Route>> configRef = new AtomicReference<>(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index b6c87e0a27e..c022af0cf07 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -237,6 +237,7 @@ public abstract class RoutableFactories60 { public static class CreateVisitorReplyFactory extends DocumentReplyFactory { @Override + @SuppressWarnings("removal")// TODO: Vespa 8: remove protected DocumentReply doDecode(DocumentDeserializer buf) { CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); reply.setLastBucket(new BucketId(buf.getLong(null))); @@ -247,13 +248,14 @@ public abstract class RoutableFactories60 { vs.setBytesVisited(buf.getLong(null)); vs.setDocumentsReturned(buf.getLong(null)); vs.setBytesReturned(buf.getLong(null)); - vs.setSecondPassDocumentsReturned(buf.getLong(null)); - vs.setSecondPassBytesReturned(buf.getLong(null)); + vs.setSecondPassDocumentsReturned(buf.getLong(null)); // TODO: on Vespa 8 remove setter (_not_ getLong()) + vs.setSecondPassBytesReturned(buf.getLong(null)); // TODO: on Vespa 8 remove setter (_not_ getLong()) reply.setVisitorStatistics(vs); return reply; } @Override + @SuppressWarnings("removal")// TODO: Vespa 8: remove protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { CreateVisitorReply reply = (CreateVisitorReply)obj; buf.putLong(null, reply.getLastBucket().getRawId()); @@ -262,8 +264,8 @@ public abstract class RoutableFactories60 { buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); // TODO: on Vespa 8 remove getter (_not_ putLong()) + buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); // TODO: on Vespa 8 remove getter (_not_ putLong()) return true; } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java index 88e87a632d2..caed3867d99 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -10,6 +10,7 @@ import static org.junit.Assert.*; public class VisitorParametersTestCase { private LoadType loadType = new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3); + @SuppressWarnings("removal")// TODO: Vespa 8: remove private VisitorParameters createVisitorParameters() { VisitorParameters params = new VisitorParameters(""); params.setDocumentSelection("id.user==5678"); @@ -42,6 +43,7 @@ public class VisitorParametersTestCase { return params; } + @SuppressWarnings("removal")// TODO: Vespa 8: remove @Test public void testCopyConstructor() { VisitorParameters params = createVisitorParameters(); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 51fef9ac557..5250a6b1db7 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -274,6 +274,7 @@ public class Messages60TestCase extends MessagesTestBase { public class testCreateVisitorReply implements RunnableTest { @Override + @SuppressWarnings("removal") public void run() { CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); reply.setLastBucket(new BucketId(16, 123)); @@ -296,7 +297,7 @@ public class Messages60TestCase extends MessagesTestBase { assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); - assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); // TODO remove on Vespa 8 assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java index f55e6433cd0..01c4383b16a 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java @@ -942,6 +942,7 @@ public class MessageBusVisitorSessionTestCase { } @Override + @SuppressWarnings("removal") // TODO: Vespa 8: remove public void onVisitorStatistics(VisitorStatistics vs) { super.onVisitorStatistics(vs); builder.append("onVisitorStatistics : "); @@ -1411,6 +1412,7 @@ public class MessageBusVisitorSessionTestCase { doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); } + @SuppressWarnings("removal")// TODO: Vespa 8: remove @Test public void testVisitingCompletedFromSufficientFirstPassHits() { VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); @@ -1432,6 +1434,7 @@ public class MessageBusVisitorSessionTestCase { doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); } + @SuppressWarnings("removal")// TODO: Vespa 8: remove test @Test public void testVisitingCompletedFromSecondPassHits() { VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); @@ -2312,6 +2315,7 @@ public class MessageBusVisitorSessionTestCase { mc.controlHandler.toString()); } + @SuppressWarnings("removal")// TODO: Vespa 8: remove test @Test public void testDynamicallyIncreaseMaxBucketsPerVisitorOption() { VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index aa5411c8a78..c0bdb71fb25 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -382,8 +382,8 @@ Messages60Test::testCreateVisitorReply() EXPECT_EQUAL(ref.getVisitorStatistics().getBytesVisited(), (uint64_t)1024000); EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsReturned(), (uint64_t)123); EXPECT_EQUAL(ref.getVisitorStatistics().getBytesReturned(), (uint64_t)512000); - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); // TODO remove on Vespa 8 + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); // TODO remove on Vespa 8 } } return true; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp index 9872663ca7b..39e67408e76 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/config-stor-distribution.h> +#include <vespa/config/helper/configfetcher.hpp> #include <vespa/config/subscription/configuri.h> #include <cassert> diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h index ac9c58bf276..9580f242c25 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h @@ -6,8 +6,6 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/messagebus/routing/hop.h> -#include <vespa/config/helper/ifetchercallback.h> -#include <vespa/config/helper/configfetcher.h> namespace config { class ICallback; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index dc55bd7a8b6..9d3dd874bb0 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -12,10 +12,11 @@ #include <vespa/messagebus/routing/routingtable.h> #include <vespa/messagebus/messagebus.h> #include <vespa/vespalib/util/stringfmt.h> - -#include <vespa/log/log.h> +#include <vespa/config/helper/configfetcher.hpp> +#include <vespa/config/subscription/configuri.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/log/log.h> LOG_SETUP(".documentrouteselectorpolicy"); using document::select::Result; @@ -30,12 +31,14 @@ DocumentRouteSelectorPolicy::DocumentRouteSelectorPolicy( _lock(), _config(), _error("Not configured."), - _fetcher(configUri.getContext()) + _fetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) { - _fetcher.subscribe<messagebus::protocol::DocumentrouteselectorpolicyConfig>(configUri.getConfigId(), this); - _fetcher.start(); + _fetcher->subscribe<messagebus::protocol::DocumentrouteselectorpolicyConfig>(configUri.getConfigId(), this); + _fetcher->start(); } +DocumentRouteSelectorPolicy::~DocumentRouteSelectorPolicy() = default; + void DocumentRouteSelectorPolicy::configure(std::unique_ptr<messagebus::protocol::DocumentrouteselectorpolicyConfig> cfg) { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h index 43b28e7d35f..8512984766e 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h @@ -6,8 +6,8 @@ #include <map> #include <vespa/messagebus/routing/iroutingpolicy.h> #include <vespa/documentapi/common.h> -#include <vespa/config/config.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> +#include <mutex> namespace document { class DocumentTypeRepo; } @@ -16,6 +16,11 @@ namespace mbus { class RoutingContext; } +namespace config { + class ConfigUri; + class ConfigFetcher; +} + namespace documentapi { /** @@ -31,11 +36,11 @@ private: typedef std::shared_ptr<document::select::Node> SelectorPtr; typedef std::map<string, SelectorPtr> ConfigMap; - const document::DocumentTypeRepo &_repo; - mutable std::mutex _lock; - ConfigMap _config; - string _error; - config::ConfigFetcher _fetcher; + const document::DocumentTypeRepo &_repo; + mutable std::mutex _lock; + ConfigMap _config; + string _error; + std::unique_ptr<config::ConfigFetcher> _fetcher; /** * This method runs the selector associated with the given location on the content of the message. If the selector @@ -56,6 +61,7 @@ public: */ DocumentRouteSelectorPolicy(const document::DocumentTypeRepo &repo, const config::ConfigUri &configUri); + ~DocumentRouteSelectorPolicy() override; /** * This is a safety mechanism to allow the constructor to fail and signal that it can not be used. diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp index 9c5d978546b..0d7df9a9482 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp @@ -4,6 +4,9 @@ #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/messagebus/message.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/config/helper/configfetcher.hpp> +#include <vespa/config/subscription/configuri.h> + using vespa::config::content::MessagetyperouteselectorpolicyConfig; @@ -25,10 +28,10 @@ MessageTypePolicy::MessageTypePolicy(const config::ConfigUri & configUri) : config::IFetcherCallback<MessagetyperouteselectorpolicyConfig>(), _map(), _defaultRoute(), - _fetcher(configUri.getContext()) + _fetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) { - _fetcher.subscribe<MessagetyperouteselectorpolicyConfig>(configUri.getConfigId(), this); - _fetcher.start(); + _fetcher->subscribe<MessagetyperouteselectorpolicyConfig>(configUri.getConfigId(), this); + _fetcher->start(); } MessageTypePolicy::~MessageTypePolicy() {} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h index 6b6ff78b810..ffeda17d778 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h @@ -4,17 +4,20 @@ #include <vespa/messagebus/routing/iroutingpolicy.h> #include <vespa/vespalib/util/ptrholder.h> #include <vespa/config-messagetyperouteselectorpolicy.h> -#include <vespa/config/config.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/documentapi/common.h> +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace mbus { class RoutingContext; class Route; } namespace documentapi { -namespace policy {class MessageTypeMap; } +namespace policy { class MessageTypeMap; } /** * This policy is responsible for selecting among the given recipient routes * according to the configured document selection properties. To factilitate @@ -31,7 +34,7 @@ private: MessageTypeHolder _map; RouteHolder _defaultRoute; - config::ConfigFetcher _fetcher; + std::unique_ptr<config::ConfigFetcher> _fetcher; public: /** diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 6d93a619e19..8b7c56117b1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -170,6 +170,7 @@ RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b vs.setBytesVisited(decodeLong(buf)); vs.setDocumentsReturned(decodeLong(buf)); vs.setBytesReturned(decodeLong(buf)); + // TODO remove second pass concept on Vespa 8 vs.setSecondPassDocumentsReturned(decodeLong(buf)); vs.setSecondPassBytesReturned(decodeLong(buf)); reply->setVisitorStatistics(vs); @@ -187,6 +188,7 @@ RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &ob buf.putLong(reply.getVisitorStatistics().getBytesVisited()); buf.putLong(reply.getVisitorStatistics().getDocumentsReturned()); buf.putLong(reply.getVisitorStatistics().getBytesReturned()); + // TODO remove second pass concept on Vespa 8 buf.putLong(reply.getVisitorStatistics().getSecondPassDocumentsReturned()); buf.putLong(reply.getVisitorStatistics().getSecondPassBytesReturned()); return true; diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp index ab43da5816c..a067610fb25 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp @@ -10,28 +10,29 @@ #include <vespa/documentapi/messagebus/policies/messagetypepolicy.h> #include <vespa/documentapi/messagebus/policies/roundrobinpolicy.h> #include <vespa/documentapi/messagebus/policies/subsetservicepolicy.h> +#include <vespa/config/subscription/configuri.h> using namespace documentapi; mbus::IRoutingPolicy::UP RoutingPolicyFactories::AndPolicyFactory::createPolicy(const string ¶m) const { - return mbus::IRoutingPolicy::UP(new ANDPolicy(param)); + return std::make_unique<ANDPolicy>(param); } mbus::IRoutingPolicy::UP RoutingPolicyFactories::MessageTypePolicyFactory::createPolicy(const string ¶m) const { - return mbus::IRoutingPolicy::UP(new MessageTypePolicy(param)); + return std::make_unique<MessageTypePolicy>(param); } mbus::IRoutingPolicy::UP RoutingPolicyFactories::ContentPolicyFactory::createPolicy(const string ¶m) const { - mbus::IRoutingPolicy::UP ret(new ContentPolicy(param)); + auto ret = std::make_unique<ContentPolicy>(param); string error = static_cast<ContentPolicy&>(*ret).getError(); if (!error.empty()) { - ret.reset(new ErrorPolicy(error)); + return std::make_unique<ErrorPolicy>(error); } return ret; } @@ -43,7 +44,7 @@ RoutingPolicyFactories::LoadBalancerPolicyFactory::createPolicy(const string &pa string error = static_cast<LoadBalancerPolicy&>(*ret).getError(); if (!error.empty()) { fprintf(stderr, "Got error %s\n", error.c_str()); - ret.reset(new ErrorPolicy(error)); + return std::make_unique<ErrorPolicy>(error); } return ret; } @@ -60,11 +61,10 @@ DocumentRouteSelectorPolicyFactory(const document::DocumentTypeRepo &repo, mbus::IRoutingPolicy::UP RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory::createPolicy(const string ¶m) const { - mbus::IRoutingPolicy::UP ret(new DocumentRouteSelectorPolicy( - _repo, param.empty() ? _configId : param)); + auto ret = std::make_unique<DocumentRouteSelectorPolicy>(_repo, param.empty() ? _configId : param); string error = static_cast<DocumentRouteSelectorPolicy&>(*ret).getError(); if (!error.empty()) { - ret.reset(new ErrorPolicy(error)); + return std::make_unique<ErrorPolicy>(error); } return ret; } @@ -83,17 +83,17 @@ RoutingPolicyFactories::ExternPolicyFactory::createPolicy(const string ¶m) c mbus::IRoutingPolicy::UP RoutingPolicyFactories::LocalServicePolicyFactory::createPolicy(const string ¶m) const { - return mbus::IRoutingPolicy::UP(new LocalServicePolicy(param)); + return std::make_unique<LocalServicePolicy>(param); } mbus::IRoutingPolicy::UP RoutingPolicyFactories::RoundRobinPolicyFactory::createPolicy(const string ¶m) const { - return mbus::IRoutingPolicy::UP(new RoundRobinPolicy(param)); + return std::make_unique<RoundRobinPolicy>(param); } mbus::IRoutingPolicy::UP RoutingPolicyFactories::SubsetServicePolicyFactory::createPolicy(const string ¶m) const { - return mbus::IRoutingPolicy::UP(new SubsetServicePolicy(param)); + return std::make_unique<SubsetServicePolicy>(param); } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index eed4fa5ce66..e6669e3fde8 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_define_module( TESTS src/tests/ann + src/tests/apps/analyze_onnx_model src/tests/apps/eval_expr src/tests/eval/addr_to_symbol src/tests/eval/aggr diff --git a/eval/src/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/apps/analyze_onnx_model/CMakeLists.txt index e2ed64cd8cc..dc89213f9eb 100644 --- a/eval/src/apps/analyze_onnx_model/CMakeLists.txt +++ b/eval/src/apps/analyze_onnx_model/CMakeLists.txt @@ -1,7 +1,8 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespa-analyze-onnx-model +vespa_add_executable(eval_analyze_onnx_model_app SOURCES analyze_onnx_model.cpp + OUTPUT_NAME vespa-analyze-onnx-model INSTALL bin DEPENDS vespaeval diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index ecf38f6772b..868e9d036f1 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -4,6 +4,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/test/test_io.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/guard.h> @@ -11,8 +12,17 @@ using vespalib::make_string_short::fmt; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::Inspector; +using vespalib::slime::Cursor; using vespalib::FilePointer; using namespace vespalib::eval; +using namespace vespalib::eval::test; + +struct MyError { + vespalib::string msg; +}; bool read_line(FilePointer &file, vespalib::string &line) { char line_buffer[1024]; @@ -133,15 +143,50 @@ struct MakeInputType { } }; +vespalib::string make_bound_str(const std::map<vespalib::string,size_t> &bound) { + vespalib::string result; + if (!bound.empty()) { + for (const auto &[name, size]: bound) { + if (result.empty()) { + result.append(" ("); + } else { + result.append(","); + } + result.append(fmt("%s=%zu", name.c_str(), size)); + } + result.append(")"); + } + return result; +} + +void bind_input(Onnx::WirePlanner &planner, const Onnx::TensorInfo &input, const ValueType &type) { + auto bound = planner.get_bound_sizes(input); + if (!planner.bind_input_type(type, input)) { + auto bound_str = make_bound_str(bound); + throw MyError{fmt("incompatible type for input '%s': %s -> %s%s", + input.name.c_str(), type.to_spec().c_str(), input.type_as_string().c_str(), bound_str.c_str())}; + } +} + +ValueType make_output(const Onnx::WirePlanner &planner, const Onnx::TensorInfo &output) { + auto type = planner.make_output_type(output); + if (type.is_error()) { + throw MyError{fmt("unable to make compatible type for output '%s': %s -> error", + output.name.c_str(), output.type_as_string().c_str())}; + } + return type; +} + Onnx::WireInfo make_plan(Options &opts, const Onnx &model) { Onnx::WirePlanner planner; MakeInputType make_input_type(opts); for (const auto &input: model.inputs()) { auto type = make_input_type(input); - REQUIRE(planner.bind_input_type(type, input)); + bind_input(planner, input, type); } + planner.prepare_output_types(model); for (const auto &output: model.outputs()) { - REQUIRE(!planner.make_output_type(output).is_error()); + make_output(planner, output); } return planner.get_wire_info(model); } @@ -168,14 +213,57 @@ int usage(const char *self) { fprintf(stderr, " load onnx model and report memory usage\n"); fprintf(stderr, " options are used to specify unknown values, like dimension sizes\n"); fprintf(stderr, " options are accepted in the order in which they are needed\n"); - fprintf(stderr, " tip: run without options first, to see which you need\n"); + fprintf(stderr, " tip: run without options first, to see which you need\n\n"); + fprintf(stderr, "usage: %s --probe-types\n", self); + fprintf(stderr, " use onnx model to infer/probe output types based on input types\n"); + fprintf(stderr, " parameters are read from stdin and results are written to stdout\n"); + fprintf(stderr, " input format (json): {model:<filename>, inputs:{<name>:vespa-type-string}}\n"); + fprintf(stderr, " output format (json): {outputs:{<name>:vespa-type-string}}\n"); return 1; } +int probe_types() { + StdIn std_in; + StdOut std_out; + Slime params; + if (!JsonFormat::decode(std_in, params)) { + throw MyError{"invalid json"}; + } + Slime result; + auto &root = result.setObject(); + auto &types = root.setObject("outputs"); + Onnx model(params["model"].asString().make_string(), Onnx::Optimize::DISABLE); + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + auto spec = params["inputs"][model.inputs()[i].name].asString().make_string(); + auto input_type = ValueType::from_spec(spec); + if (input_type.is_error()) { + if (!params["inputs"][model.inputs()[i].name].valid()) { + throw MyError{fmt("missing type for model input '%s'", + model.inputs()[i].name.c_str())}; + } else { + throw MyError{fmt("invalid type for model input '%s': '%s'", + model.inputs()[i].name.c_str(), spec.c_str())}; + } + } + bind_input(planner, model.inputs()[i], input_type); + } + planner.prepare_output_types(model); + for (const auto &output: model.outputs()) { + auto output_type = make_output(planner, output); + types.setString(output.name, output_type.to_spec()); + } + write_compact(result, std_out); + return 0; +} + int my_main(int argc, char **argv) { if (argc < 2) { return usage(argv[0]); } + if ((argc == 2) && (vespalib::string(argv[1]) == "--probe-types")) { + return probe_types(); + } Options opts; for (int i = 2; i < argc; ++i) { opts.add_option(argv[i]); @@ -210,6 +298,9 @@ int my_main(int argc, char **argv) { int main(int argc, char **argv) { try { return my_main(argc, argv); + } catch (const MyError &err) { + fprintf(stderr, "error: %s\n", err.msg.c_str()); + return 3; } catch (const std::exception &ex) { fprintf(stderr, "got exception: %s\n", ex.what()); return 2; diff --git a/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt new file mode 100644 index 00000000000..06241b0add6 --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_analyze_onnx_model_test_app TEST + SOURCES + analyze_onnx_model_test.cpp + DEPENDS + vespaeval + AFTER + eval_analyze_onnx_model_app +) +vespa_add_test(NAME eval_analyze_onnx_model_test_app COMMAND eval_analyze_onnx_model_test_app) diff --git a/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp new file mode 100644 index 00000000000..72ef0346ea3 --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/eval/eval/test/test_io.h> + +using namespace vespalib; +using namespace vespalib::eval::test; + +vespalib::string module_build_path("../../../../"); +vespalib::string binary = module_build_path + "src/apps/analyze_onnx_model/vespa-analyze-onnx-model"; +vespalib::string probe_cmd = binary + " --probe-types"; + +vespalib::string get_source_dir() { + const char *dir = getenv("SOURCE_DIRECTORY"); + return (dir ? dir : "."); +} +vespalib::string source_dir = get_source_dir(); +vespalib::string probe_model = source_dir + "/../../tensor/onnx_wrapper/probe_model.onnx"; +vespalib::string simple_model = source_dir + "/../../tensor/onnx_wrapper/simple.onnx"; +vespalib::string dynamic_model = source_dir + "/../../tensor/onnx_wrapper/dynamic.onnx"; + +//----------------------------------------------------------------------------- + +TEST_F("require that output types can be probed", ServerCmd(probe_cmd)) { + Slime params; + params.setObject(); + params.get().setString("model", probe_model); + params.get().setObject("inputs"); + params["inputs"].setString("in1", "tensor<float>(x[2],y[3])"); + params["inputs"].setString("in2", "tensor<float>(x[2],y[3])"); + Slime result = f1.invoke(params); + EXPECT_EQUAL(result["outputs"].fields(), 3u); + EXPECT_EQUAL(result["outputs"]["out1"].asString().make_string(), vespalib::string("tensor<float>(d0[2],d1[3])")); + EXPECT_EQUAL(result["outputs"]["out2"].asString().make_string(), vespalib::string("tensor<float>(d0[2],d1[3])")); + EXPECT_EQUAL(result["outputs"]["out3"].asString().make_string(), vespalib::string("tensor<float>(d0[2],d1[3])")); + EXPECT_EQUAL(f1.shutdown(), 0); +} + +//----------------------------------------------------------------------------- + +TEST_F("test error: invalid json", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { + auto out = f1.write_then_read_all("this is not valid json...\n"); + EXPECT_TRUE(out.find("invalid json") < out.size()); + EXPECT_EQUAL(f1.shutdown(), 3); +} + +TEST_F("test error: missing input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { + Slime params; + params.setObject(); + params.get().setString("model", simple_model); + params.get().setObject("inputs"); + auto out = f1.write_then_read_all(params.toString()); + EXPECT_TRUE(out.find("missing type") < out.size()); + EXPECT_EQUAL(f1.shutdown(), 3); +} + +TEST_F("test error: invalid input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { + Slime params; + params.setObject(); + params.get().setString("model", simple_model); + params.get().setObject("inputs"); + params["inputs"].setString("query_tensor", "bogus type string"); + params["inputs"].setString("attribute_tensor", "tensor<float>(x[4],y[1])"); + params["inputs"].setString("bias_tensor", "tensor<float>(x[1],y[1])"); + auto out = f1.write_then_read_all(params.toString()); + EXPECT_TRUE(out.find("invalid type") < out.size()); + EXPECT_EQUAL(f1.shutdown(), 3); +} + +TEST_F("test error: incompatible input type", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { + Slime params; + params.setObject(); + params.get().setString("model", simple_model); + params.get().setObject("inputs"); + params["inputs"].setString("query_tensor", "tensor<float>(x[1],y[5])"); + params["inputs"].setString("attribute_tensor", "tensor<float>(x[4],y[1])"); + params["inputs"].setString("bias_tensor", "tensor<float>(x[1],y[1])"); + auto out = f1.write_then_read_all(params.toString()); + EXPECT_TRUE(out.find("incompatible type") < out.size()); + EXPECT_EQUAL(f1.shutdown(), 3); +} + +TEST_F("test error: symbolic size mismatch", ServerCmd(probe_cmd, ServerCmd::capture_stderr_tag())) { + Slime params; + params.setObject(); + params.get().setString("model", dynamic_model); + params.get().setObject("inputs"); + params["inputs"].setString("query_tensor", "tensor<float>(x[1],y[4])"); + params["inputs"].setString("attribute_tensor", "tensor<float>(x[4],y[1])"); + params["inputs"].setString("bias_tensor", "tensor<float>(x[2],y[1])"); + auto out = f1.write_then_read_all(params.toString()); + EXPECT_TRUE(out.find("incompatible type") < out.size()); + EXPECT_TRUE(out.find("batch=1") < out.size()); + EXPECT_EQUAL(f1.shutdown(), 3); +} + +//----------------------------------------------------------------------------- + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/apps/eval_expr/CMakeLists.txt b/eval/src/tests/apps/eval_expr/CMakeLists.txt index e36c5aafe23..c36cf33e2b2 100644 --- a/eval/src/tests/apps/eval_expr/CMakeLists.txt +++ b/eval/src/tests/apps/eval_expr/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(eval_eval_expr_test_app TEST eval_expr_test.cpp DEPENDS vespaeval + AFTER + eval_eval_expr_app ) -vespa_add_test(NAME eval_eval_expr_test_app COMMAND eval_eval_expr_test_app - DEPENDS eval_eval_expr_test_app eval_eval_expr_app) +vespa_add_test(NAME eval_eval_expr_test_app COMMAND eval_eval_expr_test_app) diff --git a/eval/src/tests/apps/eval_expr/eval_expr_test.cpp b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp index 077d3d2f35a..fd103829618 100644 --- a/eval/src/tests/apps/eval_expr/eval_expr_test.cpp +++ b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp @@ -23,58 +23,6 @@ vespalib::string server_cmd = binary + " json-repl"; //----------------------------------------------------------------------------- -void read_until_eof(Input &input) { - for (auto mem = input.obtain(); mem.size > 0; mem = input.obtain()) { - input.evict(mem.size); - } -} - -// Output adapter used to write to stdin of a child process -class ChildIn : public Output { - ChildProcess &_child; - SimpleBuffer _output; -public: - ChildIn(ChildProcess &child) : _child(child) {} - WritableMemory reserve(size_t bytes) override { - return _output.reserve(bytes); - } - Output &commit(size_t bytes) override { - _output.commit(bytes); - Memory buf = _output.obtain(); - ASSERT_TRUE(_child.write(buf.data, buf.size)); - _output.evict(buf.size); - return *this; - } -}; - -// Input adapter used to read from stdout of a child process -class ChildOut : public Input { - ChildProcess &_child; - SimpleBuffer _input; -public: - ChildOut(ChildProcess &child) - : _child(child) - { - EXPECT_TRUE(_child.running()); - EXPECT_TRUE(!_child.failed()); - } - Memory obtain() override { - if ((_input.get().size == 0) && !_child.eof()) { - WritableMemory buf = _input.reserve(4_Ki); - uint32_t res = _child.read(buf.data, buf.size); - ASSERT_TRUE((res > 0) || _child.eof()); - _input.commit(res); - } - return _input.obtain(); - } - Input &evict(size_t bytes) override { - _input.evict(bytes); - return *this; - } -}; - -//----------------------------------------------------------------------------- - struct Result { vespalib::string error; vespalib::string result; @@ -106,34 +54,9 @@ struct Result { }; Result::~Result() = default; -void dump_message(const char *prefix, const Slime &slime) { - SimpleBuffer buf; - slime::JsonFormat::encode(slime, buf, true); - auto str = buf.get().make_string(); - fprintf(stderr, "%s%s\n", prefix, str.c_str()); -} - -class Server { -private: - TimeBomb _bomb; - ChildProcess _child; - ChildIn _child_stdin; - ChildOut _child_stdout; -public: - Server() - : _bomb(60), - _child(server_cmd.c_str()), - _child_stdin(_child), - _child_stdout(_child) {} - ~Server(); - Slime invoke(const Slime &req) { - dump_message("request --> ", req); - write_compact(req, _child_stdin); - Slime reply; - ASSERT_TRUE(JsonFormat::decode(_child_stdout, reply)); - dump_message(" reply <-- ", reply); - return reply; - } +struct Server : public ServerCmd { + TimeBomb time_bomb; + Server() : ServerCmd(server_cmd), time_bomb(60) {} Result eval(const vespalib::string &expr, const vespalib::string &name = {}, bool verbose = false) { Slime req; auto &obj = req.setObject(); @@ -147,14 +70,10 @@ public: Slime reply = invoke(req); return {reply.get()}; } + ~Server() { + EXPECT_EQUAL(shutdown(), 0); + } }; -Server::~Server() { - _child.close(); - read_until_eof(_child_stdout); - ASSERT_TRUE(_child.wait()); - ASSERT_TRUE(!_child.running()); - ASSERT_TRUE(!_child.failed()); -} //----------------------------------------------------------------------------- diff --git a/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp b/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp index 4c5b847abc1..6023d472d13 100644 --- a/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp +++ b/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp @@ -7,6 +7,10 @@ using vespalib::Slime; using vespalib::eval::TensorSpec; +auto my_nan = std::numeric_limits<double>::quiet_NaN(); +auto my_neg_inf = (-1.0/0.0); +auto my_inf = (1.0/0.0); + TEST("require that a tensor spec can be converted to and from slime") { TensorSpec spec("tensor(x[2],y{})"); spec.add({{"x", 0}, {"y", "xxx"}}, 1.0) @@ -30,6 +34,17 @@ TEST("require that a tensor spec can be converted to and from an expression") { EXPECT_EQUAL(TensorSpec::from_expr(expr), spec); } +TEST("require that nan/inf/-inf cells get converted to valid expressions") { + TensorSpec spec("tensor<float>(x[2],y{})"); + spec.add({{"x", 0}, {"y", "xxx"}}, my_nan) + .add({{"x", 0}, {"y", "yyy"}}, my_nan) + .add({{"x", 1}, {"y", "xxx"}}, my_neg_inf) + .add({{"x", 1}, {"y", "yyy"}}, my_inf); + vespalib::string expr = spec.to_expr(); + fprintf(stderr, "expr: \n%s\n", expr.c_str()); + EXPECT_EQUAL(TensorSpec::from_expr(expr), spec); +} + TEST("require that tensor specs can be diffed") { TensorSpec expect("tensor(x[2],y{})"); expect.add({{"x", 0}, {"y", "xxx"}}, 1.5) diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index 21f0044faf1..e50c41e2e09 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -2,6 +2,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/int8float.h> +#include <vespa/eval/eval/test/eval_onnx.h> #include <vespa/eval/onnx/onnx_wrapper.h> #include <vespa/eval/onnx/onnx_model_cache.h> #include <vespa/vespalib/util/bfloat16.h> @@ -28,6 +29,7 @@ std::string int_types_model = source_dir + "/int_types.onnx"; std::string guess_batch_model = source_dir + "/guess_batch.onnx"; std::string unstable_types_model = source_dir + "/unstable_types.onnx"; std::string float_to_int8_model = source_dir + "/float_to_int8.onnx"; +std::string probe_model = source_dir + "/probe_model.onnx"; void dump_info(const char *ctx, const std::vector<TensorInfo> &info) { fprintf(stderr, "%s:\n", ctx); @@ -281,7 +283,7 @@ TEST(OnnxTest, int_types_onnx_model_can_be_evaluated) //------------------------------------------------------------------------- } -TEST(OnnxTest, we_guess_batch_dimension_size_when_inference_fails) { +TEST(OnnxTest, we_probe_batch_dimension_size_when_inference_fails) { Onnx model(guess_batch_model, Onnx::Optimize::ENABLE); Onnx::WirePlanner planner_3; Onnx::WirePlanner planner_4; @@ -298,6 +300,13 @@ TEST(OnnxTest, we_guess_batch_dimension_size_when_inference_fails) { EXPECT_TRUE(planner_4.bind_input_type(in_4_type, model.inputs()[0])); EXPECT_TRUE(planner_4.bind_input_type(in_4_type, model.inputs()[1])); + // without model probe + EXPECT_TRUE(planner_3.make_output_type(model.outputs()[0]).is_error()); + EXPECT_TRUE(planner_4.make_output_type(model.outputs()[0]).is_error()); + + // with model probe + planner_3.prepare_output_types(model); + planner_4.prepare_output_types(model); EXPECT_EQ(planner_3.make_output_type(model.outputs()[0]).to_spec(), "tensor<float>(d0[3])"); EXPECT_EQ(planner_4.make_output_type(model.outputs()[0]).to_spec(), "tensor<float>(d0[4])"); @@ -497,4 +506,24 @@ TEST(OnnxModelCacheTest, share_and_evict_onnx_models) { EXPECT_EQ(OnnxModelCache::count_refs(), 0); } +TensorSpec val(const vespalib::string &expr) { + auto result = TensorSpec::from_expr(expr); + EXPECT_FALSE(ValueType::from_spec(result.type()).is_error()); + return result; +} + +TEST(OnnxTest, eval_onnx_with_probe_model) { + Onnx model(probe_model, Onnx::Optimize::ENABLE); + auto in1 = val("tensor<float>( x[2], y[3]):[[ 1, 2, 3],[ 4, 5, 6]]"); + auto in2 = val("tensor<float>( x[2], y[3]):[[ 7, 8, 9],[ 4, 5, 6]]"); + auto out1 = val("tensor<float>(d0[2],d1[3]):[[ 8,10,12],[ 8,10,12]]"); + auto out2 = val("tensor<float>(d0[2],d1[3]):[[-6,-6,-6],[ 0, 0, 0]]"); + auto out3 = val("tensor<float>(d0[2],d1[3]):[[ 7,16,27],[16,25,36]]"); + auto result = test::eval_onnx(model, {in1, in2}); + ASSERT_EQ(result.size(), 3); + EXPECT_EQ(result[0], out1); + EXPECT_EQ(result[1], out2); + EXPECT_EQ(result[2], out3); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx new file mode 100644 index 00000000000..89dab2e7c4c --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx @@ -0,0 +1,30 @@ +probe_model.py:’ + +in1 +in2out1"Add + +in1 +in2out2"Sub + +in1 +in2out3"Mul probe_modelZ# +in1 + + ÿÿÿÿÿÿÿÿÿ +innerZ# +in2 + +outer + ÿÿÿÿÿÿÿÿÿb$ +out1 + + ÿÿÿÿÿÿÿÿÿ +innerb$ +out2 + +outer + ÿÿÿÿÿÿÿÿÿb( +out3 + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿB
\ No newline at end of file diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.py b/eval/src/tests/tensor/onnx_wrapper/probe_model.py new file mode 100755 index 00000000000..529fa23b2b1 --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.py @@ -0,0 +1,35 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +from onnx import helper, TensorProto + +IN1 = helper.make_tensor_value_info('in1', TensorProto.FLOAT, [-1, 'inner']) +IN2 = helper.make_tensor_value_info('in2', TensorProto.FLOAT, ['outer', -1]) +OUT1 = helper.make_tensor_value_info('out1', TensorProto.FLOAT, [-1, 'inner']) +OUT2 = helper.make_tensor_value_info('out2', TensorProto.FLOAT, ['outer', -1]) +OUT3 = helper.make_tensor_value_info('out3', TensorProto.FLOAT, [-1, -1]) + +nodes = [ + helper.make_node( + 'Add', + ['in1', 'in2'], + ['out1'], + ), + helper.make_node( + 'Sub', + ['in1', 'in2'], + ['out2'], + ), + helper.make_node( + 'Mul', + ['in1', 'in2'], + ['out3'], + ), +] +graph_def = helper.make_graph( + nodes, + 'probe_model', + [IN1, IN2], + [OUT1, OUT2, OUT3], +) +model_def = helper.make_model(graph_def, producer_name='probe_model.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'probe_model.onnx') diff --git a/eval/src/vespa/eval/eval/array_array_map.h b/eval/src/vespa/eval/eval/array_array_map.h index 3c070210680..e3601644f3e 100644 --- a/eval/src/vespa/eval/eval/array_array_map.h +++ b/eval/src/vespa/eval/eval/array_array_map.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/allocator.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vector> #include <cassert> @@ -27,8 +28,8 @@ class ArrayArrayMap private: size_t _keys_per_entry; size_t _values_per_entry; - std::vector<K> _keys; - std::vector<V> _values; + std::vector<K, vespalib::allocator_large<K>> _keys; + std::vector<V, vespalib::allocator_large<V>> _values; public: size_t keys_per_entry() const { return _keys_per_entry; } @@ -123,14 +124,9 @@ private: } public: - ArrayArrayMap(size_t keys_per_entry_in, size_t values_per_entry_in, size_t expected_entries) - : _keys_per_entry(keys_per_entry_in), _values_per_entry(values_per_entry_in), _keys(), _values(), - _map(expected_entries * 2, Hash(), Equal(*this)), _hasher() - { - _keys.reserve(_keys_per_entry * expected_entries); - _values.reserve(_values_per_entry * expected_entries); - static_assert(!std::is_pointer_v<K>, "keys cannot be pointers due to auto-deref of alt keys"); - } + ArrayArrayMap(size_t keys_per_entry_in, size_t values_per_entry_in, size_t expected_entries); + ArrayArrayMap(const ArrayArrayMap &) = delete; + ArrayArrayMap & operator = (const ArrayArrayMap &) = delete; ~ArrayArrayMap(); size_t size() const { return _map.size(); } @@ -168,6 +164,16 @@ public: }; template <typename K, typename V, typename H, typename EQ> +ArrayArrayMap<K,V,H,EQ>::ArrayArrayMap(size_t keys_per_entry_in, size_t values_per_entry_in, size_t expected_entries) + : _keys_per_entry(keys_per_entry_in), _values_per_entry(values_per_entry_in), _keys(), _values(), + _map(expected_entries * 2, Hash(), Equal(*this)), _hasher() +{ + _keys.reserve(_keys_per_entry * expected_entries); + _values.reserve(_values_per_entry * expected_entries); + static_assert(!std::is_pointer_v<K>, "keys cannot be pointers due to auto-deref of alt keys"); +} + +template <typename K, typename V, typename H, typename EQ> ArrayArrayMap<K,V,H,EQ>::~ArrayArrayMap() = default; } diff --git a/eval/src/vespa/eval/eval/fast_addr_map.h b/eval/src/vespa/eval/eval/fast_addr_map.h index a3b73afba6d..b9bc39ad619 100644 --- a/eval/src/vespa/eval/eval/fast_addr_map.h +++ b/eval/src/vespa/eval/eval/fast_addr_map.h @@ -7,7 +7,6 @@ #include <vespa/vespalib/util/string_id.h> #include <vespa/vespalib/stllike/identity.h> #include <vespa/vespalib/stllike/hashtable.h> -#include <vector> namespace vespalib::eval { @@ -62,8 +61,8 @@ public: // view able to convert tags into sparse addresses struct LabelView { size_t addr_size; - const std::vector<string_id> &labels; - LabelView(size_t num_mapped_dims, const std::vector<string_id> &labels_in) + const StringIdVector &labels; + LabelView(size_t num_mapped_dims, const StringIdVector &labels_in) : addr_size(num_mapped_dims), labels(labels_in) {} ConstArrayRef<string_id> get_addr(size_t idx) const { return {&labels[idx * addr_size], addr_size}; @@ -105,7 +104,7 @@ private: HashType _map; public: - FastAddrMap(size_t num_mapped_dims, const std::vector<string_id> &labels_in, size_t expected_subspaces) + FastAddrMap(size_t num_mapped_dims, const StringIdVector &labels_in, size_t expected_subspaces) : _labels(num_mapped_dims, labels_in), _map(expected_subspaces * 2, Hash(), Equal(_labels)) {} ~FastAddrMap(); @@ -117,7 +116,7 @@ public: ConstArrayRef<string_id> get_addr(size_t idx) const { return _labels.get_addr(idx); } size_t size() const { return _map.size(); } constexpr size_t addr_size() const { return _labels.addr_size; } - const std::vector<string_id> &labels() const { return _labels.labels; } + const StringIdVector &labels() const { return _labels.labels; } template <typename T> size_t lookup(ConstArrayRef<T> addr, uint32_t hash) const { // assert(addr_size() == addr.size()); diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index 185529b2f51..a0a96bc4497 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -139,7 +139,7 @@ struct FastIterateView : public Value::Index::View { // operations by calling inline functions directly. struct FastValueIndex final : Value::Index { FastAddrMap map; - FastValueIndex(size_t num_mapped_dims_in, const std::vector<string_id> &labels, size_t expected_subspaces_in) + FastValueIndex(size_t num_mapped_dims_in, const StringIdVector &labels, size_t expected_subspaces_in) : map(num_mapped_dims_in, labels, expected_subspaces_in) {} size_t size() const override { return map.size(); } std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const override; @@ -168,29 +168,19 @@ struct FastCells { static constexpr size_t elem_size = sizeof(T); size_t capacity; size_t size; - void *memory; - FastCells(size_t initial_capacity) - : capacity(roundUp2inN(initial_capacity)), - size(0), - memory(malloc(elem_size * capacity)) - { - static_assert(std::is_trivially_copyable_v<T>); - static_assert(can_skip_destruction<T>::value); - } - ~FastCells() { - free(memory); - } + mutable alloc::Alloc memory; + FastCells(size_t initial_capacity); + FastCells(const FastCells &) = delete; + FastCells & operator = (const FastCells &) = delete; + ~FastCells(); void ensure_free(size_t need) { if (__builtin_expect((size + need) > capacity, false)) { - capacity = roundUp2inN(size + need); - void *new_memory = malloc(elem_size * capacity); - memcpy(new_memory, memory, elem_size * size); - free(memory); - memory = new_memory; + reallocate(need); } } + void reallocate(size_t need); constexpr T *get(size_t offset) const { - return reinterpret_cast<T*>(memory) + offset; + return reinterpret_cast<T*>(memory.get()) + offset; } void push_back_fast(T value) { *get(size++) = value; @@ -209,17 +199,38 @@ struct FastCells { } }; +template <typename T> +FastCells<T>::FastCells(size_t initial_capacity) + : capacity(roundUp2inN(initial_capacity)), + size(0), + memory(alloc::Alloc::alloc(elem_size * capacity)) +{ + static_assert(std::is_trivially_copyable_v<T>); + static_assert(can_skip_destruction<T>::value); +} + +template <typename T> +void +FastCells<T>::reallocate(size_t need) { + capacity = roundUp2inN(size + need); + alloc::Alloc new_memory = alloc::Alloc::alloc(elem_size * capacity); + memcpy(new_memory.get(), memory.get(), elem_size * size); + memory = std::move(new_memory); +} + +template <typename T> +FastCells<T>::~FastCells() = default; + //----------------------------------------------------------------------------- template <typename T, bool transient> struct FastValue final : Value, ValueBuilder<T> { - using Handles = typename std::conditional<transient, - std::vector<string_id>, + StringIdVector, SharedStringRepo::Handles>::type; - static const std::vector<string_id> &get_view(const std::vector<string_id> &handles) { return handles; } - static const std::vector<string_id> &get_view(const SharedStringRepo::Handles &handles) { return handles.view(); } + static const StringIdVector &get_view(const StringIdVector &handles) { return handles; } + static const StringIdVector &get_view(const SharedStringRepo::Handles &handles) { return handles.view(); } ValueType my_type; size_t my_subspace_size; @@ -227,14 +238,7 @@ struct FastValue final : Value, ValueBuilder<T> { FastValueIndex my_index; FastCells<T> my_cells; - FastValue(const ValueType &type_in, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in) - : my_type(type_in), my_subspace_size(subspace_size_in), - my_handles(), - my_index(num_mapped_dims_in, get_view(my_handles), expected_subspaces_in), - my_cells(subspace_size_in * expected_subspaces_in) - { - my_handles.reserve(expected_subspaces_in * num_mapped_dims_in); - } + FastValue(const ValueType &type_in, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in); ~FastValue() override; const ValueType &type() const override { return my_type; } const Value::Index &index() const override { return my_index; } @@ -245,7 +249,7 @@ struct FastValue final : Value, ValueBuilder<T> { // not called abort(); } else { - return TypedCells(my_cells.memory, get_cell_type<T>(), my_cells.size); + return TypedCells(my_cells.memory.get(), get_cell_type<T>(), my_cells.size); } } void add_mapping(ConstArrayRef<vespalib::stringref> addr) { @@ -310,7 +314,20 @@ struct FastValue final : Value, ValueBuilder<T> { return usage; } }; -template <typename T,bool transient> FastValue<T,transient>::~FastValue() = default; + +template <typename T,bool transient> +FastValue<T,transient>::FastValue(const ValueType &type_in, size_t num_mapped_dims_in, + size_t subspace_size_in, size_t expected_subspaces_in) + : my_type(type_in), my_subspace_size(subspace_size_in), + my_handles(), + my_index(num_mapped_dims_in, get_view(my_handles), expected_subspaces_in), + my_cells(subspace_size_in * expected_subspaces_in) +{ + my_handles.reserve(expected_subspaces_in * num_mapped_dims_in); +} + +template <typename T,bool transient> +FastValue<T,transient>::~FastValue() = default; //----------------------------------------------------------------------------- @@ -328,7 +345,7 @@ struct FastDenseValue final : Value, ValueBuilder<T> { ~FastDenseValue() override; const ValueType &type() const override { return my_type; } const Value::Index &index() const override { return TrivialIndex::get(); } - TypedCells cells() const override { return TypedCells(my_cells.memory, get_cell_type<T>(), my_cells.size); } + TypedCells cells() const override { return TypedCells(my_cells.memory.get(), get_cell_type<T>(), my_cells.size); } ArrayRef<T> add_subspace(ConstArrayRef<vespalib::stringref>) override { return ArrayRef<T>(my_cells.get(0), my_cells.size); } diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index 43ed724e010..9d0e921c502 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -2,6 +2,7 @@ #include "compile_cache.h" #include <vespa/eval/eval/key_gen.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <thread> namespace vespalib::eval { @@ -64,6 +65,7 @@ CompileCache::compile(const Function &function, PassParams pass_params) assert(res.second); token = std::make_unique<Token>(res.first, Token::ctor_tag()); task = std::make_unique<CompileTask>(function, pass_params, res.first->second.result); + task = CpuUsage::wrap(std::move(task), CpuUsage::Category::SETUP); if (!_executor_stack.empty()) { executor = _executor_stack.back().second; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 512e12bec71..d1a9ffc5b49 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -21,6 +21,8 @@ #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/approx.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/malloc_mmap_guard.h> #include <limits> double vespalib_eval_ldexp(double a, double b) { return std::ldexp(a, b); } @@ -728,6 +730,8 @@ LLVMWrapper::compile(llvm::raw_ostream * dumpStream) // Set relocation model to silence valgrind on CentOS 8 / aarch64 _engine.reset(llvm::EngineBuilder(std::move(_module)).setOptLevel(llvm::CodeGenOpt::Aggressive).setRelocationModel(llvm::Reloc::Static).create()); assert(_engine && "llvm jit not available for your platform"); + + MallocMmapGuard largeAllocsAsMMap(1_Mi); _engine->finalizeObject(); } diff --git a/eval/src/vespa/eval/eval/memory_usage_stuff.h b/eval/src/vespa/eval/eval/memory_usage_stuff.h index 79a4cfb0eda..10f5459b0e7 100644 --- a/eval/src/vespa/eval/eval/memory_usage_stuff.h +++ b/eval/src/vespa/eval/eval/memory_usage_stuff.h @@ -10,8 +10,9 @@ namespace vespalib::eval { template <typename T> MemoryUsage self_memory_usage() { return MemoryUsage(sizeof(T), sizeof(T), 0, 0); } -template <typename T> -MemoryUsage vector_extra_memory_usage(const std::vector<T> &vec) { +template <typename V> +MemoryUsage vector_extra_memory_usage(const V &vec) { + using T = typename V::value_type; MemoryUsage usage; usage.incAllocatedBytes(sizeof(T) * vec.capacity()); usage.incUsedBytes(sizeof(T) * vec.size()); diff --git a/eval/src/vespa/eval/eval/tensor_spec.cpp b/eval/src/vespa/eval/eval/tensor_spec.cpp index 025f3e8c0a3..223e32ce04f 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.cpp +++ b/eval/src/vespa/eval/eval/tensor_spec.cpp @@ -19,6 +19,20 @@ namespace eval { namespace { +vespalib::string number_to_expr(double value) { + if (std::isfinite(value)) { + return make_string("%g", value); + } else if (std::isnan(value)) { + return {"(0/0)"}; + } else { // -inf or inf + if (value < 0) { + return {"(-1/0)"}; + } else { + return {"(1/0)"}; + } + } +} + TensorSpec::Address extract_address(const slime::Inspector &address) { struct Extractor : slime::ObjectTraverser { TensorSpec::Address address; @@ -288,14 +302,14 @@ vespalib::string TensorSpec::to_expr() const { if (_type == "double") { - return make_string("%g", as_double()); + return number_to_expr(as_double()); } vespalib::string out = _type; out.append(":{"); CommaTracker cell_list; for (const auto &cell: _cells) { cell_list.maybe_add_comma(out); - out.append(make_string("%s:%g", as_string(cell.first).c_str(), cell.second.value)); + out.append(make_string("%s:%s", as_string(cell.first).c_str(), number_to_expr(cell.second.value).c_str())); } out.append("}"); return out; diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index e8a291adf2a..ff1505a4010 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_eval_test OBJECT SOURCES cell_type_space.cpp eval_fixture.cpp + eval_onnx.cpp eval_spec.cpp gen_spec.cpp reference_evaluation.cpp diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.cpp b/eval/src/vespa/eval/eval/test/eval_onnx.cpp new file mode 100644 index 00000000000..74a83b130c2 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.cpp @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "eval_onnx.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.eval.test.eval_onnx"); + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms) { + if (params.size() != model.inputs().size()) { + LOG(error, "model with %zu inputs run with %zu parameters", model.inputs().size(), params.size()); + return {}; // wrong number of parameters + } + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + if (!planner.bind_input_type(ValueType::from_spec(params[i].type()), model.inputs()[i])) { + LOG(error, "unable to bind input type: %s -> %s", params[i].type().c_str(), model.inputs()[i].type_as_string().c_str()); + return {}; // inconsistent input types + } + } + planner.prepare_output_types(model); + for (size_t i = 0; i < model.outputs().size(); ++i) { + if (planner.make_output_type(model.outputs()[i]).is_error()) { + LOG(error, "unable to make output type: %s -> error", model.outputs()[i].type_as_string().c_str()); + return {}; // unable to infer/probe output type + } + } + planner.prepare_output_types(model); + auto wire_info = planner.get_wire_info(model); + try { + Onnx::EvalContext context(model, wire_info); + std::vector<Value::UP> inputs; + for (const auto ¶m: params) { + inputs.push_back(value_from_spec(param, FastValueBuilderFactory::get())); + } + for (size_t i = 0; i < model.inputs().size(); ++i) { + context.bind_param(i, *inputs[i]); + } + context.eval(); + std::vector<TensorSpec> results; + for (size_t i = 0; i < model.outputs().size(); ++i) { + results.push_back(spec_from_value(context.get_result(i))); + } + return results; + } catch (const Ort::Exception &ex) { + LOG(error, "model run failed: %s", ex.what()); + return {}; // evaluation failed + } +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.h b/eval/src/vespa/eval/eval/test/eval_onnx.h new file mode 100644 index 00000000000..bb346b7f21e --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.h @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/onnx/onnx_wrapper.h> +#include <vector> + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms); + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/test_io.cpp b/eval/src/vespa/eval/eval/test/test_io.cpp index fc7faef6555..044b6779431 100644 --- a/eval/src/vespa/eval/eval/test/test_io.cpp +++ b/eval/src/vespa/eval/eval/test/test_io.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/util/size_literals.h> +#include <filesystem> #include <unistd.h> #include <assert.h> @@ -13,6 +14,8 @@ using vespalib::WritableMemory; using vespalib::slime::JsonFormat; using vespalib::slime::Cursor; +namespace fs = std::filesystem; + namespace vespalib::eval::test { //----------------------------------------------------------------------------- @@ -63,6 +66,160 @@ StdOut::commit(size_t bytes) //----------------------------------------------------------------------------- +ChildIn::ChildIn(ChildProcess &child) + : _child(child), + _output() +{ +} + +WritableMemory +ChildIn::reserve(size_t bytes) +{ + return _output.reserve(bytes); +} + +Output & +ChildIn::commit(size_t bytes) +{ + _output.commit(bytes); + Memory buf = _output.obtain(); + REQUIRE(_child.write(buf.data, buf.size)); + _output.evict(buf.size); + return *this; +} + +//----------------------------------------------------------------------------- + +ChildOut::ChildOut(ChildProcess &child) + : _child(child), + _input() +{ + REQUIRE(_child.running()); + REQUIRE(!_child.failed()); +} + +Memory +ChildOut::obtain() +{ + if ((_input.get().size == 0) && !_child.eof()) { + WritableMemory buf = _input.reserve(4_Ki); + uint32_t res = _child.read(buf.data, buf.size); + REQUIRE((res > 0) || _child.eof()); + _input.commit(res); + } + return _input.obtain(); +} + +Input & +ChildOut::evict(size_t bytes) +{ + _input.evict(bytes); + return *this; +} + +//----------------------------------------------------------------------------- + +void +ServerCmd::maybe_close() +{ + if (!_closed) { + _child.close(); + _closed = true; + } +} + +void +ServerCmd::maybe_exit() +{ + if (!_exited) { + read_until_eof(_child_stdout); + assert(_child.wait()); + assert(!_child.running()); + _exit_code = _child.getExitCode(); + _exited = true; + } +} + +void +ServerCmd::dump_string(const char *prefix, const vespalib::string &str) +{ + fprintf(stderr, "%s%s: '%s'\n", prefix, _basename.c_str(), str.c_str()); +} + +void +ServerCmd::dump_message(const char *prefix, const Slime &slime) +{ + SimpleBuffer buf; + slime::JsonFormat::encode(slime, buf, false); + auto str = buf.get().make_string(); + fprintf(stderr, "%s%s: %s", prefix, _basename.c_str(), str.c_str()); +} + +ServerCmd::ServerCmd(vespalib::string cmd) + : _child(cmd.c_str()), + _child_stdin(_child), + _child_stdout(_child), + _basename(fs::path(cmd).filename()), + _closed(false), + _exited(false), + _exit_code(31212) +{ +} + +ServerCmd::ServerCmd(vespalib::string cmd, capture_stderr_tag) + : _child(cmd.c_str(), ChildProcess::capture_stderr_tag()), + _child_stdin(_child), + _child_stdout(_child), + _basename(fs::path(cmd).filename()), + _closed(false), + _exited(false), + _exit_code(31212) +{ +} + +ServerCmd::~ServerCmd() +{ + maybe_close(); + maybe_exit(); +} + +Slime +ServerCmd::invoke(const Slime &req) +{ + dump_message("request --> ", req); + write_compact(req, _child_stdin); + Slime reply; + REQUIRE(JsonFormat::decode(_child_stdout, reply)); + dump_message("reply <-- ", reply); + return reply; +} + +vespalib::string +ServerCmd::write_then_read_all(const vespalib::string &input) +{ + vespalib::string result; + dump_string("input --> ", input); + memcpy(_child_stdin.reserve(input.size()).data, input.data(), input.size()); + _child_stdin.commit(input.size()); + maybe_close(); + for (auto mem = _child_stdout.obtain(); mem.size > 0; mem = _child_stdout.obtain()) { + result.append(mem.data, mem.size); + _child_stdout.evict(mem.size); + } + dump_string("output <-- ", result); + return result; +} + +int +ServerCmd::shutdown() +{ + maybe_close(); + maybe_exit(); + return _exit_code; +} + +//----------------------------------------------------------------------------- + bool LineReader::read_line(vespalib::string &line) { @@ -96,6 +253,12 @@ bool look_for_eof(Input &input) { return true; } +void read_until_eof(Input &input) { + for (auto mem = input.obtain(); mem.size > 0; mem = input.obtain()) { + input.evict(mem.size); + } +} + void write_compact(const Slime &slime, Output &out) { JsonFormat::encode(slime, out, true); out.reserve(1).data[0] = '\n'; diff --git a/eval/src/vespa/eval/eval/test/test_io.h b/eval/src/vespa/eval/eval/test/test_io.h index b270b07bb30..62fd6588780 100644 --- a/eval/src/vespa/eval/eval/test/test_io.h +++ b/eval/src/vespa/eval/eval/test/test_io.h @@ -9,6 +9,7 @@ #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/child_process.h> #include <functional> namespace vespalib::eval::test { @@ -39,6 +40,60 @@ public: }; /** + * Output adapter used to write to stdin of a child process. + **/ +class ChildIn : public Output { + ChildProcess &_child; + SimpleBuffer _output; +public: + ChildIn(ChildProcess &child); + WritableMemory reserve(size_t bytes) override; + Output &commit(size_t bytes) override; +}; + +/** + * Input adapter used to read from stdout of a child process. + **/ +class ChildOut : public Input { + ChildProcess &_child; + SimpleBuffer _input; +public: + ChildOut(ChildProcess &child); + Memory obtain() override; + Input &evict(size_t bytes) override; +}; + +/** + * A command run as a child process that acts as a server reading json + * from stdin and writing json to stdout. + **/ +class ServerCmd { +private: + ChildProcess _child; + ChildIn _child_stdin; + ChildOut _child_stdout; + vespalib::string _basename; + bool _closed; + bool _exited; + int _exit_code; + + void maybe_close(); + void maybe_exit(); + + void dump_string(const char *prefix, const vespalib::string &str); + void dump_message(const char *prefix, const Slime &slime); + +public: + struct capture_stderr_tag{}; + ServerCmd(vespalib::string cmd); + ServerCmd(vespalib::string cmd, capture_stderr_tag); + ~ServerCmd(); + Slime invoke(const Slime &req); + vespalib::string write_then_read_all(const vespalib::string &input); + int shutdown(); +}; + +/** * Read one line at a time from an input **/ class LineReader { @@ -55,6 +110,11 @@ public: bool look_for_eof(Input &input); /** + * Read from the input until eof is reached (data is discarded). + **/ +void read_until_eof(Input &input); + +/** * Write a slime structure as compact json with a trailing newline. **/ void write_compact(const Slime &slime, Output &out); diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index 9ddbf41f909..d6a7f54a78e 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -13,8 +13,7 @@ #include <vespa/log/log.h> LOG_SETUP(".vespalib.eval.value_cache.constant_tensor_loader"); -namespace vespalib { -namespace eval { +namespace vespalib::eval { using Inspector = slime::Inspector; using ObjectTraverser = slime::ObjectTraverser; @@ -110,5 +109,4 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin } } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp index 136ff4d6a87..ed38a385d25 100644 --- a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp @@ -75,6 +75,21 @@ struct CreateOnnxTensor { } }; +struct CreateEmptyOnnxTensor { + template <typename T> static Ort::Value invoke(const std::vector<int64_t> &sizes, size_t num_cells, OrtAllocator *alloc) { + auto value = Ort::Value::CreateTensor<T>(alloc, sizes.data(), sizes.size()); + T *cells = value.template GetTensorMutableData<T>(); + for (size_t i = 0; i < num_cells; ++i) { + cells[i] = T{}; + } + return value; + } + Ort::Value operator()(Onnx::ElementType elements, const std::vector<int64_t> &sizes, size_t num_cells, OrtAllocator *alloc) { + return typify_invoke<1,MyTypify,CreateEmptyOnnxTensor>(elements, sizes, num_cells, alloc); + } +}; +CreateEmptyOnnxTensor create_empty_onnx_tensor; + struct CreateVespaTensorRef { template <typename T> static Value::UP invoke(const ValueType &type_ref, Ort::Value &value) { size_t num_cells = type_ref.dense_subspace_size(); @@ -205,6 +220,18 @@ Onnx::TensorInfo make_tensor_info(const OnnxString &name, const Ort::TypeInfo &t return Onnx::TensorInfo{vespalib::string(name.get()), make_dimensions(tensor_info), make_element_type(element_type)}; } +Onnx::TensorType get_type_of(const Ort::Value &value) { + auto tensor_info = value.GetTensorTypeAndShapeInfo(); + auto element_type = tensor_info.GetElementType(); + auto shape = tensor_info.GetShape(); + for (int64_t dim_size: shape) { + if (dim_size < 1) { + throw Ort::Exception("[onnx wrapper] actual value has unknown dimension size", ORT_FAIL); + } + } + return Onnx::TensorType(make_element_type(element_type), shape); +} + std::vector<int64_t> extract_sizes(const ValueType &type) { std::vector<int64_t> sizes; for (const auto &dim: type.dimensions()) { @@ -253,6 +280,58 @@ Onnx::TensorType::type_as_string() const Onnx::WireInfo::~WireInfo() = default; +bool +Onnx::WirePlanner::need_model_probe(const Onnx &model) const +{ + for (const auto &output: model.outputs()) { + for (const auto &dim: output.dimensions) { + if (dim.is_symbolic()) { + if (_symbolic_sizes.find(dim.name) == _symbolic_sizes.end()) { + // symbolic output dimension with unknown size + return true; + } + } else if (dim.value == 0) { + // non-symbolic output dimension with unknown size + return true; + } + } + } + return false; +} + +void +Onnx::WirePlanner::do_model_probe(const Onnx &model) +{ + try { + std::vector<Ort::Value> param_values; + param_values.reserve(model.inputs().size()); + for (const auto &input: model.inputs()) { + const auto &pos = _input_types.find(input.name); + assert(pos != _input_types.end()); + auto vespa_type = pos->second; + auto sizes = extract_sizes(vespa_type); + size_t num_cells = vespa_type.dense_subspace_size(); + param_values.push_back(create_empty_onnx_tensor(input.elements, sizes, num_cells, _alloc)); + } + std::vector<Ort::Value> result_values; + result_values.reserve(model.outputs().size()); + for (size_t i = 0; i < model.outputs().size(); ++i) { + result_values.emplace_back(nullptr); + } + Ort::RunOptions run_opts(nullptr); + Ort::Session &session = const_cast<Ort::Session&>(model._session); + session.Run(run_opts, + model._input_name_refs.data(), param_values.data(), param_values.size(), + model._output_name_refs.data(), result_values.data(), result_values.size()); + for (size_t i = 0; i < model.outputs().size(); ++i) { + _output_types.emplace(model.outputs()[i].name, get_type_of(result_values[i])); + } + } catch (const Ort::Exception &ex) { + _output_types.clear(); + LOG(warning, "model probe failed: %s", ex.what()); + } +} + Onnx::WirePlanner::~WirePlanner() = default; CellType @@ -289,7 +368,6 @@ Onnx::WirePlanner::bind_input_type(const ValueType &vespa_in, const TensorInfo & return false; } } else { - _bound_unknown_sizes.insert(type.dimensions()[i].size); if (dimensions[i].is_symbolic()) { auto &bound_size = _symbolic_sizes[dimensions[i].name]; if (bound_size == 0) { @@ -304,13 +382,39 @@ Onnx::WirePlanner::bind_input_type(const ValueType &vespa_in, const TensorInfo & return true; } +std::map<vespalib::string,size_t> +Onnx::WirePlanner::get_bound_sizes(const TensorInfo &onnx_in) const +{ + std::map<vespalib::string,size_t> result; + for (const auto &dim: onnx_in.dimensions) { + if (dim.is_symbolic()) { + auto pos = _symbolic_sizes.find(dim.name); + if (pos != _symbolic_sizes.end()) { + assert(pos->second != 0); + result.emplace(dim.name, pos->second); + } + } + } + return result; +} + +void +Onnx::WirePlanner::prepare_output_types(const Onnx &model) +{ + if (need_model_probe(model)) { + do_model_probe(model); + } +} + ValueType Onnx::WirePlanner::make_output_type(const TensorInfo &onnx_out) const { const auto &dimensions = onnx_out.dimensions; const auto &elements = onnx_out.elements; + auto probed = _output_types.find(onnx_out.name); std::vector<ValueType::Dimension> dim_list; - for (const auto &dim: dimensions) { + for (size_t i = 0; i < dimensions.size(); ++i) { + const auto &dim = dimensions[i]; size_t dim_size = dim.value; if (dim.is_symbolic()) { auto pos = _symbolic_sizes.find(dim.name); @@ -318,14 +422,23 @@ Onnx::WirePlanner::make_output_type(const TensorInfo &onnx_out) const dim_size = pos->second; } } - // if the output dimension is still unknown, but all unknown - // input dimensions have been bound to the same size, we use - // that size as a guess for the size of the unknown output - // dimension as well. (typical scenario would be batch - // dimension not tagged as having the same symbolic size - // across input and output values). - if ((dim_size == 0) && (_bound_unknown_sizes.size() == 1)) { - dim_size = *_bound_unknown_sizes.begin(); + if (probed != _output_types.end()) { + const auto &type = probed->second; + if (type.dimensions.size() != dimensions.size()) { + LOG(warning, "probed output '%s' does not have the same number of dimensions as " + "the output declared by the model (probed: %zu, declared: %zu)", onnx_out.name.c_str(), + type.dimensions.size(), dimensions.size()); + return ValueType::error_type(); + } + size_t probed_size = type.dimensions[i]; + if (dim_size == 0) { + dim_size = probed_size; + } else if (probed_size != dim_size) { + LOG(warning, "probed dimension size for output '%s' dimension %zu does not match symbolic " + "dimension size inferred from inputs (probed: %zu, inferred: %zu)", onnx_out.name.c_str(), + i, probed_size, dim_size); + return ValueType::error_type(); + } } if ((dim_size == 0) || (dim_list.size() > 9)) { return ValueType::error_type(); @@ -372,8 +485,6 @@ Onnx::WirePlanner::get_wire_info(const Onnx &model) const //----------------------------------------------------------------------------- -Ort::AllocatorWithDefaultOptions Onnx::EvalContext::_alloc; - template <typename T> void Onnx::EvalContext::adapt_param(EvalContext &self, size_t idx, const Value ¶m) @@ -515,6 +626,8 @@ Onnx::EvalContext::get_result(size_t i) const //----------------------------------------------------------------------------- +Ort::AllocatorWithDefaultOptions Onnx::_alloc; + Onnx::Shared::Shared() : _env(ORT_LOGGING_LEVEL_WARNING, "vespa-onnx-wrapper") { diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.h b/eval/src/vespa/eval/onnx/onnx_wrapper.h index bbbbe541145..31421019e3c 100644 --- a/eval/src/vespa/eval/onnx/onnx_wrapper.h +++ b/eval/src/vespa/eval/onnx/onnx_wrapper.h @@ -85,12 +85,17 @@ public: private: std::map<vespalib::string,ValueType> _input_types; std::map<vespalib::string,size_t> _symbolic_sizes; - std::set<size_t> _bound_unknown_sizes; + std::map<vespalib::string,Onnx::TensorType> _output_types; + + bool need_model_probe(const Onnx &model) const; + void do_model_probe(const Onnx &model); public: - WirePlanner() : _input_types(), _symbolic_sizes(), _bound_unknown_sizes() {} + WirePlanner() : _input_types(), _symbolic_sizes(), _output_types() {} ~WirePlanner(); static CellType best_cell_type(Onnx::ElementType type); bool bind_input_type(const ValueType &vespa_in, const TensorInfo &onnx_in); + std::map<vespalib::string,size_t> get_bound_sizes(const TensorInfo &onnx_in) const; + void prepare_output_types(const Onnx &model); ValueType make_output_type(const TensorInfo &onnx_out) const; WireInfo get_wire_info(const Onnx &model) const; }; @@ -103,8 +108,6 @@ public: using param_fun_t = void (*)(EvalContext &, size_t i, const Value &); using result_fun_t = void (*)(EvalContext &, size_t i); - static Ort::AllocatorWithDefaultOptions _alloc; - const Onnx &_model; const WireInfo &_wire_info; Ort::MemoryInfo _cpu_memory; @@ -149,6 +152,8 @@ private: Ort::Env &env() { return _env; } }; + static Ort::AllocatorWithDefaultOptions _alloc; + Shared &_shared; Ort::SessionOptions _options; Ort::Session _session; diff --git a/eval/src/vespa/eval/streamed/streamed_value.h b/eval/src/vespa/eval/streamed/streamed_value.h index ef44eeafce3..a1927e793b2 100644 --- a/eval/src/vespa/eval/streamed/streamed_value.h +++ b/eval/src/vespa/eval/streamed/streamed_value.h @@ -2,10 +2,10 @@ #pragma once +#include "streamed_value_index.h" #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/value.h> #include <vespa/vespalib/util/shared_string_repo.h> -#include "streamed_value_index.h" #include <cassert> namespace vespalib::eval { diff --git a/eval/src/vespa/eval/streamed/streamed_value_index.h b/eval/src/vespa/eval/streamed/streamed_value_index.h index 724874af577..d3f0f2781c4 100644 --- a/eval/src/vespa/eval/streamed/streamed_value_index.h +++ b/eval/src/vespa/eval/streamed/streamed_value_index.h @@ -16,10 +16,10 @@ class StreamedValueIndex : public Value::Index private: uint32_t _num_mapped_dims; uint32_t _num_subspaces; - const std::vector<string_id> &_labels_ref; + const StringIdVector &_labels_ref; public: - StreamedValueIndex(uint32_t num_mapped_dims, uint32_t num_subspaces, const std::vector<string_id> &labels_ref) + StreamedValueIndex(uint32_t num_mapped_dims, uint32_t num_subspaces, const StringIdVector &labels_ref) : _num_mapped_dims(num_mapped_dims), _num_subspaces(num_subspaces), _labels_ref(labels_ref) diff --git a/eval/src/vespa/eval/streamed/streamed_value_utils.h b/eval/src/vespa/eval/streamed/streamed_value_utils.h index efcaf4c6e7a..b808ad3c573 100644 --- a/eval/src/vespa/eval/streamed/streamed_value_utils.h +++ b/eval/src/vespa/eval/streamed/streamed_value_utils.h @@ -13,9 +13,9 @@ namespace vespalib::eval { * Reading more labels than available will trigger an assert. **/ struct LabelStream { - const std::vector<string_id> &source; + const StringIdVector &source; size_t pos; - LabelStream(const std::vector<string_id> &data) : source(data), pos(0) {} + LabelStream(const StringIdVector &data) : source(data), pos(0) {} string_id next_label() { assert(pos < source.size()); return source[pos++]; @@ -42,7 +42,7 @@ private: size_t _num_subspaces; LabelStream _labels; size_t _subspace_index; - std::vector<string_id> _current_address; + StringIdVector _current_address; public: LabelBlock next_block() { if (_subspace_index < _num_subspaces) { @@ -61,7 +61,7 @@ public: } LabelBlockStream(uint32_t num_subspaces, - const std::vector<string_id> &labels, + const StringIdVector &labels, uint32_t num_mapped_dims) : _num_subspaces(num_subspaces), _labels(labels), diff --git a/eval/src/vespa/eval/streamed/streamed_value_view.h b/eval/src/vespa/eval/streamed/streamed_value_view.h index 908176d5cac..53b9e733de5 100644 --- a/eval/src/vespa/eval/streamed/streamed_value_view.h +++ b/eval/src/vespa/eval/streamed/streamed_value_view.h @@ -24,7 +24,7 @@ private: public: StreamedValueView(const ValueType &type, size_t num_mapped_dimensions, TypedCells cells, size_t num_subspaces, - const std::vector<string_id> &labels) + const StringIdVector &labels) : _type(type), _cells_ref(cells), _my_index(num_mapped_dimensions, num_subspaces, labels) diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp index e6972c501f5..56ffbf40eca 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp @@ -13,7 +13,7 @@ const size_t MIN_ALIGNMENT = 0x1000; } void -Fast_BufferedFile::flushWriteBuf(void) +Fast_BufferedFile::flushWriteBuf() { if (_bufi != buf()) { _file->WriteBuf(buf(), _bufi - buf()); @@ -31,7 +31,7 @@ Fast_BufferedFile::flushWriteBuf(void) } void -Fast_BufferedFile::fillReadBuf(void) +Fast_BufferedFile::fillReadBuf() { size_t toread = std::min(static_cast<int64_t>(_buf.size()), _fileleft); if (toread > 0) { @@ -71,20 +71,14 @@ Fast_BufferedFile::addNum(unsigned int num, int fieldw, char fill) } } -uint64_t -Fast_BufferedFile::BytesLeft(void) const -{ - return _fileleft + (_bufe - _bufi); -} - bool -Fast_BufferedFile::Eof(void) const +Fast_BufferedFile::Eof() const { return _fileleft == 0 && _bufi == _bufe; } int64_t -Fast_BufferedFile::GetSize (void) +Fast_BufferedFile::GetSize() { return _file->GetSize(); } @@ -101,41 +95,41 @@ Fast_BufferedFile::SetSize (int64_t s) } bool -Fast_BufferedFile::IsOpened (void) const +Fast_BufferedFile::IsOpened () const { return _file->IsOpened(); } bool -Fast_BufferedFile::Sync(void) +Fast_BufferedFile::Sync() { Flush(); return _file->Sync(); } time_t -Fast_BufferedFile::GetModificationTime(void) +Fast_BufferedFile::GetModificationTime() { time_t retval = _file->GetModificationTime(); return retval; } void -Fast_BufferedFile::EnableDirectIO(void) +Fast_BufferedFile::EnableDirectIO() { _file->EnableDirectIO(); _directIOEnabled = true; } void -Fast_BufferedFile::EnableSyncWrites(void) +Fast_BufferedFile::EnableSyncWrites() { FastOS_FileInterface::EnableSyncWrites(); _file->EnableSyncWrites(); } int64_t -Fast_BufferedFile::GetPosition(void) +Fast_BufferedFile::GetPosition() { if (_file->IsWriteMode()) { int64_t filePosition = _file->GetPosition(); @@ -147,7 +141,7 @@ Fast_BufferedFile::GetPosition(void) void -Fast_BufferedFile::Flush(void) +Fast_BufferedFile::Flush() { if (_file->IsWriteMode()) { flushWriteBuf(); @@ -202,11 +196,9 @@ Fast_BufferedFile::SetPosition(const int64_t s) } const char * -Fast_BufferedFile::GetFileName(void) const +Fast_BufferedFile::GetFileName() const { - return (_file.get() == NULL) - ? "" - : _file->GetFileName(); + return _file ? _file->GetFileName() : ""; } char * @@ -228,19 +220,20 @@ Fast_BufferedFile::ReadLine(char *line, size_t buflen) fillReadBuf(); if (_bufi >= _bufe) { if (p == line) - return NULL; + return nullptr; *p = 0; return line; } continue; } *p++ = *_bufi++; - *p++ = 0; + *p = 0; return line; } } -ssize_t Fast_BufferedFile::Write2(const void * src, size_t srclen) +ssize_t +Fast_BufferedFile::Write2(const void * src, size_t srclen) { const char *p, *pe; p = static_cast<const char *>(src); @@ -295,22 +288,11 @@ Fast_BufferedFile::WriteByte(char byte) *_bufi++ = byte; } -int -Fast_BufferedFile::GetByte(void) -{ - if (_bufi < _bufe) - return *reinterpret_cast<unsigned char *>(_bufi++); - fillReadBuf(); - if (_bufi < _bufe) - return *reinterpret_cast<unsigned char *>(_bufi++); - return -1; -} - void Fast_BufferedFile::ReadOpenExisting(const char *name) { - Close(); - bool ok = _file->OpenReadOnlyExisting(true, name); + bool ok = Close(); + ok &= _file->OpenReadOnlyExisting(true, name); if (!ok) { fprintf(stderr, "ERROR opening %s for read: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -326,8 +308,8 @@ Fast_BufferedFile::ReadOpenExisting(const char *name) void Fast_BufferedFile::ReadOpen(const char *name) { - Close(); - bool ok = _file->OpenReadOnly(name); + bool ok = Close(); + ok &= _file->OpenReadOnly(name); if (!ok) { fprintf(stderr, "ERROR opening %s for read: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -346,8 +328,8 @@ Fast_BufferedFile::ReadOpen(const char *name) void Fast_BufferedFile::WriteOpen(const char *name) { - Close(); - bool ok = _file->OpenWriteOnly(name); + bool ok = Close(); + ok &= _file->OpenWriteOnly(name); if (!ok) { fprintf(stderr, "ERROR opening %s for write: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -378,7 +360,7 @@ namespace { size_t computeBufLen(size_t buflen) { - size_t bitCount(0); + size_t bitCount; for ( bitCount = 1; buflen >> bitCount; bitCount++); buflen = 1 << (bitCount - 1); @@ -394,8 +376,8 @@ Fast_BufferedFile::Fast_BufferedFile(FastOS_FileInterface *file, size_t bufferSi FastOS_FileInterface(), _fileleft(static_cast<uint64_t>(-1)), _buf(vespalib::alloc::Alloc::allocMMap(computeBufLen(bufferSize))), - _bufi(NULL), - _bufe(NULL), + _bufi(nullptr), + _bufe(nullptr), _filepos(0), _directIOEnabled(false), _file(file) @@ -403,20 +385,21 @@ Fast_BufferedFile::Fast_BufferedFile(FastOS_FileInterface *file, size_t bufferSi ResetBuf(); } -Fast_BufferedFile::~Fast_BufferedFile(void) +Fast_BufferedFile::~Fast_BufferedFile() { - Close(); + bool close_ok = Close(); + assert(close_ok); } void -Fast_BufferedFile::ResetBuf(void) +Fast_BufferedFile::ResetBuf() { _bufi = buf(); _bufe = _bufi; } bool -Fast_BufferedFile::Close(void) +Fast_BufferedFile::Close() { Flush(); _openFlags = 0; @@ -426,14 +409,14 @@ Fast_BufferedFile::Close(void) bool Fast_BufferedFile::Open(unsigned int openFlags, const char * name) { - bool ok = false; + bool ok; if (openFlags & FASTOS_FILE_OPEN_READ) { - Close(); + ok = Close(); _filepos = 0; _fileleft = 0; ResetBuf(); - ok = _file->Open(openFlags, name); + ok &= _file->Open(openFlags, name); if (ok) { _openFlags = openFlags; //CASTWARN @@ -442,10 +425,10 @@ bool Fast_BufferedFile::Open(unsigned int openFlags, const char * name) // caller will have to check return value } } else { - Close(); + ok = Close(); _filepos = 0; ResetBuf(); - ok = _file->Open(FASTOS_FILE_OPEN_WRITE | openFlags, name); + ok &= _file->Open(FASTOS_FILE_OPEN_WRITE | openFlags, name); if (ok) { _openFlags = FASTOS_FILE_OPEN_WRITE | openFlags; } else { diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.h b/fastlib/src/vespa/fastlib/io/bufferedfile.h index 014223c9b93..48f90262ad9 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.h +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.h @@ -26,7 +26,6 @@ private: /** True if the file should be read using direct IO */ bool _directIOEnabled; - void setupDirectIOAlign(); char * buf() { return static_cast<char *>(_buf.get()); } protected: /** The file instance used for low-level file access. */ @@ -51,7 +50,7 @@ public: /** * Delete the file instance used for low-level file access. **/ - virtual ~Fast_BufferedFile(void); + virtual ~Fast_BufferedFile(); /** * Open an existing file for reading. * @@ -75,13 +74,13 @@ public: * Reset the internal start and end pointers to the * head of the buffer, thus "emptying" it. */ - void ResetBuf(void); + void ResetBuf(); /** * Write the buffer to the file. Caution: Uses obsolete * FastOS_FileInterface::WriteBuf. * Allocates a 32kB buffer if not previously allocated. */ - void flushWriteBuf(void); + void flushWriteBuf(); /** * Read from the file into the buffer. Allocates a 32kB * buffer if not previously allocated. Fills the buffer, @@ -90,7 +89,7 @@ public: * Caution: If the amount read is smaller than the expected * amount, the method will abort. */ - void fillReadBuf(void); + void fillReadBuf(); /** * Read the next line of the buffered file into a buffer, * reading from the file as necessary. @@ -107,7 +106,7 @@ public: * @param src The source buffer. * @param srclen The length of the source buffer. */ - ssize_t Write2(const void*, size_t) override; + [[nodiscard]] ssize_t Write2(const void*, size_t) override; /** * Write a string to a buffered file, flushing to file * as necessary. @@ -123,7 +122,7 @@ public: * @param dstlen The length of the destination buffer. * @return The number of bytes read. */ - ssize_t Read(void *dst, size_t dstlen) override; + [[nodiscard]] ssize_t Read(void *dst, size_t dstlen) override; /** * Write one byte to the buffered file, flushing to * file if necessary. @@ -131,13 +130,7 @@ public: * @param byte The byte to write. */ void WriteByte(char byte); - /** - * Get one byte from the buffered file, reading from - * the file if necessary. - * - * @return int The byte read, or -1 if not read. - */ - int GetByte(void); + /** * Add an unsigned int number as ASCII text in base 10 to the buffered * file using a fixed width with a designated fill character. @@ -148,22 +141,14 @@ public: * for instance '0' or ' '. */ void addNum(unsigned int num, int fieldw, char fill); - /** - * Get the number of bytes left to read from the buffered - * file. This is the sum of bytes left in the buffer, and - * the number of bytes left in the file that has not yet - * been read into the buffer. - * - * @return The number of bytes left. - */ - uint64_t BytesLeft(void) const; + /** * Test for end of file. * * @return bool True if all bytes have been read from the * buffered file. */ - bool Eof(void) const; + bool Eof() const; /** * Get the size of the file. * @@ -186,7 +171,7 @@ public: /** * Force completion of pending disk writes (flush cache). */ - bool Sync() override; + [[nodiscard]] bool Sync() override; /** * Get the time the file was last modified. * @@ -208,7 +193,7 @@ public: * Flush the buffer, and close the file instance. * @return The result of the Close operation. */ - bool Close () override; + [[nodiscard]] bool Close () override; /** * Get the buffered file position, in bytes. * This takes into account the data in the buffer, that has diff --git a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp b/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp index 4cc386b2d17..8aa9b943419 100644 --- a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp +++ b/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp @@ -4,8 +4,6 @@ TEST("main") { - int i = 0; - int j = 0; int value = 0; FastOS_StatInfo statInfo; @@ -21,8 +19,8 @@ TEST("main") { printf ("testing 11 byte long file\n"); bufFile.WriteOpen("testfile1"); bufFile.addNum(1,10,' '); - bufFile.CheckedWrite("\n",1); - bufFile.Close(); + ASSERT_TRUE(bufFile.CheckedWrite("\n",1)); + ASSERT_TRUE(bufFile.Close()); FastOS_File::Stat("testfile1", &statInfo); if (statInfo._size != 11) { printf (" -- FAILURE\n\n"); @@ -35,8 +33,8 @@ TEST("main") { bufFile.WriteOpen("testfile2"); char buf[8192]; // allocate 8K buffer memset(buf,0xff,8192); - bufFile.CheckedWrite(buf,4095); // write almost 4K - bufFile.Close(); + ASSERT_TRUE(bufFile.CheckedWrite(buf,4095)); // write almost 4K + ASSERT_TRUE(bufFile.Close()); FastOS_File::Stat("testfile2", &statInfo); if (statInfo._size != 4095) { printf (" -- FAILURE\n\n"); @@ -47,8 +45,8 @@ TEST("main") { // test 3 printf ("testing 4096 byte long file\n"); bufFile.WriteOpen("testfile3"); - bufFile.CheckedWrite(buf,4096); // write exactly 4K - bufFile.Close(); + ASSERT_TRUE(bufFile.CheckedWrite(buf,4096)); // write exactly 4K + ASSERT_TRUE(bufFile.Close()); FastOS_File::Stat("testfile3", &statInfo); if (statInfo._size != 4096) { printf (" -- FAILURE\n\n"); @@ -59,8 +57,8 @@ TEST("main") { // test 4 printf ("testing 4097 byte long file\n"); bufFile.WriteOpen("testfile4"); - bufFile.CheckedWrite(buf,4097); // write a bit over 4K - bufFile.Close(); + ASSERT_TRUE(bufFile.CheckedWrite(buf,4097)); // write a bit over 4K + ASSERT_TRUE(bufFile.Close()); FastOS_File::Stat("testfile4", &statInfo); if (statInfo._size != 4097) { printf (" -- FAILURE\n\n"); @@ -71,14 +69,14 @@ TEST("main") { // test 5 printf ("testing 610000 byte long file with repeated addNum\n"); bufFile.WriteOpen("testfile5"); - for (i = 0; i < 10000; i++) { - for (j = 0; j < 10; j++) { + for (int i = 0; i < 10000; i++) { + for (int j = 0; j < 10; j++) { bufFile.addNum(value,6,' '); value++; } - bufFile.CheckedWrite("\n",1); + ASSERT_TRUE(bufFile.CheckedWrite("\n",1)); } - bufFile.Close(); + ASSERT_TRUE(bufFile.Close()); FastOS_File::Stat("testfile5", &statInfo); if (statInfo._size != 610000) { printf (" -- FAILURE\n\n"); diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp index 09ed1ea33ae..80045269c9e 100644 --- a/fastos/src/tests/filetest.cpp +++ b/fastos/src/tests/filetest.cpp @@ -32,7 +32,6 @@ bool createFile(const char* fileName, success = true; } } - cf.Close(); } return success; } @@ -269,8 +268,8 @@ public: ssize_t wroteB = file.Write2(buffer, bufSize); Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); - file.Close(); - + bool close_ok = file.Close(); + assert(close_ok); file.enableMemoryMap(mmap_flags); rc = file.OpenReadOnly(); @@ -323,7 +322,8 @@ public: ssize_t wroteB = file.Write2(buffer, bufSize); Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); - file.Close(); + bool close_ok = file.Close(); + assert(close_ok); if (rc) { file.EnableDirectIO(); @@ -556,7 +556,8 @@ public: if (myFile->OpenExisting()) { Progress(false, "OpenExisting() should not work when '%s' does not exist.", rwFilename.c_str()); - myFile->Close(); + bool close_ok = myFile->Close(); + assert(close_ok); } else { Progress(true, "OpenExisting() should fail when '%s' does not exist, and it did.", rwFilename.c_str()); } @@ -673,7 +674,8 @@ public: int64_t position = file.GetPosition(); Progress(position == 0, "File pointer should be 0 after opening file"); - file.Read(buffer, 4); + ssize_t has_read = file.Read(buffer, 4); + Progress(has_read == 4, "Must read 4 bytes"); buffer[4] = '\0'; position = file.GetPosition(); Progress(position == 4, "File pointer should be 4 after reading 4 bytes"); @@ -684,8 +686,6 @@ public: position = file.GetPosition(); Progress(position == 4, "File pointer should still be 4 after ReadBuf"); Progress(strcmp(buffer, "a test") == 0, "[a test]=[%s]", buffer); - - file.Close(); } PrintSeparator(); diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp index 1382aef7386..4f585e7daeb 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/fastos/src/vespa/fastos/file.cpp @@ -11,6 +11,7 @@ #include <cstring> #include <fcntl.h> #include <cstdlib> +#include <cassert> DirectIOException::DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset) : std::exception(), @@ -34,11 +35,11 @@ int FastOS_FileInterface::_defaultFAdviseOptions = POSIX_FADV_NORMAL; int FastOS_FileInterface::_defaultFAdviseOptions = 0; #endif -static const size_t MAX_WRITE_CHUNK_SIZE = 0x4000000; // 64 MB +static const size_t MAX_CHUNK_SIZE = 0x4000000; // 64 MB FastOS_FileInterface::FastOS_FileInterface(const char *filename) : _fAdviseOptions(_defaultFAdviseOptions), - _writeChunkSize(MAX_WRITE_CHUNK_SIZE), + _chunkSize(MAX_CHUNK_SIZE), _filename(), _openFlags(0), _directIOEnabled(false), @@ -140,13 +141,6 @@ FastOS_FileInterface::EnableDirectIO() void -FastOS_FileInterface::SetWriteChunkSize(size_t writeChunkSize) -{ - _writeChunkSize = writeChunkSize; -} - - -void FastOS_FileInterface::EnableSyncWrites() { if (!IsOpened()) @@ -251,8 +245,8 @@ FastOS_FileInterface::CopyFile( const char *src, const char *dst ) do { unsigned int readBytes = s.Read( tmpBuf, bufSize ); if (readBytes > 0) { - - if ( !d.CheckedWrite( tmpBuf, readBytes)) { + ssize_t written = d.Write2(tmpBuf, readBytes); + if ( written != readBytes) { success = false; } copied += readBytes; @@ -265,8 +259,10 @@ FastOS_FileInterface::CopyFile( const char *src, const char *dst ) delete [] tmpBuf; } // else out of memory ? - s.Close(); - d.Close(); + bool close_ok = s.Close(); + assert(close_ok); + close_ok = d.Close(); + assert(close_ok); } // else Could not open source or destination file. } // else Source file does not exist, or input args are invalid. diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h index 2d83a1766f0..238da364d54 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/fastos/src/vespa/fastos/file.h @@ -84,7 +84,7 @@ private: // And setFAdviseOptions() per file. static int _defaultFAdviseOptions; int _fAdviseOptions; - size_t _writeChunkSize; + size_t _chunkSize; void WriteBufInternal(const void *buffer, size_t length); protected: @@ -309,7 +309,7 @@ public: * already is closed. * @return Boolean success/failure */ - virtual bool Close() = 0; + [[nodiscard]] virtual bool Close() = 0; /** * Is the file currently opened? @@ -324,7 +324,7 @@ public: * @return The number of bytes which was actually read, * or -1 on error. */ - virtual ssize_t Read(void *buffer, size_t length) = 0; + [[nodiscard]] virtual ssize_t Read(void *buffer, size_t length) = 0; /** * Write [len] bytes from [buffer]. This is just a wrapper for @@ -334,7 +334,7 @@ public: * @param len number of bytes to write * @return Boolean success/failure */ - bool CheckedWrite(const void *buffer, size_t len); + [[nodiscard]] bool CheckedWrite(const void *buffer, size_t len); /** * Write [len] bytes from [buffer]. @@ -342,7 +342,7 @@ public: * @param len number of bytes to write * @return The number of bytes actually written, or -1 on error */ - virtual ssize_t Write2(const void *buffer, size_t len) = 0; + [[nodiscard]] virtual ssize_t Write2(const void *buffer, size_t len) = 0; /** * Read [length] bytes into [buffer]. Caution! If the actual number @@ -439,7 +439,7 @@ public: /** * Force completion of pending disk writes (flush cache). */ - virtual bool Sync() = 0; + [[nodiscard]] virtual bool Sync() = 0; /** * Are we in some kind of file read mode? @@ -485,8 +485,8 @@ public: /** * Set the write chunk size used in WriteBuf. */ - void SetWriteChunkSize(size_t writeChunkSize); - size_t getWriteChunkSize() const { return _writeChunkSize; } + void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } + size_t getChunkSize() const { return _chunkSize; } /** * Get restrictions for direct disk I/O. The file should be opened diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/fastos/src/vespa/fastos/linux_file.cpp index 3a59b862a52..1074c02b4ac 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/fastos/src/vespa/fastos/linux_file.cpp @@ -9,13 +9,14 @@ #ifdef __linux__ #include "file.h" +#include "file_rw_ops.h" #include <sstream> #include <unistd.h> #include <fcntl.h> -#include "file_rw_ops.h" #include <cstdio> #include <cstring> #include <system_error> +#include <cassert> using fastos::File_RW_Ops; @@ -29,6 +30,11 @@ FastOS_Linux_File::FastOS_Linux_File(const char *filename) { } +FastOS_Linux_File::~FastOS_Linux_File () { + bool ok = Close(); + assert(ok); +} + #define DIRECTIOPOSSIBLE(buf, len, off) \ ((off & (_directIOFileAlign - 1)) == 0 && \ (len & (_directIOFileAlign - 1)) == 0 && \ @@ -37,14 +43,36 @@ FastOS_Linux_File::FastOS_Linux_File(const char *filename) ssize_t FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, int64_t readOffset) { - return File_RW_Ops::pread(fh, buffer, length, readOffset); + char * data = static_cast<char *>(buffer); + ssize_t has_read(0); + while (has_read < ssize_t(length)) { + size_t lenNow = std::min(getChunkSize(), length - has_read); + ssize_t readNow = File_RW_Ops::pread(fh, data + has_read, lenNow, readOffset + has_read); + if (readNow > 0) { + has_read += readNow; + } else { + return (has_read > 0) ? has_read : readNow; + } + } + return has_read; } ssize_t FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length) { - return File_RW_Ops::read(fh, buffer, length); + char * data = static_cast<char *>(buffer); + ssize_t has_read(0); + while (has_read < ssize_t(length)) { + size_t lenNow = std::min(getChunkSize(), length - has_read); + ssize_t readNow = File_RW_Ops::read(fh, data + has_read, lenNow); + if (readNow > 0) { + has_read += readNow; + } else { + return (has_read > 0) ? has_read : readNow; + } + } + return has_read; } @@ -169,7 +197,7 @@ FastOS_Linux_File::Write2(const void *buffer, size_t length) const char * data = static_cast<const char *>(buffer); ssize_t written(0); while (written < ssize_t(length)) { - size_t lenNow = std::min(getWriteChunkSize(), length - written); + size_t lenNow = std::min(getChunkSize(), length - written); ssize_t writtenNow = internalWrite2(data + written, lenNow); if (writtenNow > 0) { written += writtenNow; @@ -274,17 +302,6 @@ FastOS_Linux_File::AllocateDirectIOBuffer (size_t byteSize, void *&realPtr) return align(realPtr, memoryAlignment); } - -void * -FastOS_Linux_File:: -allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr) -{ - size_t memoryAlignment = _directIOMemAlign; - realPtr = malloc(byteSize + memoryAlignment - 1); - return align(realPtr, memoryAlignment); -} - - size_t FastOS_Linux_File::getMaxDirectIOMemAlign() { @@ -295,18 +312,14 @@ FastOS_Linux_File::getMaxDirectIOMemAlign() bool FastOS_Linux_File::GetDirectIORestrictions (size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) { - bool rc = false; - if (_directIOEnabled) { memoryAlignment = _directIOMemAlign; transferGranularity = _directIOFileAlign; transferMaximum = 0x7FFFFFFF; - rc = true; + return true; } else { - rc = FastOS_UNIX_File::GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); + return FastOS_UNIX_File::GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); } - - return rc; } @@ -376,12 +389,14 @@ FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) if (POSIX_FADV_NORMAL != fadviseOptions) { rc = (posix_fadvise(_filedes, 0, 0, fadviseOptions) == 0); if (!rc) { - Close(); + bool close_ok = Close(); + assert(close_ok); } } } if (rc) { - Sync(); + bool sync_ok = Sync(); + assert(sync_ok); _cachedSize = GetSize(); _filePointer = 0; } @@ -390,7 +405,8 @@ FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) if (rc && (POSIX_FADV_NORMAL != getFAdviseOptions())) { rc = (posix_fadvise(_filedes, 0, 0, getFAdviseOptions()) == 0); if (!rc) { - Close(); + bool close_ok = Close(); + assert(close_ok); } } } diff --git a/fastos/src/vespa/fastos/linux_file.h b/fastos/src/vespa/fastos/linux_file.h index 7a8fc6245bb..f3c66d49f12 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/fastos/src/vespa/fastos/linux_file.h @@ -25,9 +25,7 @@ protected: public: FastOS_Linux_File (const char *filename = nullptr); - ~FastOS_Linux_File () { - Close(); - } + ~FastOS_Linux_File (); bool GetDirectIORestrictions(size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) override; bool DirectIOPadding(int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) override; void EnableDirectIO() override; @@ -38,13 +36,12 @@ public: void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) override; - ssize_t Read(void *buffer, size_t len) override; - ssize_t Write2(const void *buffer, size_t len) override; + [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; + [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; bool Open(unsigned int openFlags, const char *filename) override; static bool InitializeClass(); static size_t getMaxDirectIOMemAlign(); - static void *allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr); static int count_open_files(); private: ssize_t internalWrite2(const void *buffer, size_t len); diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp index 8dd589d5144..39e31c87702 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/fastos/src/vespa/fastos/unix_file.cpp @@ -407,7 +407,7 @@ bool FastOS_UNIX_File::Rename (const char *currentFileName, const char *newFileN } bool -FastOS_UNIX_File::Sync(void) +FastOS_UNIX_File::Sync() { assert(IsOpened()); diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h index c0a908a5948..70a8db5036e 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/fastos/src/vespa/fastos/unix_file.h @@ -58,10 +58,10 @@ public: { } void ReadBuf(void *buffer, size_t length, int64_t readOffset) override; - ssize_t Read(void *buffer, size_t len) override; - ssize_t Write2(const void *buffer, size_t len) override; + [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; + [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; bool Open(unsigned int openFlags, const char *filename) override; - bool Close() override; + [[nodiscard]] bool Close() override; bool IsOpened() const override { return _filedes >= 0; } void enableMemoryMap(int flags) override { @@ -87,7 +87,7 @@ public: int64_t GetSize() override; time_t GetModificationTime() override; bool Delete() override; - bool Sync() override; + [[nodiscard]] bool Sync() override; bool SetSize(int64_t newSize) override; void dropFromCache() const override; diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index 9f8e7d253fc..5ab2d3d7246 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -37,8 +37,11 @@ class FileAcquirerImpl implements FileAcquirer { private static final Logger log = Logger.getLogger(FileAcquirerImpl.class.getName()); private final Supervisor supervisor = new Supervisor(new Transport("fileaquirer")); + + @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSubscriber configSubscriber; + @SuppressWarnings("removal") // TODO Vespa 8: remove private class Connection implements ConfigSubscriber.SingleSubscriber<FiledistributorrpcConfig> { private final Lock targetLock = new ReentrantLock(); private Target target; @@ -122,6 +125,7 @@ class FileAcquirerImpl implements FileAcquirer { } } + @SuppressWarnings("removal") // TODO Vespa 8: remove public FileAcquirerImpl(String configId) { configSubscriber = new ConfigSubscriber(); configSubscriber.subscribe(connection, FiledistributorrpcConfig.class, configId); diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 124703c24b8..e78dec5df9b 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -102,17 +102,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java index 3a03e6a87d5..c292080a0e5 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java @@ -22,6 +22,7 @@ import java.util.List; */ public class FileDistributionConnectionPool extends JRTConnectionPool { + @SuppressWarnings("removal") // TODO Vespa 8: remove public FileDistributionConnectionPool(ConfigSourceSet sourceSet, Supervisor supervisor) { super(sourceSet, supervisor); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index eea0efef05f..7aeaf85a80b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -47,35 +47,35 @@ public class Flags { public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Default limit for when to apply termwise query evaluation", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_TASK_LIMIT = defineIntFlag( "feed-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-10-14", "2022-02-01", + List.of("geirst, baldersheim"), "2021-10-14", "2022-06-01", "The task limit used by the executors handling feed in proton", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_MASTER_TASK_LIMIT = defineIntFlag( "feed-master-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-11-18", "2022-02-01", + List.of("geirst, baldersheim"), "2021-11-18", "2022-06-01", "The task limit used by the master thread in each document db in proton. Ignored when set to 0.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag SHARED_FIELD_WRITER_EXECUTOR = defineStringFlag( "shared-field-writer-executor", "NONE", - List.of("geirst, baldersheim"), "2021-11-05", "2022-02-01", + List.of("geirst, baldersheim"), "2021-11-05", "2022-06-01", "Whether to use a shared field writer executor for the document database(s) in proton. " + "Valid values: NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB", "Takes effect at redeployment (requires restart)", @@ -83,112 +83,90 @@ public class Flags { public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, - List.of("geirst, baldersheim"), "2021-10-21", "2022-02-01", + List.of("geirst, baldersheim"), "2021-10-21", "2022-06-01", "Max amount of memory holding updates to an attribute before we do a commit.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicationmanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the mbus request thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the mbus reply thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2022-02-01", + List.of("vekterli"), "2020-12-02", "2022-05-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( - "hide-shared-routing-endpoint", false, - List.of("tokle", "bjormel"), "2020-12-02", "2022-02-01", - "Whether the controller should hide shared routing layer endpoint", - "Takes effect immediately", - APPLICATION_ID - ); - public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Optionally deliver async messages in own thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "How much concurrency should be allowed for feed", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( - "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2022-01-31", - "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, - List.of("baldersheim"), "2021-09-25", "2022-02-01", + List.of("baldersheim"), "2021-09-25", "2022-06-01", "Will trigger a heap dump during if container shutdown times out", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, - List.of("baldersheim"), "2021-09-25", "2022-02-01", + List.of("baldersheim"), "2021-09-25", "2022-06-01", "Timeout for shutdown of a jdisc container", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2022-02-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2022-06-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); - public static final UnboundBooleanFlag GENERATE_NON_MTLS_ENDPOINT = defineFeatureFlag( - "generate-non-mtls-endpoint", true, - List.of("tokle"), "2021-02-18", "2022-02-01", - "Whether to generate the non-mtls endpoint", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2022-02-01", + List.of("vekterli"), "2021-02-19", "2022-05-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -196,116 +174,54 @@ public class Flags { public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 16, - List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-05-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 100, - List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-05-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag( - "ignore-merge-queue-limit", true, - List.of("vekterli", "geirst"), "2021-10-06", "2022-03-01", - "Specifies if merges that are forwarded (chained) from another content node are always " + - "allowed to be enqueued even if the queue is otherwise full.", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2022-03-01", + List.of("geirst", "vekterli"), "2021-07-16", "2022-05-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag METRICSPROXY_NUM_THREADS = defineIntFlag( "metricsproxy-num-threads", 2, - List.of("balder"), "2021-09-01", "2022-02-01", + List.of("balder"), "2021-09-01", "2022-06-01", "Number of threads for metrics proxy", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag AVAILABLE_PROCESSORS = defineIntFlag( + "available-processors", 2, + List.of("balder"), "2022-01-18", "2022-04-01", + "Number of processors the jvm sees in non-application clusters", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, - List.of("olaa"), "2021-09-13", "2022-02-01", + List.of("olaa"), "2021-09-13", "2022-04-01", "Enable Horizon dashboard", "Takes effect immediately", TENANT_ID, CONSOLE_USER_EMAIL ); - public static final UnboundBooleanFlag ENABLE_ONPREM_TENANT_S3_ARCHIVE = defineFeatureFlag( - "enable-onprem-tenant-s3-archive", false, - List.of("bjorncs"), "2021-09-14", "2022-02-01", - "Enable tenant S3 buckets in cd/main. Must be set on controller cluster only.", - "Takes effect immediately", - ZONE_ID, TENANT_ID - ); - public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, - List.of("andreer"), "2021-09-23", "2022-02-01", + List.of("andreer"), "2021-09-23", "2022-03-14", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" ); - public static final UnboundBooleanFlag USE_NEW_ENDPOINT_CERTIFICATE_PROVIDER_URL = defineFeatureFlag( - "use-new-endpoint-certificate-provider-url", true, - List.of("andreer"), "2021-12-14", "2022-01-14", - "Use the new URL for the endpoint certificate provider API", - "Takes effect immediately" - ); - - public static final UnboundBooleanFlag ENABLE_TENANT_DEVELOPER_ROLE = defineFeatureFlag( - "enable-tenant-developer-role", false, - List.of("bjorncs"), "2021-09-23", "2022-02-01", - "Enable tenant developer Athenz role in cd/main. Must be set on controller cluster only.", - "Takes effect immediately", - TENANT_ID - ); - - public static final UnboundBooleanFlag ENABLE_ROUTING_REUSE_PORT = defineFeatureFlag( - "enable-routing-reuse-port", true, - List.of("mortent"), "2021-09-29", "2022-02-01", - "Enable reuse port in routing configuration", - "Takes effect on container restart", - HOSTNAME - ); - - public static final UnboundBooleanFlag ENABLE_TENANT_OPERATOR_ROLE = defineFeatureFlag( - "enable-tenant-operator-role", false, - List.of("bjorncs"), "2021-09-29", "2022-02-01", - "Enable tenant specific operator roles in public systems. For controllers only.", - "Takes effect on subsequent maintainer invocation", - TENANT_ID - ); - - public static final UnboundIntFlag DISTRIBUTOR_MERGE_BUSY_WAIT = defineIntFlag( - "distributor-merge-busy-wait", 1, - List.of("geirst", "vekterli"), "2021-10-04", "2022-03-01", - "Number of seconds that scheduling of new merge operations in the distributor should be inhibited " + - "towards a content node that has indicated merge busy", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - - public static final UnboundBooleanFlag DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING = defineFeatureFlag( - "distributor-enhanced-maintenance-scheduling", true, - List.of("vekterli", "geirst"), "2021-10-14", "2022-01-31", - "Enable enhanced maintenance operation scheduling semantics on the distributor", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - - public static final UnboundBooleanFlag ASYNC_APPLY_BUCKET_DIFF = defineFeatureFlag( - "async-apply-bucket-diff", true, - List.of("geirst", "vekterli"), "2021-10-22", "2022-01-31", - "Whether portions of apply bucket diff handling will be performed asynchronously", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( "unordered-merge-chaining", true, List.of("vekterli", "geirst"), "2021-11-15", "2022-03-01", @@ -315,7 +231,7 @@ public class Flags { public static final UnboundStringFlag JDK_VERSION = defineStringFlag( "jdk-version", "11", - List.of("hmusum"), "2021-10-25", "2022-03-01", + List.of("hmusum"), "2021-10-25", "2022-03-15", "JDK version to use on host and inside containers. Note application-id dimension only applies for container, " + "while hostname and node type applies for host.", "Takes effect on restart for Docker container and on next host-admin tick for host", @@ -326,7 +242,7 @@ public class Flags { public static final UnboundBooleanFlag IGNORE_THREAD_STACK_SIZES = defineFeatureFlag( "ignore-thread-stack-sizes", false, - List.of("arnej"), "2021-11-12", "2022-01-31", + List.of("arnej"), "2021-11-12", "2022-06-01", "Whether C++ thread creation should ignore any requested stack size", "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); @@ -338,13 +254,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag USE_LEGACY_LB_SERVICES = defineFeatureFlag( - "use-legacy-lb-services", false, - List.of("tokle"), "2021-11-22", "2022-02-01", - "Whether to generate routing table based on legacy lb-services config", - "Takes effect on container reboot", - ZONE_ID, HOSTNAME); - public static final UnboundBooleanFlag USE_V8_DOC_MANAGER_CFG = defineFeatureFlag( "use-v8-doc-manager-cfg", false, List.of("arnej", "baldersheim"), "2021-12-09", "2022-12-31", @@ -361,7 +270,7 @@ public class Flags { public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( "fail-deployment-with-invalid-jvm-options", false, - List.of("hmusum"), "2021-12-20", "2022-01-20", + List.of("hmusum"), "2021-12-20", "2022-03-01", "Whether to fail deployments with invalid JVM options in services.xml", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -381,8 +290,8 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag ZOOKEEPER_SNAPSHOT_METHOD = defineStringFlag( - "zookeeper-snapshot-method", "", - List.of("hmusum"), "2022-01-11", "2022-02-11", + "zookeeper-snapshot-method", "gz", + List.of("hmusum"), "2022-01-11", "2022-03-01", "ZooKeeper snapshot method. Valid values are '', 'gz' and 'snappy'", "Takes effect on Docker container restart", ZONE_ID, APPLICATION_ID, NODE_TYPE); @@ -395,6 +304,52 @@ public class Flags { "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag( + "merge-throttling-policy", "STATIC", + List.of("vekterli"), "2022-01-25", "2022-05-01", + "Sets the policy used for merge throttling on the content nodes. " + + "Valid values: STATIC, DYNAMIC", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR = defineDoubleFlag( + "persistence-throttling-ws-decrement-factor", 1.2, + List.of("vekterli"), "2022-01-27", "2022-05-01", + "Sets the dynamic throttle policy window size decrement factor for persistence " + + "async throttling. Only applies if DYNAMIC policy is used.", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_BACKOFF = defineDoubleFlag( + "persistence-throttling-ws-backoff", 0.95, + List.of("vekterli"), "2022-01-27", "2022-05-01", + "Sets the dynamic throttle policy window size backoff for persistence " + + "async throttling. Only applies if DYNAMIC policy is used. Valid range [0, 1]", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING = defineFeatureFlag( + "inhibit-default-merges-when-global-merges-pending", false, + List.of("geirst", "vekterli"), "2022-02-11", "2022-06-01", + "Inhibits all merges to buckets in the default bucket space if the current " + + "cluster state bundle indicates that global merges are pending in the cluster", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag USE_QRSERVER_SERVICE_NAME = defineFeatureFlag( + "use-qrserver-service-name", true, + List.of("arnej"), "2022-01-18", "2022-12-31", + "Use backwards-compatible 'qrserver' service name for containers with only 'search' API", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag ENABLE_JDISC_PRESHUTDOWN_COMMAND = defineFeatureFlag( + "enable-jdisc-preshutdown-command", false, + List.of("bjorncs", "baldersheim"), "2022-01-31", "2022-05-31", + "Enable pre-shutdown command for jdisc", + "Takes effect at redeployment", + APPLICATION_ID, HOSTNAME, TENANT_ID); + public static final UnboundBooleanFlag AVOID_RENAMING_SUMMARY_FEATURES = defineFeatureFlag( "avoid-renaming-summary-features", false, List.of("arnej"), "2022-01-15", "2023-12-31", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 75d6cab273a..23daf3c4138 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -177,15 +177,15 @@ public class PermanentFlags { APPLICATION_ID); public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( - "resource-limit-disk", 0.8, - "Resource limit (between 0.0 and 1.0) for disk used by cluster controller for when to block feed", + "resource-limit-disk", 0.75, + "Resource limit (between 0.0 and 1.0) for disk usage on content nodes, used by cluster controller for when to block feed", "Takes effect on next deployment", APPLICATION_ID ); public static final UnboundDoubleFlag RESOURCE_LIMIT_MEMORY = defineDoubleFlag( "resource-limit-memory", 0.8, - "Resource limit (between 0.0 and 1.0) for memory used by cluster controller for when to block feed", + "Resource limit (between 0.0 and 1.0) for memory usage on content nodes, used by cluster controller for when to block feed", "Takes effect on next deployment", APPLICATION_ID ); @@ -198,6 +198,14 @@ public class PermanentFlags { APPLICATION_ID, HOSTNAME ); + public static final UnboundListFlag<String> ENVIRONMENT_VARIABLES = defineListFlag( + "environment-variables", List.of(), String.class, + "A list of environment variables set for all services. " + + "Each item should be on the form <ENV_VAR>=<VALUE>", + "Takes effect on service restart", + ZONE_ID, APPLICATION_ID + ); + public static final UnboundStringFlag CONFIG_PROXY_JVM_ARGS = defineStringFlag( "config-proxy-jvm-args", "", "Sets jvm args for config proxy (added at the end of startup command, will override existing ones)", diff --git a/fnet/src/examples/frt/rpc/.gitignore b/fnet/src/examples/frt/rpc/.gitignore index 76876fa7cda..5d952c69238 100644 --- a/fnet/src/examples/frt/rpc/.gitignore +++ b/fnet/src/examples/frt/rpc/.gitignore @@ -14,3 +14,4 @@ fnet_rpc_server_app vespa-rpc-info vespa-rpc-invoke-bin vespa-rpc-proxy +vespa-rpc-invoke diff --git a/functions.cmake b/functions.cmake index 1ec68a4585f..65c02885ddb 100644 --- a/functions.cmake +++ b/functions.cmake @@ -51,6 +51,7 @@ endfunction() function(vespa_add_package_dependency PACKAGE_NAME) find_package(${PACKAGE_NAME} REQUIRED) string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME) + string(REPLACE "-" "_" PACKAGE_NAME ${PACKAGE_NAME}) set(PACKAGE_INCLUDE_DIR ${${PACKAGE_NAME}_INCLUDE_DIR}) set(PACKAGE_LIBRARIES ${${PACKAGE_NAME}_LIBRARIES}) link_libraries(${PACKAGE_LIBRARIES}) @@ -61,6 +62,7 @@ endfunction() function(vespa_add_target_package_dependency TARGET PACKAGE_NAME) find_package(${PACKAGE_NAME} REQUIRED) string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME) + string(REPLACE "-" "_" PACKAGE_NAME ${PACKAGE_NAME}) set(PACKAGE_INCLUDE_DIR ${${PACKAGE_NAME}_INCLUDE_DIR}) set(PACKAGE_LIBRARIES ${${PACKAGE_NAME}_LIBRARIES}) target_link_libraries(${TARGET} PUBLIC ${PACKAGE_LIBRARIES}) diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index e7c364cb7de..7a29ba88d46 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -182,6 +182,25 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-no-log4j</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for apps with log4j deps in compile or provided scope. --> + <excludes> + <exclude>log4j:log4j:*:jar:compile</exclude> + <exclude>log4j:log4j:*:jar:provided</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:compile</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:provided</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> </executions> </plugin> diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index 86dc9f0fbb8..f9ee18a4602 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -47,7 +47,6 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <scope>test</scope> </dependency> </dependencies> <build> @@ -65,17 +64,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 66d912cd987..0da9d907718 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.expressions; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; @@ -33,7 +34,7 @@ public class EmbedExpression extends Expression { @Override public void setStatementOutput(DocumentType documentType, Field field) { - targetType = ((TensorDataType)field.getDataType()).getTensorType(); + targetType = toTargetTensor(field.getDataType()); destination = documentType.getName() + "." + field.getName(); } @@ -52,11 +53,7 @@ public class EmbedExpression extends Expression { if (outputField == null) throw new VerificationException(this, "No output field in this statement: " + "Don't know what tensor type to embed into."); - DataType outputFieldType = context.getInputType(this, outputField); - if ( ! (outputFieldType instanceof TensorDataType) ) - throw new VerificationException(this, "The type of the output field " + outputField + - " is not a tensor but " + outputField); - targetType = ((TensorDataType) outputFieldType).getTensorType(); + targetType = toTargetTensor(context.getInputType(this, outputField)); context.setValueType(createdOutputType()); } @@ -65,6 +62,14 @@ public class EmbedExpression extends Expression { return new TensorDataType(targetType); } + private static TensorType toTargetTensor(DataType dataType) { + if (dataType instanceof ArrayDataType) return toTargetTensor(((ArrayDataType) dataType).getNestedType()); + if ( ! ( dataType instanceof TensorDataType)) + throw new IllegalArgumentException("Expected a tensor data type but got " + dataType); + return ((TensorDataType)dataType).getTensorType(); + + } + @Override public String toString() { return "embed"; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index e7c215383aa..ad5ecba8ff4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -52,28 +52,35 @@ public final class ForEachExpression extends CompositeExpression { @Override protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - if (input instanceof ArrayDataType || input instanceof WeightedSetDataType) { - context.setValueType(((CollectionDataType)input).getNestedType()).execute(exp); - if (input instanceof ArrayDataType) { + DataType valueType = context.getValueType(); + if (valueType instanceof ArrayDataType || valueType instanceof WeightedSetDataType) { + // Set type for block evaluation + context.setValueType(((CollectionDataType)valueType).getNestedType()); + + // Evaluate block, which sets value>Type to the output of the block + context.execute(exp); + + // Value type outside block becomes the collection type having the block output type as argument + if (valueType instanceof ArrayDataType) { context.setValueType(DataType.getArray(context.getValueType())); } else { - WeightedSetDataType wset = (WeightedSetDataType)input; + WeightedSetDataType wset = (WeightedSetDataType)valueType; context.setValueType(DataType.getWeightedSet(context.getValueType(), wset.createIfNonExistent(), wset.removeIfZero())); } - } else if (input instanceof StructDataType) { - for (Field field : ((StructDataType)input).getFields()) { + } + else if (valueType instanceof StructDataType) { + for (Field field : ((StructDataType)valueType).getFields()) { DataType fieldType = field.getDataType(); - DataType valueType = context.setValueType(fieldType).execute(exp).getValueType(); - if (!fieldType.isAssignableFrom(valueType)) { + DataType structValueType = context.setValueType(fieldType).execute(exp).getValueType(); + if (!fieldType.isAssignableFrom(structValueType)) throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " + - valueType.getName() + "."); - } + structValueType.getName() + "."); } - context.setValueType(input); - } else { + context.setValueType(valueType); + } + else { throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " + - input.getName() + "."); + valueType.getName() + "."); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java new file mode 100644 index 00000000000..dd8aadecb33 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.nio.charset.StandardCharsets; + +/** + * Hashes a string value to a long or int (by type inference on the target value). + * + * @author bratseth + */ +public class HashExpression extends Expression { + + private final HashFunction hasher = Hashing.sipHash24(); + + /** The target *primitive* type we are hashing into. */ + private DataType targetType; + + public HashExpression() { + super(DataType.STRING); + } + + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + if ( ! canStoreHash(field.getDataType())) + throw new IllegalArgumentException("Cannot use the hash function on an indexing statement for " + + field.getName() + + ": The hash function can only be used when the target field " + + "is int or long or an array of int or long, not " + field.getDataType()); + targetType = primitiveTypeOf(field.getDataType()); + } + + @Override + protected void doExecute(ExecutionContext context) { + StringFieldValue input = (StringFieldValue) context.getValue(); + if (targetType.equals(DataType.INT)) + context.setValue(new IntegerFieldValue(hashToInt(input.getString()))); + else if (targetType.equals(DataType.LONG)) + context.setValue(new LongFieldValue(hashToLong(input.getString()))); + else + throw new IllegalStateException(); // won't happen + } + + private int hashToInt(String value) { + return hasher.hashString(value, StandardCharsets.UTF_8).asInt(); + } + + private long hashToLong(String value) { + return hasher.hashString(value, StandardCharsets.UTF_8).asLong(); + } + + @Override + protected void doVerify(VerificationContext context) { + String outputField = context.getOutputField(); + if (outputField == null) + throw new VerificationException(this, "No output field in this statement: " + + "Don't know what value to hash to."); + DataType outputFieldType = context.getInputType(this, outputField); + if ( ! canStoreHash(outputFieldType)) + throw new VerificationException(this, "The type of the output field " + outputField + + " is not int or long but " + outputFieldType); + targetType = primitiveTypeOf(outputFieldType); + context.setValueType(createdOutputType()); + } + + private boolean canStoreHash(DataType type) { + if (type.equals(DataType.INT)) return true; + if (type.equals(DataType.LONG)) return true; + if (type instanceof ArrayDataType) return canStoreHash(((ArrayDataType)type).getNestedType()); + return false; + } + + private static DataType primitiveTypeOf(DataType type) { + if (type instanceof ArrayDataType) return ((ArrayDataType)type).getNestedType(); + return type; + } + + @Override + public DataType createdOutputType() { return targetType; } + + @Override + public String toString() { return "hash"; } + + @Override + public int hashCode() { return 987; } + + @Override + public boolean equals(Object o) { return o instanceof HashExpression; } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java index 5e7288b8ecc..ca2be7c3400 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java @@ -13,6 +13,7 @@ public final class HexEncodeExpression extends Expression { public HexEncodeExpression() { super(DataType.LONG); } + @Override protected void doExecute(ExecutionContext context) { long input = ((LongFieldValue) context.getValue()).getLong(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index c266a0da430..40aa0f58413 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -48,7 +48,8 @@ public final class StatementExpression extends ExpressionList<Expression> { if (expression instanceof OutputExpression) outputField = ((OutputExpression)expression).getFieldName(); } - context.setOutputField(outputField); + if (outputField != null) + context.setOutputField(outputField); for (Expression expression : this) context.execute(expression); } diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index bdbecadecd3..e6b21f7c07b 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -164,6 +164,7 @@ TOKEN : <GET_FIELD: "get_field"> | <GET_VAR: "get_var"> | <GUARD: "guard"> | + <HASH: "hash"> | <HEX_DECODE: "hexdecode"> | <HEX_ENCODE: "hexencode"> | <HOST_NAME: "hostname"> | @@ -283,13 +284,14 @@ Expression value() : val = base64EncodeExp() | val = clearStateExp() | val = echoExp() | - val = embedExp() | + val = embedExp() | val = exactExp() | val = flattenExp() | val = forEachExp() | val = getFieldExp() | val = getVarExp() | val = guardExp() | + val = hashExp() | val = hexDecodeExp() | val = hexEncodeExp() | val = hostNameExp() | @@ -419,6 +421,12 @@ Expression guardExp() : { return new GuardExpression(val); } } +Expression hashExp() : { } +{ + ( <HASH> ) + { return new HashExpression(); } +} + Expression hexDecodeExp() : { } { ( <HEX_DECODE> ) @@ -744,12 +752,13 @@ String identifier() : <ECHO> | <EXACT> | <ELSE> | - <EMBED> | + <EMBED> | <FLATTEN> | <FOR_EACH> | <GET_FIELD> | <GET_VAR> | <GUARD> | + <HASH> | <HEX_DECODE> | <HEX_ENCODE> | <HOST_NAME> | diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index f9a6f2225b3..27723c6649d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.tensor.Tensor; @@ -100,6 +102,78 @@ public class ScriptTestCase { } @Test + public void testIntHash() throws ParseException { + var expression = Expression.fromString("input myText | hash | attribute 'myInt'"); + + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myText", DataType.STRING)); + var intField = new Field("myInt", DataType.INT); + adapter.createField(intField); + adapter.setValue("myText", new StringFieldValue("input text")); + expression.setStatementOutput(new DocumentType("myDocument"), intField); + + // Necessary to resolve output type + VerificationContext verificationContext = new VerificationContext(adapter); + assertEquals(DataType.INT, expression.verify(verificationContext)); + + ExecutionContext context = new ExecutionContext(adapter); + context.setValue(new StringFieldValue("input text")); + expression.execute(context); + assertTrue(adapter.values.containsKey("myInt")); + assertEquals(-1425622096, adapter.values.get("myInt").getWrappedValue()); + } + + @SuppressWarnings("unchecked") + @Test + public void testIntArrayHash() throws ParseException { + var expression = Expression.fromString("input myTextArray | for_each { hash } | attribute 'myIntArray'"); + + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myTextArray", new ArrayDataType(DataType.STRING))); + var intField = new Field("myIntArray", new ArrayDataType(DataType.INT)); + adapter.createField(intField); + var array = new Array<StringFieldValue>(new ArrayDataType(DataType.STRING)); + array.add(new StringFieldValue("first")); + array.add(new StringFieldValue("second")); + adapter.setValue("myTextArray", array); + expression.setStatementOutput(new DocumentType("myDocument"), intField); + + // Necessary to resolve output type + VerificationContext verificationContext = new VerificationContext(adapter); + assertEquals(new ArrayDataType(DataType.INT), expression.verify(verificationContext)); + + ExecutionContext context = new ExecutionContext(adapter); + context.setValue(array); + expression.execute(context); + assertTrue(adapter.values.containsKey("myIntArray")); + var intArray = (Array<IntegerFieldValue>)adapter.values.get("myIntArray"); + assertEquals( 368658787, intArray.get(0).getInteger()); + assertEquals(-1382874952, intArray.get(1).getInteger()); + } + + @Test + public void testLongHash() throws ParseException { + var expression = Expression.fromString("input myText | hash | attribute 'myLong'"); + + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myText", DataType.STRING)); + var intField = new Field("myLong", DataType.LONG); + adapter.createField(intField); + adapter.setValue("myText", new StringFieldValue("input text")); + expression.setStatementOutput(new DocumentType("myDocument"), intField); + + // Necessary to resolve output type + VerificationContext verificationContext = new VerificationContext(adapter); + assertEquals(DataType.LONG, expression.verify(verificationContext)); + + ExecutionContext context = new ExecutionContext(adapter); + context.setValue(new StringFieldValue("input text")); + expression.execute(context); + assertTrue(adapter.values.containsKey("myLong")); + assertEquals(7678158186624760752L, adapter.values.get("myLong").getWrappedValue()); + } + + @Test public void testEmbed() throws ParseException { TensorType tensorType = TensorType.fromSpec("tensor(d[4])"); var expression = Expression.fromString("input myText | embed | attribute 'myTensor'", @@ -120,12 +194,44 @@ public class ScriptTestCase { ExecutionContext context = new ExecutionContext(adapter); context.setValue(new StringFieldValue("input text")); expression.execute(context); - assertNotNull(context); assertTrue(adapter.values.containsKey("myTensor")); assertEquals(Tensor.from(tensorType, "[7,3,0,0]"), ((TensorFieldValue)adapter.values.get("myTensor")).getTensor().get()); } + @SuppressWarnings("unchecked") + @Test + public void testArrayEmbed() throws ParseException { + TensorType tensorType = TensorType.fromSpec("tensor(d[4])"); + var expression = Expression.fromString("input myTextArray | for_each { embed } | attribute 'myTensorArray'", + new SimpleLinguistics(), + new MockEmbedder("myDocument.myTensorArray")); + + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myTextArray", new ArrayDataType(DataType.STRING))); + + var tensorField = new Field("myTensorArray", new ArrayDataType(new TensorDataType(tensorType))); + adapter.createField(tensorField); + + var array = new Array<StringFieldValue>(new ArrayDataType(DataType.STRING)); + array.add(new StringFieldValue("first")); + array.add(new StringFieldValue("second")); + adapter.setValue("myTextArray", array); + expression.setStatementOutput(new DocumentType("myDocument"), tensorField); + + // Necessary to resolve output type + VerificationContext verificationContext = new VerificationContext(adapter); + assertEquals(new ArrayDataType(new TensorDataType(tensorType)), expression.verify(verificationContext)); + + ExecutionContext context = new ExecutionContext(adapter); + context.setValue(array); + expression.execute(context); + assertTrue(adapter.values.containsKey("myTensorArray")); + var tensorArray = (Array<TensorFieldValue>)adapter.values.get("myTensorArray"); + assertEquals(Tensor.from(tensorType, "[7,3,0,0]"), tensorArray.get(0).getTensor().get()); + assertEquals(Tensor.from(tensorType, "[7,3,0,0]"), tensorArray.get(1).getTensor().get()); + } + private static class MockEmbedder implements Embedder { private final String expectedDestination; diff --git a/integration/intellij/README.md b/integration/intellij/README.md index fec547f8a06..d293b320028 100644 --- a/integration/intellij/README.md +++ b/integration/intellij/README.md @@ -36,17 +36,19 @@ open a project with some sd file and see how the plugin works on it. ## Some useful links: -1. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and - https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html +1. Plugin development documentation: https://plugins.jetbrains.com/docs/intellij/welcome.html -2. Grammar-Kit HOWTO: Helps to understand the BNF syntax. +2. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and + https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html + +3. Grammar-Kit HOWTO: Helps to understand the BNF syntax. https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md -3. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here: +4. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here: https://intellij-support.jetbrains.com/hc/en-us/community/posts/360001258300-What-s-the-alternative-to-left-recursion-in-GrammarKit- -4. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer): +5. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer): https://medium.com/@shan1024/custom-language-plugin-development-for-intellij-idea-part-01-d6a41ab96bc9 -5. Code of Dart (some custom language) plugin for IntelliJ: +6. Code of Dart (some custom language) plugin for IntelliJ: https://github.com/JetBrains/intellij-plugins/tree/0f07ca63355d5530b441ca566c98f17c560e77f8/Dart
\ No newline at end of file diff --git a/integration/intellij/build.gradle b/integration/intellij/build.gradle index 2d5ad1f33d6..5529d3d7c15 100644 --- a/integration/intellij/build.gradle +++ b/integration/intellij/build.gradle @@ -36,7 +36,7 @@ compileJava { } group 'ai.vespa' -version '1.0.2' // Also update pom.xml version if this is changed +version '1.0.3' // Also update pom.xml version if this is changed sourceCompatibility = 11 diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml index 984968b2834..94d8fd90f99 100644 --- a/integration/intellij/pom.xml +++ b/integration/intellij/pom.xml @@ -9,7 +9,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> - <version>1.0.2</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> + <version>1.0.3</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> <description> Maven wrapper for the gradle build of this IntelliJ plugin. </description> diff --git a/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf index 6be7302a28c..04368bf29af 100644 --- a/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf +++ b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf @@ -40,7 +40,7 @@ ] } -SdFile ::= SchemaDefinition | DocumentDefinition +SdFile ::= SchemaDefinition | DocumentDefinition | RankProfileDefinition SchemaDefinition ::= (search | schema) IdentifierVal? (inherits IdentifierVal)? '{' SchemaBody '}' SchemaBody ::= SchemaBodyOptions* DocumentDefinition SchemaBodyOptions* // Does not support zero-or-one occurrences private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition | @@ -129,7 +129,7 @@ PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | Ra //------------------------- //-- Rank Profile rules --- //------------------------- -RankProfileDefinition ::= (rank-profile | model) IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' RankProfileBody '}' +RankProfileDefinition ::= (rank-profile | model) IdentifierWithDashVal (inherits IdentifierWithDashVal (',' IdentifierWithDashVal)*)? '{' RankProfileBody '}' { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" implements=["ai.vespa.intellij.schema.psi.SdDeclaration"] } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java index d9f769d07cc..cba1bd04ce4 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java @@ -132,7 +132,8 @@ public class SdUtil { for (VirtualFile vfile : virtualFiles) { SdFile sdFile = (SdFile) PsiManager.getInstance(project).findFile(vfile); - if (sdFile != null && !sdFile.getName().equals(docName + ".sd")) { + if (sdFile != null && + ( !sdFile.getName().equals(docName + ".sd") && !sdFile.getName().equals(docName + ".profile"))) { continue; } result.addAll(SdUtil.findDeclarationsByName(sdFile, name)); diff --git a/integration/intellij/src/main/resources/META-INF/plugin.xml b/integration/intellij/src/main/resources/META-INF/plugin.xml index 673a66f3228..ea6fe3f2f15 100644 --- a/integration/intellij/src/main/resources/META-INF/plugin.xml +++ b/integration/intellij/src/main/resources/META-INF/plugin.xml @@ -29,7 +29,7 @@ <!-- Extension points defined by the plugin --> <extensions defaultExtensionNs="com.intellij"> <fileType name="Sd File" implementationClass="ai.vespa.intellij.schema.SdFileType" fieldName="INSTANCE" - language="Sd" extensions="sd"/> + language="Sd" extensions="sd;profile"/> <lang.parserDefinition language="Sd" implementationClass="ai.vespa.intellij.schema.parser.SdParserDefinition"/> <lang.syntaxHighlighterFactory language="Sd" implementationClass="ai.vespa.intellij.schema.SdSyntaxHighlighterFactory"/> <completion.contributor language="Sd" implementationClass="ai.vespa.intellij.schema.SdCompletionContributor"/> diff --git a/jaxrs_client_utils/pom.xml b/jaxrs_client_utils/pom.xml index 7182f2c53cc..cd83e4c8713 100644 --- a/jaxrs_client_utils/pom.xml +++ b/jaxrs_client_utils/pom.xml @@ -25,6 +25,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>application-model</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java index 8b35935f0fa..bd498dc02df 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java @@ -10,14 +10,14 @@ public interface JaxRsTimeouts { /** * The connect timeout, which must be at least 1ms. Called once per real REST call. * - * Throws com.google.common.util.concurrent.UncheckedTimeoutException on timeout. + * @throws com.yahoo.concurrent.UncheckedTimeoutException on timeout. */ Duration getConnectTimeoutOrThrow(); /** * The read timeout, which must be at least 1ms. Called once per real REST call. * - * Throws com.google.common.util.concurrent.UncheckedTimeoutException on timeout. + * @throws com.yahoo.concurrent.UncheckedTimeoutException on timeout. */ Duration getReadTimeoutOrThrow(); } diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java index 387088d10b4..299bf2facc5 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java @@ -4,14 +4,14 @@ package com.yahoo.jdisc.cloud.aws; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.cloud.aws.AwsParameterStore.AwsSettings; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; -import com.yahoo.jdisc.cloud.aws.AwsParameterStore.AwsSettings; import java.io.IOException; import java.io.InputStream; @@ -25,7 +25,7 @@ import java.util.logging.Logger; * * @author ogronnesby */ -public class AwsParameterStoreValidationHandler extends LoggingRequestHandler { +public class AwsParameterStoreValidationHandler extends ThreadedHttpRequestHandler { private static final Logger log = Logger.getLogger(AwsParameterStoreValidationHandler.class.getName()); diff --git a/jdisc_core/abi-spec.json b/jdisc_core/abi-spec.json index d1b676b330f..06e76f88898 100644 --- a/jdisc_core/abi-spec.json +++ b/jdisc_core/abi-spec.json @@ -32,7 +32,7 @@ ], "methods": [ "public abstract com.yahoo.jdisc.handler.RequestHandler resolveHandler(com.yahoo.jdisc.Request)", - "public abstract java.lang.Object getInstance(com.google.inject.Key)", + "public java.lang.Object getInstance(com.google.inject.Key)", "public abstract java.lang.Object getInstance(java.lang.Class)" ], "fields": [] diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java index a131fc557c4..cabadafa8a0 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java @@ -5,7 +5,6 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.refcount.DebugReferencesByContextMap; import com.yahoo.jdisc.refcount.DebugReferencesWithStack; import com.yahoo.jdisc.refcount.DestructableResource; -import com.yahoo.jdisc.refcount.ReferencesByCount; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.jdisc.refcount.References; @@ -25,12 +24,10 @@ public abstract class AbstractResource implements SharedResource { protected AbstractResource() { DestructableResource destructable = new WrappedResource(this); - if (debug == Debug.SIMPLE) { - references = new DebugReferencesByContextMap(destructable, this); - } else if (debug == Debug.STACK) { + if (debug == Debug.STACK) { references = new DebugReferencesWithStack(destructable); } else { - references = new ReferencesByCount(destructable); + references = new DebugReferencesByContextMap(destructable, this); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java index f3ed16c65e7..e96f7f08fe8 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java @@ -49,8 +49,10 @@ public interface Container extends SharedResource, Timer { * @return The appropriate instance of the given class. * @throws ConfigurationException If this injector cannot find or create the provider. * @throws ProvisionException If there was a runtime failure while providing an instance. + * @deprecated Use {@link #getInstance(Class)} instead. */ - <T> T getInstance(Key<T> key); + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove + default <T> T getInstance(Key<T> key) { throw new UnsupportedOperationException(); } /** * Returns the appropriate instance for the given injection type. When feasible, avoid using this method in diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/SharedResource.java b/jdisc_core/src/main/java/com/yahoo/jdisc/SharedResource.java index 051cebee465..654402d181c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/SharedResource.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/SharedResource.java @@ -39,7 +39,7 @@ public interface SharedResource { return Debug.valueOf(val); } catch (IllegalArgumentException e) { } } - return Debug.NO; + return Debug.SIMPLE; } /** diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java index 4d9a843b8fb..1ddf91aee2b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java @@ -29,7 +29,6 @@ import java.util.regex.Pattern; */ public class UriPattern implements Comparable<UriPattern> { - public static final int DEFAULT_PRIORITY = 0; private static final Pattern PATTERN = Pattern.compile("([^:]+)://([^:/]+)(:((\\*)|([0-9]+)))?/(.*)", Pattern.UNICODE_CASE | Pattern.CANON_EQ); private final String pattern; @@ -38,9 +37,6 @@ public class UriPattern implements Comparable<UriPattern> { private final int port; private final GlobPattern path; - // TODO Vespa 8 jonmv remove - private final int priority; - /** * <p>Creates a new instance of this class that represents the given pattern string, with a priority of <code>0</code>. * The input string must be on the form <code><scheme>://<host>[:<port>]<path></code>, where @@ -59,31 +55,6 @@ public class UriPattern implements Comparable<UriPattern> { port = parseOrZero(matcher.group(4)); path = GlobPattern.compile(nonNullOrWildcard(matcher.group(7))); pattern = scheme + "://" + host + ":" + (port > 0 ? port : "*") + "/" + path; - this.priority = DEFAULT_PRIORITY; - } - - /** - * <p>Creates a new instance of this class that represents the given pattern string, with the given priority. The - * input string must be on the form <code><scheme>://<host>[:<port>]<path></code>, where - * '*' can be used as a wildcard character at any position.</p> - * - * @deprecated Use {@link #UriPattern(String)} and let's avoid another complication here. - * @param uri The pattern to parse. - * @param priority The priority of this pattern. - * @throws IllegalArgumentException If the pattern could not be parsed. - */ - @Deprecated(forRemoval = true, since = "7") - public UriPattern(String uri, int priority) { - Matcher matcher = PATTERN.matcher(uri); - if (!matcher.find()) { - throw new IllegalArgumentException(uri); - } - scheme = GlobPattern.compile(normalizeScheme(nonNullOrWildcard(matcher.group(1)))); - host = GlobPattern.compile(nonNullOrWildcard(matcher.group(2))); - port = parseOrZero(matcher.group(4)); - path = GlobPattern.compile(nonNullOrWildcard(matcher.group(7))); - pattern = scheme + "://" + host + ":" + (port > 0 ? port : "*") + "/" + path; - this.priority = priority; } /** @@ -135,10 +106,6 @@ public class UriPattern implements Comparable<UriPattern> { @Override public int compareTo(UriPattern rhs) { int cmp; - cmp = rhs.priority - priority; - if (cmp != 0) { - return cmp; - } cmp = scheme.compareTo(rhs.scheme); if (cmp != 0) { return cmp; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java index f3641f2475b..808c8e89b1b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java @@ -37,7 +37,10 @@ class ContainerSnapshot extends AbstractResource implements Container { this.containerReference = container.refer(context); } + /** @deprecated Use {@link #getInstance(Class)} instead. */ @Override + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove + @SuppressWarnings("removal") public <T> T getInstance(Key<T> key) { return container.guiceInjector().getInstance(key); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/ReferencesByCount.java b/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/ReferencesByCount.java deleted file mode 100644 index 0f417c81a8b..00000000000 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/ReferencesByCount.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.refcount; - -import com.yahoo.jdisc.ResourceReference; -import com.yahoo.jdisc.SharedResource; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Does reference counting by using atomic counting of references - * Default in production - * - * @author baldersheim - */ -public class ReferencesByCount implements References { - private final AtomicInteger refCount; - private final DestructableResource resource; - private final NoDebugResourceReference initialReference; - - public ReferencesByCount(DestructableResource resource) { - refCount = new AtomicInteger(1); - this.resource = resource; - initialReference = new NoDebugResourceReference(this); - } - - @Override - public void release() { - initialReference.close(); - } - - @Override - public int referenceCount() { - return refCount.get(); - } - - @Override - public ResourceReference refer(Object context) { - addRef(1); - return new NoDebugResourceReference(this); - } - - @Override - public String currentState() { - return "Active references: " + refCount.get() + "." - + " Resource reference debugging is turned off. Consider toggling the " - + SharedResource.SYSTEM_PROPERTY_NAME_DEBUG - + " system property to get debugging assistance with reference tracking."; - } - - private void removeRef() { - int refCount = addRef(-1); - if (refCount == 0) { - resource.close(); - } - } - - private int addRef(int value) { - while (true) { - int prev = refCount.get(); - if (prev == 0) { - throw new IllegalStateException(getClass().getName() + ".addRef(" + value + "):" - + " Object is already destroyed." - + " Consider toggling the " + SharedResource.SYSTEM_PROPERTY_NAME_DEBUG - + " system property to get debugging assistance with reference tracking."); - } - int next = prev + value; - if (refCount.compareAndSet(prev, next)) { - return next; - } - } - } - - private static class NoDebugResourceReference extends CloseableOnce { - private final ReferencesByCount resource; - - NoDebugResourceReference(final ReferencesByCount resource) { - this.resource = resource; - } - - @Override final void onClose() { resource.removeRef(); } - @Override References getReferences() { return resource; } - } -} diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java index 987a24fe332..75154dd588e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.jdisc; import com.google.inject.AbstractModule; import com.google.inject.Guice; -import com.google.inject.Key; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.jdisc.application.UriPattern; import com.yahoo.jdisc.handler.CompletionHandler; @@ -307,11 +306,6 @@ public class RequestTestCase { } @Override - public <T> T getInstance(Key<T> key) { - return Guice.createInjector().getInstance(key); - } - - @Override public <T> T getInstance(Class<T> type) { return Guice.createInjector().getInstance(type); } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java index f7488aa392a..32327ace4f4 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java @@ -147,20 +147,6 @@ public class UriPatternTestCase { } @Test - @SuppressWarnings("removal") - public void requireThatPrioritiesAreOrderedDescending() { - assertCompareLt(new UriPattern("scheme://host:69/path", 1), - new UriPattern("scheme://host:69/path", 0)); - } - - @Test - @SuppressWarnings("removal") - public void requireThatPriorityOrdersBeforeScheme() { - assertCompareLt(new UriPattern("*://host:69/path", 1), - new UriPattern("scheme://host:69/path", 0)); - } - - @Test public void requireThatSchemesAreOrdered() { assertCompareLt("b://host:69/path", "a://host:69/path"); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerSnapshotTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerSnapshotTestCase.java index 3e6987c8c6f..dc1f3c7609e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerSnapshotTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerSnapshotTestCase.java @@ -2,8 +2,6 @@ package com.yahoo.jdisc.core; import com.google.inject.AbstractModule; -import com.google.inject.Key; -import com.google.inject.name.Names; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.application.BindingMatch; @@ -142,13 +140,11 @@ public class ContainerSnapshotTestCase { @Override protected void configure() { bind(Object.class).toInstance(obj); - bind(String.class).annotatedWith(Names.named("foo")).toInstance("foo"); } }); ActiveContainer active = new ActiveContainer(driver.newContainerBuilder()); ContainerSnapshot snapshot = new ContainerSnapshot(active, null, null, null); assertSame(obj, snapshot.getInstance(Object.class)); - assertEquals("foo", snapshot.getInstance(Key.get(String.class, Names.named("foo")))); snapshot.release(); assertTrue(driver.close()); } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/test/NonWorkingRequestTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/test/NonWorkingRequestTestCase.java index ac55334308c..b0d67102a12 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/test/NonWorkingRequestTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/test/NonWorkingRequestTestCase.java @@ -2,13 +2,11 @@ package com.yahoo.jdisc.test; import com.google.inject.AbstractModule; -import com.google.inject.Key; -import com.google.inject.name.Names; import com.yahoo.jdisc.Request; import org.junit.Test; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; /** @@ -23,13 +21,14 @@ public class NonWorkingRequestTestCase { @Test public void requireThatGuiceModulesAreInjected() { + Object obj = new Object(); Request request = NonWorkingRequest.newInstance("scheme://host/path", new AbstractModule() { @Override protected void configure() { - bind(String.class).annotatedWith(Names.named("foo")).toInstance("bar"); + bind(Object.class).toInstance(obj); } }); - assertEquals("bar", request.container().getInstance(Key.get(String.class, Names.named("foo")))); + assertSame(obj, request.container().getInstance(Object.class)); } } diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 63de287486e..7963cd51c75 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -199,6 +199,7 @@ public class Mirror implements IMirror { updateTask.schedule(delay); return; } + log.fine(() -> "Try connecting to "+currSlobrok); target = orb.connect(new Spec(currSlobrok)); specsGeneration = 0; } @@ -216,6 +217,9 @@ public class Mirror implements IMirror { if (! logOnSuccess) { log.log(Level.INFO, "Error with location broker "+currSlobrok+" update: " + req.errorMessage() + " (error code " + req.errorCode() + ")"); + } else { + log.fine(() -> "Error with location broker "+currSlobrok+" update: " + req.errorMessage() + + " (error code " + req.errorCode() + ")"); } target.close(); target = null; // try next slobrok @@ -260,6 +264,8 @@ public class Mirror implements IMirror { if (logOnSuccess) { log.log(Level.INFO, "successfully connected to location broker "+currSlobrok+" (mirror initialized with "+newSpecs.length+" service names)"); logOnSuccess = false; + } else { + log.fine(() -> "successfully updated from location broker "+currSlobrok+" (now "+newSpecs.length+" service names)"); } specs.set(newSpecs); @@ -269,6 +275,8 @@ public class Mirror implements IMirror { u++; } updates = u; + } else { + log.fine(() -> "NOP update from location broker "+currSlobrok+" (curr gen "+specsGeneration+")"); } backOff.reset(); } diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java index 57eaa63ef98..3114dd34f79 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java @@ -3,9 +3,12 @@ package com.yahoo.jrt.slobrok.api; import java.util.Arrays; import java.util.concurrent.ThreadLocalRandom; +import java.util.logging.Logger; public class SlobrokList { + private static final Logger log = Logger.getLogger(SlobrokList.class.getName()); + private final Internal internal; private String[] slobroks; private int idx = 0; @@ -20,18 +23,22 @@ public class SlobrokList { private void checkUpdate() { synchronized (internal) { - if (slobroks != internal.slobroks) { - slobroks = internal.slobroks; - idx = 0; + if (slobroks == internal.slobroks) { + return; } + slobroks = internal.slobroks; + log.fine(() -> "checkUpdate() updated tmp list="+Arrays.toString(slobroks)+" from shared list="+Arrays.toString(internal.slobroks)); + idx = 0; } } public String nextSlobrokSpec() { checkUpdate(); if (idx < slobroks.length) { + log.fine(() -> "nextSlobrokSpec() returns: "+slobroks[idx]); return slobroks[idx++]; } + log.fine(() -> "nextSlobrokSpec() reached end of internal list, idx="+idx+"/"+slobroks.length+", tmp list="+Arrays.toString(slobroks)+", shared list="+Arrays.toString(internal.slobroks)); idx = 0; return null; } diff --git a/juniper/src/test/auxTest.cpp b/juniper/src/test/auxTest.cpp index 069a114bfd3..15f5ad1749e 100644 --- a/juniper/src/test/auxTest.cpp +++ b/juniper/src/test/auxTest.cpp @@ -135,12 +135,12 @@ AuxTest::TestDoubleWidth() juniper::Config myConfig("best", juniper); juniper::QueryParser q("\xef\xbd\x93\xef\xbd\x8f\xef\xbd\x8e\xef\xbd\x99"); - juniper::QueryHandle qh(q, NULL, juniper.getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); juniper::Result* res = juniper::Analyse(&myConfig, &qh, input, 17, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); (void) sum; // this should work // _test(sum->Length() != 0); @@ -154,12 +154,13 @@ AuxTest::TestPartialUTF8() { const int inputSize = 5769; // NB: update this if input is changed char input[inputSize]; - FastOS_File file((GetSourceDirectory() + "partialutf8.input.utf8").c_str()); - _test(file.OpenReadOnly()); - _test(file.GetSize() == inputSize); - _test(file.Read(input, inputSize)); - _test(countBrokenUTF8(input, inputSize) == 0); - file.Close(); + { + FastOS_File file((GetSourceDirectory() + "partialutf8.input.utf8").c_str()); + _test(file.OpenReadOnly()); + _test(file.GetSize() == inputSize); + _test(file.Read(input, inputSize)); + _test(countBrokenUTF8(input, inputSize) == 0); + } juniper::PropertyMap myprops; myprops // config taken from vespa test case @@ -173,12 +174,12 @@ AuxTest::TestPartialUTF8() juniper::Config myConfig("best", juniper); juniper::QueryParser q("ipod"); - juniper::QueryHandle qh(q, NULL, juniper.getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); juniper::Result* res = juniper::Analyse(&myConfig, &qh, input, inputSize, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); _test(sum->Length() != 0); // check for partial/broken utf-8 @@ -191,12 +192,13 @@ void AuxTest::TestLargeBlockChinese() { const int inputSize = 10410; // NB: update this if input is changed char input[inputSize]; - FastOS_File file((GetSourceDirectory() + "largeblockchinese.input.utf8").c_str()); - _test(file.OpenReadOnly()); - _test(file.GetSize() == inputSize); - _test(file.Read(input, inputSize)); - _test(countBrokenUTF8(input, inputSize) == 0); - file.Close(); + { + FastOS_File file((GetSourceDirectory() + "largeblockchinese.input.utf8").c_str()); + _test(file.OpenReadOnly()); + _test(file.GetSize() == inputSize); + _test(file.Read(input, inputSize)); + _test(countBrokenUTF8(input, inputSize) == 0); + } juniper::PropertyMap myprops; myprops // config taken from reported bug @@ -212,12 +214,12 @@ void AuxTest::TestLargeBlockChinese() juniper::Config myConfig("best", juniper); juniper::QueryParser q("希望"); - juniper::QueryHandle qh(q, NULL, juniper.getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); juniper::Result* res = juniper::Analyse(&myConfig, &qh, input, inputSize, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); _test(sum->Length() != 0); // check that the entire block of chinese data is not returned in the summary @@ -232,7 +234,7 @@ void AuxTest::TestLargeBlockChinese() void AuxTest::TestExample() { juniper::QueryParser q("AND(consume,sleep,tree)"); - juniper::QueryHandle qh(q, NULL, juniper::_Juniper->getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper::_Juniper->getModifier()); // some content const char* content = "the monkey consumes bananas and sleeps afterwards." @@ -244,7 +246,7 @@ void AuxTest::TestExample() &qh, content, content_len, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); res->Scan(); Matcher& m = *res->_matcher; @@ -259,7 +261,7 @@ AuxTest::TestPropertyMap() juniper::PropertyMap map; IJuniperProperties *props = ↦ map.set("foo", "bar").set("one", "two"); - _test(props->GetProperty("bogus") == NULL); + _test(props->GetProperty("bogus") == nullptr); _test(strcmp(props->GetProperty("bogus", "default"), "default") == 0); _test(strcmp(props->GetProperty("foo"), "bar") == 0); _test(strcmp(props->GetProperty("one", "default"), "two") == 0); @@ -395,7 +397,7 @@ void AuxTest::TestUTF8context() { const char* iso_cont = char_from_u8(u8"AND(m\u00b5ss,fast,s\u00f8kemotor,\u00e5relang)"); juniper::QueryParser q(iso_cont); - juniper::QueryHandle qh(q, NULL, juniper::_Juniper->getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper::_Juniper->getModifier()); // some content std::string s(char_from_u8(u8"Fast leverer s\u00d8kemotorer og andre nyttige ting for \u00e5 finne frem p\u00e5 ")); @@ -409,7 +411,7 @@ void AuxTest::TestUTF8context() s.append(char_from_u8(u8"Hvis bare UTF8-kodingen virker som den skal for tegn som tar mer enn \u00e9n byte.")); juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, s.c_str(), s.size(), 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); size_t charsize; Matcher& m = *res->_matcher; @@ -456,8 +458,6 @@ void AuxTest::TestUTF8context() } -const char* japanese_sep_ex = "。"; - struct TermTextPair { const char* term; @@ -477,24 +477,24 @@ static TermTextPair testjap[] = { "hit", " -. hit at start" }, { "hit", "hit at end .,: " }, { "hit", "---------------------------------------------------------------------------------------------------------------------this is a text that is long enough to generate a hit that does have dots on both sides ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; " }, - { NULL, NULL } + { nullptr, nullptr } }; void AuxTest::TestJapanese() { - for (int i = 0; testjap[i].term != NULL; i++) + for (int i = 0; testjap[i].term != nullptr; i++) { const char* qstr = testjap[i].term; juniper::QueryParser q(qstr); - juniper::QueryHandle qh(q, NULL, juniper::_Juniper->getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper::_Juniper->getModifier()); const char* content = testjap[i].text; int content_len = strlen(content); juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, content, content_len, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); size_t charsize; Matcher& m = *res->_matcher; @@ -506,7 +506,7 @@ void AuxTest::TestJapanese() _sumconf = CreateSummaryConfig("<hit>", "</hit>", "...", "", connectors); SummaryDesc* sumdesc = m.CreateSummaryDesc(256, 256, 4, 80); - _test(sumdesc != NULL); + _test(sumdesc != nullptr); if (!sumdesc) return; std::string sum = BuildSummary(content, content_len, sumdesc, _sumconf, charsize); @@ -556,7 +556,7 @@ void AuxTest::test_summary(Matcher& m, const char* content, size_t content_len, int size, int matches, int surround, size_t& charsize) { SummaryDesc* sum = m.CreateSummaryDesc(size, size, matches, surround); - _test(sum != NULL); + _test(sum != nullptr); if (!sum) { // No summary generated! @@ -571,16 +571,6 @@ void AuxTest::test_summary(Matcher& m, const char* content, size_t content_len, DeleteSummaryDesc(sum); } - -class DefProps : public IJuniperProperties -{ -public: - const char* GetProperty(const char*, const char* def) override { - return def; - } -}; - - void AuxTest::TestStartHits() { juniper::QueryParser q("elvis"); @@ -595,9 +585,9 @@ void AuxTest::TestStartHits() juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, content, content_len, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); (void) sum; // TODO: ReEnable _test(sum->Length() != 0); juniper::ReleaseResult(res); @@ -620,36 +610,13 @@ void AuxTest::TestEndHit() juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, content, content_len, 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); _test(sum->Length() != 0); juniper::ReleaseResult(res); } - - -class TokenChecker : public ITokenProcessor -{ -private: - TokenChecker(const TokenChecker&); - TokenChecker& operator= (const TokenChecker&); - - Token* _out; - int i; -public: - TokenChecker(Token* output) : _out(output), i(0) - { } - - void handle_token(Token& token) override { - _out[i] = token; - i++; - } - - void handle_end(Token&) override {} -}; - - void AuxTest::TestJuniperStack() { // Stack simplification tests @@ -673,7 +640,7 @@ void AuxTest::TestJuniperStack() q->_arity = 0; SimplifyStack(q); std::string s1; - _test(q == NULL); + _test(q == nullptr); if (GetNumFailed() > 0) fprintf(stderr, "TestJuniperStack: %s\n", s.c_str()); @@ -697,7 +664,6 @@ public: //_tokens.back().c_str(), //(int)t.bytepos, (int)t.wordpos, t.bytelen, t.curlen); } - void clearTokens() { _tokens.clear(); } const std::vector<std::string> & getTokens() const { return _tokens; } }; @@ -913,11 +879,11 @@ AuxTest::TestWhiteSpacePreserved() juniper::Config myConfig("myconfig", juniper); juniper::QueryParser q("best"); - juniper::QueryHandle qh(q, NULL, juniper.getModifier()); + juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); juniper::Result* res = juniper::Analyse(&myConfig, &qh, input.c_str(), input.size(), 0, 0, 0); - _test(res != NULL); + _test(res != nullptr); - juniper::Summary* sum = juniper::GetTeaser(res, NULL); + juniper::Summary* sum = juniper::GetTeaser(res, nullptr); vespalib::string expected = "<hi>best</hi> of \nmetallica"; vespalib::string actual(sum->Text(), sum->Length()); _test(actual == expected); diff --git a/logd/src/apps/retention/retention-enforcer.sh b/logd/src/apps/retention/retention-enforcer.sh index 6355600ee4a..24bc61e5764 100755 --- a/logd/src/apps/retention/retention-enforcer.sh +++ b/logd/src/apps/retention/retention-enforcer.sh @@ -64,6 +64,7 @@ mark_pid() { } check_pidfile() { + [ -f $PIDF ] || return 0 read pid < $PIDF [ "$pid" = $$ ] && return 0 if [ "$pid" ] && [ $pid -gt $$ ]; then @@ -105,8 +106,17 @@ maybe_collect() { process_file() { dbfile="$1" now=$(date +%s) + dbf_ts_prefix=${dbfile##*.} + dbf_ts_beg=${dbf_ts_prefix}00000 + dbf_ts_end=${dbf_ts_prefix}99999 + add=$((86400 * $RETAIN_DAYS)) + earliest_expire=$((${dbf_ts_beg} + $add)) + if [ $earliest_expire -gt $now ]; then + return 0 + fi found=0 while read timestamp logfilename; do + sleep 1 for fn in $logfilename $logfilename.*z*; do if [ -f "$fn" ]; then found=1 @@ -115,8 +125,7 @@ process_file() { done done < $dbfile if [ $found = 0 ]; then - ts=${dbfile##*.}99999 - maybe_collect "$ts" "$dbfile" + maybe_collect "${dbf_ts_end}" "$dbfile" fi } @@ -124,6 +133,7 @@ process_all() { for dbf in $DBDIR/logfiles.* ; do [ -f "$dbf" ] || continue process_file "$dbf" + sleep 1 done } @@ -139,5 +149,6 @@ mainloop() { # MAIN: prepare_stuff +sleep 600 mainloop exit 0 diff --git a/logd/src/logd/config_subscriber.cpp b/logd/src/logd/config_subscriber.cpp index be3d42fb6f0..0b47d22096a 100644 --- a/logd/src/logd/config_subscriber.cpp +++ b/logd/src/logd/config_subscriber.cpp @@ -3,6 +3,7 @@ #include "config_subscriber.h" #include "empty_forwarder.h" #include "rpc_forwarder.h" +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/log/log.h> LOG_SETUP(""); diff --git a/logd/src/logd/config_subscriber.h b/logd/src/logd/config_subscriber.h index 9f7dea12d33..3c5ac202832 100644 --- a/logd/src/logd/config_subscriber.h +++ b/logd/src/logd/config_subscriber.h @@ -3,7 +3,8 @@ #include "forwarder.h" #include <logd/config-logd.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/vespalib/util/time.h> diff --git a/logd/src/tests/watcher/watcher_test.cpp b/logd/src/tests/watcher/watcher_test.cpp index f897ab32b5c..50c8f47a203 100644 --- a/logd/src/tests/watcher/watcher_test.cpp +++ b/logd/src/tests/watcher/watcher_test.cpp @@ -34,7 +34,7 @@ struct ConfigFixture { const std::string configId; LogdConfigBuilder logdBuilder; ConfigSet set; - IConfigContext::SP context; + std::shared_ptr<IConfigContext> context; int idcounter; ConfigFixture(const std::string & id); diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index c76012d146a..b8e2a95783e 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -1,15 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cf-handler.h" -#include <cstdlib> -#include <dirent.h> -#include <stdio.h> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <vespa/defaults.h> #include <vespa/config/common/configsystem.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/configsubscriber.hpp> +#include <dirent.h> +#include <sys/stat.h> #include <vespa/log/log.h> LOG_SETUP(".cf-handler"); diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h index 8e7196fb034..651b8d22fa8 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/config/config.h> -#include <vespa/config-logforwarder.h> #include "child-handler.h" +#include <vespa/config-logforwarder.h> +#include <vespa/config/subscription/configsubscriber.h> using cloud::config::LogforwarderConfig; diff --git a/logserver/pom.xml b/logserver/pom.xml index 19f422f3a86..27381f7aa42 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -63,17 +63,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>com.yahoo.logserver.Server</mainClass> diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java index bc3fe8e2fbf..a389ff24347 100644 --- a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -79,6 +79,7 @@ class ProtobufSerialization { .build(); } + @SuppressWarnings("deprecation") private static Level fromLogMessageLevel(LogProtocol.LogMessage.Level level) { switch (level) { case FATAL: @@ -104,6 +105,7 @@ class ProtobufSerialization { } } + @SuppressWarnings("deprecation") private static LogProtocol.LogMessage.Level toLogMessageLevel(Level level) { Level vespaLevel = LogLevel.getVespaLogLevel(level); if (vespaLevel.equals(LogLevel.FATAL)) { diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index a0f2082e706..ff9939ff244 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -22,7 +22,7 @@ import java.util.logging.Logger; * @author Bjorn Borud * @author Stig Bakken */ - +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class Server implements Runnable { private final AtomicBoolean signalCaught = new AtomicBoolean(false); static final String APPNAME = "logserver"; diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java index b343758b2bf..99a2b5d7c72 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -14,6 +14,7 @@ import java.util.Map; * * @author Bjorn Borud */ +@SuppressWarnings("deprecation") public class LogFilterManager { private static final LogFilterManager instance; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index 849e3f6ac0b..59475972279 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -24,6 +24,7 @@ import com.yahoo.logserver.handlers.AbstractLogHandler; * * @author hmusum */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class LogMetricsHandler extends AbstractLogHandler { private static final long EVENTINTERVAL = 5 * 60; // in seconds @@ -33,6 +34,7 @@ public class LogMetricsHandler extends AbstractLogHandler { private final List<LevelCount> logMetrics = new ArrayList<LevelCount>(); // The log levels that are handled by this plugin + @SuppressWarnings("deprecation") private static final Level[] levels = {LogLevel.INFO, LogLevel.WARNING, LogLevel.SEVERE, diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java index 928b4b1c687..0c9af994c3d 100644 --- a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -17,6 +17,7 @@ import com.yahoo.log.LogMessage; * * @author Bjorn Borud */ +@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove (com.yahoo.log.event) public class VerifyLogfile { public static void main (String[] args) throws IOException { diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 7bb84ec28f3..367744e53bf 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.verify; /** * @author bjorncs */ +@SuppressWarnings("deprecation") public class ArchiveLogMessagesMethodTest { private static final LogMessage MESSAGE_1 = diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index bff28986119..039f22c7525 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -73,7 +73,8 @@ "abstract" ], "methods": [ - "public abstract void connect()" + "public abstract void connect()", + "public abstract void disconnect()" ], "fields": [] }, @@ -96,7 +97,8 @@ "public java.lang.String getConnectionSpec()", "public java.lang.String getName()", "public void handleMessage(com.yahoo.messagebus.Message)", - "public void connect()" + "public void connect()", + "public void disconnect()" ], "fields": [] }, @@ -245,7 +247,8 @@ "public java.lang.String getName()", "public void handleMessage(com.yahoo.messagebus.Message)", "public void handleReply(com.yahoo.messagebus.Reply)", - "public void connect()" + "public void connect()", + "public void disconnect()" ], "fields": [] }, diff --git a/messagebus/pom.xml b/messagebus/pom.xml index ff55a4eca96..a59499b97e1 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -67,17 +67,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java b/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java index 4db97f0c083..10ebb2fc1ae 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java @@ -14,6 +14,7 @@ import com.yahoo.messagebus.routing.RoutingTableSpec; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class ConfigAgent implements ConfigSubscriber.SingleSubscriber<MessagebusConfig>{ private final ConfigURI configURI; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Connectable.java b/messagebus/src/main/java/com/yahoo/messagebus/Connectable.java index db8d926db43..354548a3cf3 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Connectable.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Connectable.java @@ -9,5 +9,6 @@ package com.yahoo.messagebus; public interface Connectable { void connect(); + void disconnect(); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DestinationSession.java b/messagebus/src/main/java/com/yahoo/messagebus/DestinationSession.java index 509ea5903c5..32f7ccd2e18 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DestinationSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DestinationSession.java @@ -130,4 +130,6 @@ public final class DestinationSession implements Connectable, MessageHandler { mbus.connect(name, broadcastName); } + @Override public void disconnect() { close(); } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java index d5b7e045254..cbc5d89be79 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java @@ -119,4 +119,6 @@ public final class IntermediateSession implements MessageHandler, ReplyHandler, mbus.connect(name, broadcastName); } + @Override public void disconnect() { close(); } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 5ff9a4b0313..c457a703ecc 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; */ public class RPCNetwork implements Network, MethodHandler { - private static Logger log = Logger.getLogger(RPCNetwork.class.getName()); + private static final Logger log = Logger.getLogger(RPCNetwork.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final Identity identity; @@ -69,8 +69,8 @@ public class RPCNetwork implements Network, MethodHandler { private final LinkedHashMap<String, Route> lruRouteMap = new LinkedHashMap<>(10000, 0.5f, true); private final ExecutorService executor = new ThreadPoolExecutor(getNumThreads(), getNumThreads(), 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), - ThreadFactoryFactory.getDaemonThreadFactory("mbus.net"), new ThreadPoolExecutor.CallerRunsPolicy()); + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getDaemonThreadFactory("mbus.net")); private static int getNumThreads() { return Math.max(2, Runtime.getRuntime().availableProcessors()/2); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java index 903a31d3f3a..02472d8f31f 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java @@ -13,6 +13,7 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscriber<SlobroksConfig>{ private static final Logger log = Logger.getLogger(SlobrokConfigSubscriber.class.getName()); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/Resender.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/Resender.java index 753f8e28e36..5aa4296885b 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/Resender.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/Resender.java @@ -20,8 +20,10 @@ import java.util.List; */ public class Resender { + private final Object monitor = new Object(); private final PriorityQueue<Entry> queue = new PriorityQueue<>(); private final RetryPolicy retryPolicy; + private boolean destroyed = false; /** * Constructs a new resender. @@ -59,7 +61,9 @@ public class Resender { return false; } } - return true; + synchronized (monitor) { + return !destroyed; + } } /** @@ -83,13 +87,13 @@ public class Resender { node.addError(ErrorCode.TIMEOUT, "Timeout exceeded by resender, giving up."); return false; } - node.prepareForRetry(); // consumes the reply - node.getTrace().trace(TraceLevel.COMPONENT, - "Message scheduled for retry " + retry + " in " + delay + " seconds."); - msg.setRetry(retry); - Entry entry = new Entry(node, SystemTimer.INSTANCE.milliTime() + (long) (delay * 1000)); - synchronized (queue) { - queue.add(entry); + synchronized (monitor) { + if (destroyed) return false; + node.prepareForRetry(); // consumes the reply + node.getTrace().trace(TraceLevel.COMPONENT, + "Message scheduled for retry " + retry + " in " + delay + " seconds."); + msg.setRetry(retry); + queue.add(new Entry(node, SystemTimer.INSTANCE.milliTime() + (long) (delay * 1000))); } return true; } @@ -101,7 +105,7 @@ public class Resender { List<RoutingNode> sendList; long now = SystemTimer.INSTANCE.milliTime(); - synchronized (queue) { + synchronized (monitor) { if (queue.isEmpty()) return; sendList = new LinkedList<>(); while (!queue.isEmpty() && queue.peek().time <= now) { @@ -119,10 +123,11 @@ public class Resender { * Discards all the routing nodes currently scheduled for resending. */ public void destroy() { - synchronized (queue) { + synchronized (monitor) { while (!queue.isEmpty()) { queue.poll().node.discard(); } + destroyed = true; } } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java index a32175ff414..287529ae449 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java @@ -22,6 +22,7 @@ public class ConfigAgentTestCase { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); + @SuppressWarnings("removal") // TODO Vespa 8: remove @Test public void testRoutingConfig() throws InterruptedException { LocalHandler handler = new LocalHandler(); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java index b73dc52152a..13a4e154122 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java @@ -97,10 +97,10 @@ public class DynamicThrottlePolicyTest { /** Sort of a dummy test, as the conditions are perfect. In a more realistic scenario, below, the algorithm needs luck to climb this high. */ @Test public void singlePolicySingleWorkerWithIncreasingParallelism() { - for (int i = 0; i < 4; i++) { + for (int exponent = 0; exponent < 4; exponent++) { CustomTimer timer = new CustomTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); - int scaleFactor = (int) Math.pow(10, i); + int scaleFactor = (int) Math.pow(10, exponent); long operations = 3_000L * scaleFactor; int workPerSuccess = 6; int numberOfWorkers = 1; @@ -120,10 +120,10 @@ public class DynamicThrottlePolicyTest { /** A more realistic test, where throughput gradually flattens with increasing window size, and with more variance in throughput. */ @Test public void singlePolicyIncreasingWorkersWithNoParallelism() { - for (int i = 0; i < 4; i++) { + for (int exponent = 0; exponent < 4; exponent++) { CustomTimer timer = new CustomTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); - int scaleFactor = (int) Math.pow(10, i); + int scaleFactor = (int) Math.pow(10, exponent); long operations = 2_000L * scaleFactor; // workPerSuccess determines the latency of the simulated server, which again determines the impact of the // synthetic attractors of the algorithm, around latencies which give (close to) integer log10(1 / latency). @@ -143,14 +143,14 @@ public class DynamicThrottlePolicyTest { double maxMaxPending = numberOfWorkers * maximumTasksPerWorker; assertInRange(minMaxPending, summary.averagePending, maxMaxPending); assertInRange(minMaxPending, summary.averageWindows[0], maxMaxPending); - assertInRange(1, summary.inefficiency, 1 + 0.25 * i); // Even slower ramp-up. + assertInRange(1, summary.inefficiency, 1 + 0.25 * exponent); // Even slower ramp-up. assertInRange(0, summary.waste, 0); } } @Test public void twoWeightedPoliciesWithUnboundedTaskQueue() { - for (int i = 0; i < 10; i++) { + for (int repeat = 0; repeat < 3; repeat++) { long operations = 1_000_000; int workPerSuccess = 6 + (int) (30 * Math.random()); int numberOfWorkers = 1 + (int) (10 * Math.random()); @@ -174,7 +174,7 @@ public class DynamicThrottlePolicyTest { @Test public void tenPoliciesVeryParallelServerWithShortTaskQueue() { - for (int i = 0; i < 10; i++) { + for (int repeat = 0; repeat < 2; repeat++) { long operations = 1_000_000; int workPerSuccess = 6; int numberOfWorkers = 6; diff --git a/messagebus/src/tests/configagent/configagent.cpp b/messagebus/src/tests/configagent/configagent.cpp index 32c28fd8c3d..06940c682f0 100644 --- a/messagebus/src/tests/configagent/configagent.cpp +++ b/messagebus/src/tests/configagent/configagent.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/config/print/fileconfigreader.h> #include <vespa/messagebus/configagent.h> #include <vespa/messagebus/iconfighandler.h> #include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/config-messagebus.h> +#include <vespa/config/print/fileconfigreader.hpp> using namespace mbus; using namespace messagebus; diff --git a/messagebus/src/tests/routingspec/routingspec.cpp b/messagebus/src/tests/routingspec/routingspec.cpp index 0079c7c8673..c3e6754ee7d 100644 --- a/messagebus/src/tests/routingspec/routingspec.cpp +++ b/messagebus/src/tests/routingspec/routingspec.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/config.h> #include <vespa/messagebus/configagent.h> #include <vespa/messagebus/iconfighandler.h> #include <vespa/messagebus/routing/routingspec.h> diff --git a/messagebus/src/tests/throttling/throttling.cpp b/messagebus/src/tests/throttling/throttling.cpp index 07db548be48..a601845a5fc 100644 --- a/messagebus/src/tests/throttling/throttling.cpp +++ b/messagebus/src/tests/throttling/throttling.cpp @@ -252,36 +252,39 @@ Test::testMinOne() void Test::testDynamicWindowSize() { - std::unique_ptr<DynamicTimer> ptr(new DynamicTimer()); - DynamicTimer *timer = ptr.get(); + auto ptr = std::make_unique<DynamicTimer>(); + auto* timer = ptr.get(); DynamicThrottlePolicy policy(std::move(ptr)); - policy.setWindowSizeIncrement(5); + policy.setWindowSizeIncrement(5) + .setResizeRate(1); double windowSize = getWindowSize(policy, *timer, 100); - ASSERT_TRUE(windowSize >= 90 && windowSize <= 110); + ASSERT_TRUE(windowSize >= 90 && windowSize <= 105); windowSize = getWindowSize(policy, *timer, 200); - ASSERT_TRUE(windowSize >= 90 && windowSize <= 210); + ASSERT_TRUE(windowSize >= 180 && windowSize <= 205); windowSize = getWindowSize(policy, *timer, 50); - ASSERT_TRUE(windowSize >= 9 && windowSize <= 55); + ASSERT_TRUE(windowSize >= 45 && windowSize <= 55); windowSize = getWindowSize(policy, *timer, 500); - ASSERT_TRUE(windowSize >= 90 && windowSize <= 505); + ASSERT_TRUE(windowSize >= 450 && windowSize <= 505); windowSize = getWindowSize(policy, *timer, 100); - ASSERT_TRUE(windowSize >= 90 && windowSize <= 110); + ASSERT_TRUE(windowSize >= 90 && windowSize <= 115); } void Test::testIdleTimePeriod() { - ITimer::UP ptr(new DynamicTimer()); - DynamicTimer *timer = static_cast<DynamicTimer*>(ptr.get()); + auto ptr = std::make_unique<DynamicTimer>(); + auto* timer = ptr.get(); DynamicThrottlePolicy policy(std::move(ptr)); - policy.setWindowSizeIncrement(5); + policy.setWindowSizeIncrement(5) + .setMinWindowSize(1) + .setResizeRate(1); double windowSize = getWindowSize(policy, *timer, 100); ASSERT_TRUE(windowSize >= 90 && windowSize <= 110); @@ -303,12 +306,13 @@ Test::testIdleTimePeriod() void Test::testMinWindowSize() { - ITimer::UP ptr(new DynamicTimer()); - DynamicTimer *timer = static_cast<DynamicTimer*>(ptr.get()); + auto ptr = std::make_unique<DynamicTimer>(); + auto* timer = ptr.get(); DynamicThrottlePolicy policy(std::move(ptr)); - policy.setWindowSizeIncrement(5); - policy.setMinWindowSize(150); + policy.setWindowSizeIncrement(5) + .setResizeRate(1) + .setMinWindowSize(150); double windowSize = getWindowSize(policy, *timer, 200); ASSERT_TRUE(windowSize >= 150 && windowSize <= 210); @@ -317,12 +321,13 @@ Test::testMinWindowSize() void Test::testMaxWindowSize() { - ITimer::UP ptr(new DynamicTimer()); - DynamicTimer *timer = static_cast<DynamicTimer*>(ptr.get()); + auto ptr = std::make_unique<DynamicTimer>(); + auto* timer = ptr.get(); DynamicThrottlePolicy policy(std::move(ptr)); - policy.setWindowSizeIncrement(5); - policy.setMaxWindowSize(50); + policy.setWindowSizeIncrement(5) + .setResizeRate(1) + .setMaxWindowSize(50); double windowSize = getWindowSize(policy, *timer, 100); ASSERT_TRUE(windowSize >= 40 && windowSize <= 50); @@ -338,6 +343,7 @@ Test::getWindowSize(DynamicThrottlePolicy &policy, DynamicTimer &timer, uint32_t { SimpleMessage msg("foo"); SimpleReply reply("bar"); + reply.setContext(mbus::Context(uint64_t(1))); // To offset pending size bump in static policy for (uint32_t i = 0; i < 999; ++i) { uint32_t numPending = 0; @@ -354,6 +360,6 @@ Test::getWindowSize(DynamicThrottlePolicy &policy, DynamicTimer &timer, uint32_t } } uint32_t ret = policy.getMaxPendingCount(); - printf("getWindowSize() = %d\n", ret); + fprintf(stderr, "getWindowSize() = %u\n", ret); return ret; } diff --git a/messagebus/src/vespa/messagebus/configagent.h b/messagebus/src/vespa/messagebus/configagent.h index c7327b60527..84bf18aab99 100644 --- a/messagebus/src/vespa/messagebus/configagent.h +++ b/messagebus/src/vespa/messagebus/configagent.h @@ -2,7 +2,6 @@ #pragma once -#include <memory> #include <vespa/messagebus/common.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/messagebus/config-messagebus.h> diff --git a/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.cpp b/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.cpp index 00a76ca8b12..be61dcdc675 100644 --- a/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.cpp +++ b/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.cpp @@ -12,8 +12,8 @@ DynamicThrottlePolicy::DynamicThrottlePolicy() : _timer(new SteadyTimer()), _numSent(0), _numOk(0), - _resizeRate(3), - _resizeTime(_timer->getMilliTime()), + _resizeRate(3.0), + _resizeTime(0), _timeOfLastMessage(_timer->getMilliTime()), _idleTimePeriod(60000), _efficiencyThreshold(1), @@ -21,6 +21,7 @@ DynamicThrottlePolicy::DynamicThrottlePolicy() : _windowSize(_windowSizeIncrement), _maxWindowSize(INT_MAX), _minWindowSize(_windowSizeIncrement), + _decrementFactor(2.0), _windowSizeBackOff(0.9), _weight(1), _localMaxThroughput(0) @@ -30,8 +31,8 @@ DynamicThrottlePolicy::DynamicThrottlePolicy(double windowSizeIncrement) : _timer(new SteadyTimer()), _numSent(0), _numOk(0), - _resizeRate(3), - _resizeTime(_timer->getMilliTime()), + _resizeRate(3.0), + _resizeTime(0), _timeOfLastMessage(_timer->getMilliTime()), _idleTimePeriod(60000), _efficiencyThreshold(1), @@ -39,6 +40,7 @@ DynamicThrottlePolicy::DynamicThrottlePolicy(double windowSizeIncrement) : _windowSize(_windowSizeIncrement), _maxWindowSize(INT_MAX), _minWindowSize(_windowSizeIncrement), + _decrementFactor(2.0), _windowSizeBackOff(0.9), _weight(1), _localMaxThroughput(0) @@ -48,8 +50,8 @@ DynamicThrottlePolicy::DynamicThrottlePolicy(ITimer::UP timer) : _timer(std::move(timer)), _numSent(0), _numOk(0), - _resizeRate(3), - _resizeTime(_timer->getMilliTime()), + _resizeRate(3.0), + _resizeTime(0), _timeOfLastMessage(_timer->getMilliTime()), _idleTimePeriod(60000), _efficiencyThreshold(1), @@ -57,6 +59,7 @@ DynamicThrottlePolicy::DynamicThrottlePolicy(ITimer::UP timer) : _windowSize(_windowSizeIncrement), _maxWindowSize(INT_MAX), _minWindowSize(_windowSizeIncrement), + _decrementFactor(2.0), _windowSizeBackOff(0.9), _weight(1), _localMaxThroughput(0) @@ -73,20 +76,21 @@ DynamicThrottlePolicy & DynamicThrottlePolicy::setWindowSizeIncrement(double windowSizeIncrement) { _windowSizeIncrement = windowSizeIncrement; + _windowSize = std::max(_windowSize, _windowSizeIncrement); return *this; } DynamicThrottlePolicy & DynamicThrottlePolicy::setWindowSizeBackOff(double windowSizeBackOff) { - _windowSizeBackOff = windowSizeBackOff; + _windowSizeBackOff = std::max(0.0, std::min(1.0, windowSizeBackOff)); return *this; } DynamicThrottlePolicy & -DynamicThrottlePolicy::setResizeRate(uint32_t resizeRate) +DynamicThrottlePolicy::setResizeRate(double resizeRate) { - _resizeRate = resizeRate; + _resizeRate = std::max(2.0, resizeRate); return *this; } @@ -115,6 +119,14 @@ DynamicThrottlePolicy & DynamicThrottlePolicy::setMinWindowSize(double min) { _minWindowSize = min; + _windowSize = std::max(_minWindowSize, _windowSizeIncrement); + return *this; +} + +DynamicThrottlePolicy& +DynamicThrottlePolicy::setWindowSizeDecrementFactor(double decrementFactor) +{ + _decrementFactor = decrementFactor; return *this; } @@ -134,10 +146,14 @@ DynamicThrottlePolicy::canSend(const Message &msg, uint32_t pendingCount) } uint64_t time = _timer->getMilliTime(); if (time - _timeOfLastMessage > _idleTimePeriod) { - _windowSize = std::min(_windowSize, (double) pendingCount + _windowSizeIncrement); + _windowSize = std::max(_minWindowSize, std::min(_windowSize, pendingCount + _windowSizeIncrement)); + LOG(debug, "Idle time exceeded; WindowSize = %.2f", _windowSize); } _timeOfLastMessage = time; - return pendingCount < _windowSize; + auto windowSizeFloored = static_cast<uint32_t>(_windowSize); + // Use floating point window sizes, so the algorithm sees the difference between 1.1 and 1.9 window size. + bool carry = _numSent < ((_windowSize * _resizeRate) * (_windowSize - windowSizeFloored)); + return pendingCount < (windowSizeFloored + (carry ? 1 : 0)); } void @@ -156,7 +172,7 @@ DynamicThrottlePolicy::processMessage(Message &msg) _numSent = 0; _numOk = 0; - if (throughput > _localMaxThroughput * 1.01) { + if (throughput > _localMaxThroughput) { LOG(debug, "WindowSize = %.2f, Throughput = %f", _windowSize, throughput); _localMaxThroughput = throughput; _windowSize += _weight*_windowSizeIncrement; @@ -170,15 +186,15 @@ DynamicThrottlePolicy::processMessage(Message &msg) period *= 0.1; } double efficiency = throughput*period/_windowSize; - LOG(debug, "WindowSize = %.2f, Throughput = %f, Efficiency = %.2f, Elapsed = %.2f, Period = %.2f", _windowSize, throughput, efficiency, elapsed, period); if (efficiency < _efficiencyThreshold) { - double newSize = std::min(throughput * period, _windowSize); - _windowSize = std::min(newSize * _windowSizeBackOff, _windowSize - 2 * _windowSizeIncrement); + _windowSize = std::min(_windowSize * _windowSizeBackOff, _windowSize - _decrementFactor * _windowSizeIncrement); _localMaxThroughput = 0; } else { _windowSize += _weight*_windowSizeIncrement; } + LOG(debug, "WindowSize = %.2f, Throughput = %f, Efficiency = %.2f, Elapsed = %.2f, Period = %.2f", + _windowSize, throughput, efficiency, elapsed, period); } _windowSize = std::max(_minWindowSize, _windowSize); _windowSize = std::min(_maxWindowSize, _windowSize); diff --git a/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.h b/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.h index 99a549beef2..1ae7e88f876 100644 --- a/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.h +++ b/messagebus/src/vespa/messagebus/dynamicthrottlepolicy.h @@ -19,7 +19,7 @@ private: ITimer::UP _timer; uint32_t _numSent; uint32_t _numOk; - uint32_t _resizeRate; + double _resizeRate; uint64_t _resizeTime; uint64_t _timeOfLastMessage; uint64_t _idleTimePeriod; @@ -28,6 +28,7 @@ private: double _windowSize; double _maxWindowSize; double _minWindowSize; + double _decrementFactor; double _windowSizeBackOff; double _weight; double _localMaxThroughput; @@ -96,7 +97,7 @@ public: * @param resizeRate The rate to set. * @return This, to allow chaining. */ - DynamicThrottlePolicy &setResizeRate(uint32_t resizeRate); + DynamicThrottlePolicy &setResizeRate(double resizeRate); /** * Sets the weight for this client. The larger the value, the more resources @@ -151,6 +152,14 @@ public: DynamicThrottlePolicy &setMinWindowSize(double min); /** + * Sets the relative step size when decreasing window size. + * + * @param decrementFactor the step size to set + * @return this, to allow chaining + */ + DynamicThrottlePolicy& setWindowSizeDecrementFactor(double decrementFactor); + + /** * Get the minimum number of pending operations allowed at any time. * * @return The minimum number of operations. diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp index 0700415e691..a79171f5af2 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpcmessagebus.h" #include <vespa/messagebus/network/rpcnetworkparams.h> -#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.h b/messagebus/src/vespa/messagebus/rpcmessagebus.h index 822677e48e6..ed4258bef2d 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.h +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.h @@ -5,7 +5,6 @@ #include "configagent.h" #include "protocolset.h" #include <vespa/messagebus/network/rpcnetwork.h> -#include <vespa/config/helper/legacysubscriber.h> namespace config {class ConfigUri; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index df670addcd8..b316a620a9b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -31,8 +31,8 @@ public class ValuesFetcher { private final MetricsConsumers metricsConsumers; public ValuesFetcher(MetricsManager metricsManager, - VespaServices vespaServices, - MetricsConsumers metricsConsumers) { + VespaServices vespaServices, + MetricsConsumers metricsConsumers) { this.metricsManager = metricsManager; this.vespaServices = vespaServices; this.metricsConsumers = metricsConsumers; @@ -47,7 +47,7 @@ public class ValuesFetcher { .collect(Collectors.toList()); } - public MetricsPacket.Builder [] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { + public MetricsPacket.Builder[] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now(), consumer) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java index f58c63cd76c..317f790b08e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -31,7 +31,6 @@ import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.app import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.OK; import static java.util.Collections.singletonMap; -import static java.util.stream.Collectors.toList; /** * Http handler for the metrics/v2 rest api. diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 82d8521d6db..cfc2e722732 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -13,11 +13,15 @@ #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/util/atomic.h> +#include <mutex> #include <thread> #include <vespa/log/log.h> LOG_SETUP(".test.metricmanager"); +using namespace vespalib::atomic; + namespace metrics { struct MetricManagerTest : public ::testing::Test { @@ -363,10 +367,14 @@ TEST_F(MetricManagerTest, test_consumer_visitor) namespace { -struct FakeTimer : public MetricManager::Timer { - time_t _time; +class FakeTimer : public MetricManager::Timer { + std::atomic<time_t> _time; +public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return _time; } + time_t getTime() const override { return load_relaxed(_time); } + void set_time(time_t t) noexcept { store_relaxed(_time, t); } + // Not safe for multiple writers, only expected to be called by test. + void add_time(time_t t) noexcept { set_time(getTime() + t); } }; struct BriefValuePrinter : public MetricVisitor { @@ -516,7 +524,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(7); mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->_time += 5 * 60; + timer->add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -530,17 +538,15 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(8); mySet.val10.a.val2.addValue(3); mySet.val10.b.val1.addValue(2); - timer->_time += 5 * 60; + timer->add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); ASSERT_VALUES(mm, 60 * 60, ""); ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); - //std::cerr << dumpAllSnapshots(mm, "snapper") << "\n"; - // Adding another five minute period where nothing have happened. // Metric for last 5 minutes should be 0. - timer->_time += 5 * 60; + timer->add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -551,7 +557,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past - timer->_time += 5 * 60; + timer->add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); } ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); @@ -599,7 +605,7 @@ TEST_F(MetricManagerTest, test_xml_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->_time = 1300; + timer->set_time(1300); takeSnapshots(mm, 1300); std::string expected( @@ -674,7 +680,7 @@ TEST_F(MetricManagerTest, test_json_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->_time = 1300; + timer->set_time(1300); takeSnapshots(mm, 1300); // Create json output @@ -769,7 +775,7 @@ struct MetricSnapshotTestFixture // Take snapshot of metric values from time 1000 to time 1300 void takeSnapshotsOnce() { - timer->_time = 1300; + timer->set_time(1300); test.takeSnapshots(manager, 1300); } @@ -1049,13 +1055,22 @@ TEST_F(MetricManagerTest, text_output_supports_dimensions) namespace { struct MyUpdateHook : public UpdateHook { std::ostringstream& _output; - FakeTimer& _timer; + std::mutex& _output_mutex; + FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, const char* name, + MyUpdateHook(std::ostringstream& output, + std::mutex& output_mutex, + const char* name, FakeTimer& timer) - : UpdateHook(name), _output(output), _timer(timer) {} + : UpdateHook(name), + _output(output), + _output_mutex(output_mutex), + _timer(timer) + {} + ~MyUpdateHook() override = default; void updateMetrics(const MetricLockGuard & ) override { + std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread _output << _timer.getTime() << ": " << getName() << " called\n"; } }; @@ -1063,6 +1078,7 @@ namespace { TEST_F(MetricManagerTest, test_update_hooks) { + std::mutex output_mutex; std::ostringstream output; FastOS_ThreadPool pool(256_Ki); FakeTimer* timer = new FakeTimer(1000); @@ -1075,9 +1091,9 @@ TEST_F(MetricManagerTest, test_update_hooks) mm.registerMetric(lockGuard, mySet.set); } - MyUpdateHook preInitShort(output, "BIS", *timer); - MyUpdateHook preInitLong(output, "BIL", *timer); - MyUpdateHook preInitInfinite(output, "BII", *timer); + MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer); + MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer); + MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer); mm.addMetricUpdateHook(preInitShort, 5); mm.addMetricUpdateHook(preInitLong, 300); mm.addMetricUpdateHook(preInitInfinite, 0); @@ -1097,55 +1113,55 @@ TEST_F(MetricManagerTest, test_update_hooks) pool); output << "Init done\n"; - MyUpdateHook postInitShort(output, "AIS", *timer); - MyUpdateHook postInitLong(output, "AIL", *timer); - MyUpdateHook postInitInfinite(output, "AII", *timer); + MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); + MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer); + MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer); mm.addMetricUpdateHook(postInitShort, 5); mm.addMetricUpdateHook(postInitLong, 400); mm.addMetricUpdateHook(postInitInfinite, 0); // After 5 seconds the short ones should get another. - timer->_time = 1006; + timer->set_time(1006); waitForTimeProcessed(mm, 1006); // After 4 more seconds the short ones should get another // since last update was a second late. (Stable periods, process time // should not affect how often they are updated) - timer->_time = 1010; + timer->set_time(1010); waitForTimeProcessed(mm, 1010); // Bumping considerably ahead, such that next update is in the past, // we should only get one update called in this period. - timer->_time = 1200; + timer->set_time(1200); waitForTimeProcessed(mm, 1200); // No updates at this time. - timer->_time = 1204; + timer->set_time(1204); waitForTimeProcessed(mm, 1204); // Give all hooks an update mm.updateMetrics(true); // Last update should not have interfered with periods - timer->_time = 1205; + timer->set_time(1205); waitForTimeProcessed(mm, 1205); // Time is just ahead of a snapshot. - timer->_time = 1299; + timer->set_time(1299); waitForTimeProcessed(mm, 1299); // At time 1300 we are at a 5 minute snapshot bump // All hooks should thus get an update. The one with matching period // should only get one - timer->_time = 1300; + timer->set_time(1300); waitForTimeProcessed(mm, 1300); // The snapshot time currently doesn't count for the metric at period // 400. It will get an event at this time. - timer->_time = 1450; + timer->set_time(1450); waitForTimeProcessed(mm, 1450); std::string expected( @@ -1179,8 +1195,11 @@ TEST_F(MetricManagerTest, test_update_hooks) "1450: AIS called\n" "1450: AIL called\n" ); - std::string actual(output.str()); - EXPECT_EQ(expected, actual); + { + std::lock_guard lock(output_mutex); // Need to ensure we observe all writes by metric mgr thread + std::string actual(output.str()); + EXPECT_EQ(expected, actual); + } } } diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 6336672ac08..2fd09094f4c 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -11,8 +11,9 @@ #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hashtable.hpp> +#include <vespa/config/subscription/configsubscriber.hpp> +#include <set> #include <sstream> -#include <algorithm> #include <cassert> #include <vespa/log/bufferedlogger.h> @@ -160,8 +161,7 @@ MetricManager::isInitialized() const { } void -MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, - bool startThread) +MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool startThread) { if (isInitialized()) { throw vespalib::IllegalStateException( @@ -452,7 +452,7 @@ MetricManager::createSnapshotPeriods(const Config& config) void MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> config) { - assert(config.get() != 0); + assert(config); if (LOG_WOULD_LOG(debug)) { std::ostringstream ost; config::OstreamConfigWriter w(ost); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 300604d1f9f..0efbe1b091a 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -43,7 +43,6 @@ */ #pragma once -#include <vespa/metrics/config-metricsmanager.h> #include "metricset.h" #include "metricsnapshot.h" #include "memoryconsumption.h" @@ -52,7 +51,9 @@ #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/jsonwriter.h> -#include <vespa/config/config.h> +#include <vespa/metrics/config-metricsmanager.h> +#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/subscription/configuri.h> #include <map> #include <list> @@ -96,7 +97,7 @@ public: private: MetricSnapshot _activeMetrics; std::unique_ptr<config::ConfigSubscriber> _configSubscriber; - config::ConfigHandle<MetricsmanagerConfig>::UP _configHandle; + std::unique_ptr<config::ConfigHandle<MetricsmanagerConfig>> _configHandle; std::unique_ptr<MetricsmanagerConfig> _config; std::map<Metric::String, ConsumerSpec::SP> _consumerConfig; std::list<UpdateHook*> _periodicUpdateHooks; diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 3f1a86e083b..a36215e005f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -1,30 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import com.yahoo.config.FileReference; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; -import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.serialization.TypedBinaryFormat; -import com.yahoo.text.Utf8; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import java.io.IOException; import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -45,17 +32,19 @@ public class ModelTester { public Map<String, Model> models() { return models; } private static Map<String, Model> createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expresionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId(path, "rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId(path, "ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId(path, "onnx-models.cfg")); + + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)) - .importFrom(config, constantsConfig, expresionsConfig, onnxModelsConfig); + .importFrom(config, constantsConfig, expressionsConfig, onnxModelsConfig); + } + + private static String fileConfigId(String path, String filename) { + return "file:" + path + filename; } public ExpressionFunction assertFunction(String name, String expression, Model model) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 4cb52216137..540c534925e 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -28,10 +28,11 @@ import static org.junit.Assert.assertTrue; public class ModelsEvaluatorTest { private static final double delta = 0.00000000001; + private static final String CONFIG_DIR = "src/test/resources/config/rankexpression/"; @Test public void testEvaluationDependingFunctionTakingArguments() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); function.bind("match", 3); function.bind("rankBoost", 5); @@ -41,7 +42,7 @@ public class ModelsEvaluatorTest { /** Tests a function defined as 4 * (var1 + var2) */ @Test public void testSettingMissingValue() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); { FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); @@ -127,18 +128,18 @@ public class ModelsEvaluatorTest { // TODO: Test argument-less function // TODO: Test with nested functions - private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)), + private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); + + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR).append("constants"), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index a15c35fe854..27d1c08ea39 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -1,11 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; -import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; @@ -19,6 +18,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * @author lesters @@ -26,10 +26,12 @@ import static org.junit.Assert.assertTrue; public class OnnxEvaluatorTest { private static final double delta = 0.00000000001; + private static final String CONFIG_DIR = "src/test/resources/config/onnx/"; @Test public void testOnnxEvaluation() { - ModelsEvaluator models = createModels("src/test/resources/config/onnx/"); + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + ModelsEvaluator models = createModels(); assertTrue(models.models().containsKey("add_mul")); assertTrue(models.models().containsKey("one_layer")); @@ -49,24 +51,23 @@ public class OnnxEvaluatorTest { assertEquals(function.evaluate(), Tensor.from("tensor<float>(d0[2],d1[1]):[0.63931,0.67574]")); } - private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); Map<String, File> fileMap = new HashMap<>(); for (OnnxModelsConfig.Model onnxModel : onnxModelsConfig.model()) { - fileMap.put(onnxModel.fileref().value(), new File(path + onnxModel.fileref().value())); + fileMap.put(onnxModel.fileref().value(), new File(CONFIG_DIR + onnxModel.fileref().value())); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); return new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 33e56d5d465..215e230b45d 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporterWithMockedConstants; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; @@ -19,13 +19,16 @@ import org.junit.Test; import java.util.HashMap; import java.util.Map; +import static org.junit.Assume.assumeTrue; + public class ModelsEvaluationHandlerTest { + private static final String MODELS_DIR = "src/test/resources/config/models/"; private static HandlerTester handler; @BeforeClass static public void setUp() { - handler = new HandlerTester(createModels("src/test/resources/config/models/")); + handler = new HandlerTester(createModels()); } @Test @@ -244,6 +247,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testMnistSavedEvaluateSpecificFunction() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Map<String, String> properties = new HashMap<>(); properties.put("input", inputTensor()); String url = "http://localhost/model-evaluation/v1/mnist_saved/serving_default.y/eval"; @@ -251,20 +255,20 @@ public class ModelsEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } - static private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)), + static private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); + + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR).append("constants"), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } + private static String fileConfigId(String filename) { + return "file:" + MODELS_DIR + filename; + } + private String inputTensor() { Tensor.Builder b = Tensor.Builder.of(TensorType.fromSpec("tensor(d0[],d1[784])")); for (int i = 0; i < 784; i++) diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 6014bd7c7ef..f065435ec15 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; -import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; @@ -19,13 +18,17 @@ import java.io.File; import java.util.HashMap; import java.util.Map; +import static org.junit.Assume.assumeTrue; + public class OnnxEvaluationHandlerTest { private static HandlerTester handler; + private static final String CONFIG_DIR = "src/test/resources/config/onnx/"; @BeforeClass static public void setUp() { - handler = new HandlerTester(createModels("src/test/resources/config/onnx/")); + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + handler = new HandlerTester(createModels()); } @Test @@ -114,24 +117,23 @@ public class OnnxEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } - static private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - RankingExpressionsConfig expressionsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-expressions.cfg").toFile()), - RankingExpressionsConfig.class).getConfig(""); - OnnxModelsConfig onnxModelsConfig = new ConfigGetter<>(new FileSource(configDir.append("onnx-models.cfg").toFile()), - OnnxModelsConfig.class).getConfig(""); + static private ModelsEvaluator createModels() { + RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); + RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); + RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); + OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); Map<String, File> fileMap = new HashMap<>(); for (OnnxModelsConfig.Model onnxModel : onnxModelsConfig.model()) { - fileMap.put(onnxModel.fileref().value(), new File(path + onnxModel.fileref().value())); + fileMap.put(onnxModel.fileref().value(), new File(CONFIG_DIR + onnxModel.fileref().value())); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); return new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); } + private static String fileConfigId(String filename) { + return "file:" + CONFIG_DIR + filename; + } + } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 87b964a2c56..c9ab9924214 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -93,4 +93,13 @@ public class OnnxEvaluator { } } + public static boolean isRuntimeAvailable() { + try { + new OnnxEvaluator(""); + return true; + } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { + return false; + } + } + } diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index 4f8ea362467..6266dcef174 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * @author lesters @@ -17,7 +18,8 @@ import static org.junit.Assert.assertEquals; public class OnnxEvaluatorTest { @Test - public void testSimpleMoodel() { + public void testSimpleModel() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/simple/simple.onnx"); // Input types @@ -42,6 +44,7 @@ public class OnnxEvaluatorTest { @Test public void testBatchDimension() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/pytorch/one_layer.onnx"); // Input types @@ -60,6 +63,7 @@ public class OnnxEvaluatorTest { @Test public void testMatMul() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]"; String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]"; String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]"; @@ -68,6 +72,7 @@ public class OnnxEvaluatorTest { @Test public void testTypes() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java index 12934c23926..af167fda5c6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -15,7 +15,7 @@ import java.util.Set; public class AddNode { public final String hostname; - public final Optional<String> id; + public final String id; public final Optional<String> parentHostname; public final Optional<String> nodeFlavor; public final Optional<FlavorOverrides> flavorOverrides; @@ -24,15 +24,15 @@ public class AddNode { public final Set<String> ipAddresses; public final Set<String> additionalIpAddresses; - public static AddNode forHost(String hostname, Optional<String> id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + public static AddNode forHost(String hostname, String id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { return new AddNode(hostname, id, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); } - public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { - return new AddNode(hostname, Optional.empty(), Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); + public static AddNode forNode(String hostname, String id, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { + return new AddNode(hostname, id, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); } - private AddNode(String hostname, Optional<String> id, Optional<String> parentHostname, + private AddNode(String hostname, String id, Optional<String> parentHostname, Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index acb6ece6fc1..6e31e699e2c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -26,7 +26,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; public class NodeSpec { private final String hostname; - private final Optional<String> id; + private final String id; private final NodeState state; private final NodeType type; private final String flavor; @@ -72,7 +72,7 @@ public class NodeSpec { public NodeSpec( String hostname, - Optional<String> id, + String id, Optional<DockerImage> wantedDockerImage, Optional<DockerImage> currentDockerImage, NodeState state, @@ -148,8 +148,8 @@ public class NodeSpec { return hostname; } - /** Returns the cloud-specific ID of the host. */ - public Optional<String> id() { + /** Returns unique node ID */ + public String id() { return id; } @@ -406,7 +406,7 @@ public class NodeSpec { public static class Builder { private String hostname; - private Optional<String> id = Optional.empty(); + private String id; private NodeState state; private NodeType type; private String flavor; @@ -441,6 +441,7 @@ public class NodeSpec { public Builder(NodeSpec node) { hostname(node.hostname); + id(node.id); state(node.state); type(node.type); flavor(node.flavor); @@ -477,7 +478,7 @@ public class NodeSpec { } public Builder id(String id) { - this.id = Optional.of(id); + this.id = id; return this; } @@ -786,6 +787,7 @@ public class NodeSpec { */ public static Builder testSpec(String hostname, NodeState state) { Builder builder = new Builder() + .id(hostname) .hostname(hostname) .state(state) .type(NodeType.tenant) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 38e725360a0..b99a3bb84d7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -162,7 +162,7 @@ public class RealNodeRepository implements NodeRepository { return new NodeSpec( node.hostname, - Optional.ofNullable(node.id), + node.id, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString), nodeState, @@ -244,7 +244,7 @@ public class RealNodeRepository implements NodeRepository { private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { NodeRepositoryNode node = new NodeRepositoryNode(); - node.id = addNode.id.orElse("fake-" + addNode.hostname); + node.id = addNode.id; node.hostname = addNode.hostname; node.parentHostname = addNode.parentHostname.orElse(null); addNode.nodeFlavor.ifPresent(f -> node.flavor = f); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java index 56e690864ac..6e88a41b8c6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java @@ -14,10 +14,7 @@ public enum ContainerNetworkMode { NPT("vespa-bridge"), /** A host running a single container in the host network namespace. */ - HOST_NETWORK("host"), - - /** A host running multiple containers in a shared local network. */ - LOCAL("vespa-bridge"); + HOST_NETWORK("host"); private final String networkName; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index dd0b8a2acdc..b6fb2bf345f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -214,6 +214,13 @@ public class StorageMaintainer { containerLogsOnHost.moveIfExists(containerLogsInArchiveDir); } new UnixPath(context.paths().of("/")).deleteRecursively(); + + // Operations on ContainerPath will fail if Container FS root doesn't exist, it is therefore important that + // it exists as long as NodeAgent is running. Normally the root is only created when NodeAgent is first + // started. Because non-tenant nodes are never removed from node-repo, we immediately re-create the new root + // after archiving the previous + if (context.nodeType() != NodeType.tenant) + context.paths().of("/").getFileSystem().createRoot(); } private String getMicrocodeVersion() { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index 8449f4e902f..0ea8c57dc57 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -88,7 +88,7 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { handleFailure(context, request, startedAt, "No artifacts requested"); return; } - ContainerPath directory = context.paths().underVespaHome("tmp/vespa-service-dump"); + ContainerPath directory = context.paths().underVespaHome("tmp/vespa-service-dump-" + request.getCreatedMillisOrNull()); UnixPath unixPathDirectory = new UnixPath(directory); try { context.log(log, Level.INFO, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 3d7a3b73ccd..dff72fe81f1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -79,7 +79,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts) { Map<String, NodeAgentContext> nodeAgentContextsByHostname = nodeAgentContexts.stream() - .collect(Collectors.toMap(NodeAdminImpl::nodeAgentId, Function.identity())); + .collect(Collectors.toMap(ctx -> ctx.node().id(), Function.identity())); // Stop and remove NodeAgents that should no longer be running diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet()) @@ -222,14 +222,4 @@ public class NodeAdminImpl implements NodeAdmin { NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context); return new NodeAgentWithScheduler(nodeAgent, contextManager); } - - private static String nodeAgentId(NodeAgentContext nac) { - // NodeAgentImpl has some internal state that should not be reused when the same hostname is re-allocated - // to a different application/cluster, solve this by including reservation timestamp in the key. - return nac.hostname().value() + "-" + nac.node().events().stream() - .filter(event -> "reserved".equals(event.type())) - .findFirst() - .map(event -> Long.toString(event.at().toEpochMilli())) - .orElse(""); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java new file mode 100644 index 00000000000..3249dab1ca8 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * Class wrapping a float stored on disk + * + * @author freva + */ +public class StoredDouble implements Supplier<OptionalDouble> { + + private static final Logger logger = Logger.getLogger(StoredDouble.class.getName()); + + private final UnixPath path; + + public StoredDouble(Path path) { + this.path = new UnixPath(path); + } + + @Override + public OptionalDouble get() { + return path.readUtf8FileIfExists().stream().mapToDouble(Double::parseDouble).findAny(); + } + + public void write(TaskContext taskContext, double value) { + path.writeUtf8File(Double.toString(value)); + taskContext.log(logger, "Stored new double in %s: %f", path, value); + } + + public void clear() { + path.deleteIfExists(); + } + + public Optional<Instant> getLastModifiedTime() { + return path.getAttributesIfExists().map(FileAttributes::lastModifiedTime); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java deleted file mode 100644 index 553f2ffba36..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; - -/** - * A very simple template engine when there's little complexity and lots of Velocity special characters $ and #, - * i.e. typically shell script. - * - * @author hakonhall - */ -public class Templar { - private final String template; - - private static final String prefix = "<%="; - private static final String suffix = "%>"; - - private final Map<String, String> settings = new HashMap<>(); - - public static Templar fromUtf8File(Path path) { - return new Templar(new UnixPath(path).readUtf8File()); - } - - public Templar(String template) { - this.template = template; - } - - public Templar set(String name, String value) { - settings.put(name, value); - return this; - } - - public String resolve() { - StringBuilder text = new StringBuilder(template.length() * 2); - - int start= 0; - int end; - - for (; start < template.length(); start = end) { - int prefixStart = template.indexOf(prefix, start); - - - if (prefixStart == -1) { - text.append(template, start, template.length()); - break; - } else { - text.append(template, start, prefixStart); - } - - int suffixStart = template.indexOf(suffix, prefixStart + prefix.length()); - if (suffixStart == -1) { - throw new IllegalArgumentException("Prefix at offset " + prefixStart + " is not terminated"); - } - - int prefixEnd = prefixStart + prefix.length(); - String name = template.substring(prefixEnd, suffixStart).trim(); - String value = settings.get(name); - if (value == null) { - throw new IllegalArgumentException("No value is set for name '" + name + "' at offset " + prefixEnd); - } - - text.append(value); - - end = suffixStart + suffix.length(); - } - - return text.toString(); - } - - public FileWriter getFileWriterTo(Path path) { - return new FileWriter(path, this::resolve); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java index 8663aba4244..852564e10a6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; -import com.yahoo.vespa.hosted.node.admin.task.util.text.SnippetGenerator; +import com.yahoo.text.internal.SnippetGenerator; /** * Base class for child process related exceptions, with a util to build an error message diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java index 42e13d63c19..14f72a5fa61 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.template; +import java.util.Locale; + /** * Public methods common to both Template and ListElement. * @@ -16,11 +18,11 @@ public interface Form { default Template set(String name, int value) { return set(name, Integer.toString(value)); } default Template set(String name, long value) { return set(name, Long.toString(value)); } - default Template set(String name, String format, String first, String... rest) { + default Template set(String name, String format, Object first, Object... rest) { var args = new Object[1 + rest.length]; args[0] = first; System.arraycopy(rest, 0, args, 1, rest.length); - var value = String.format(format, args); + var value = String.format(Locale.US, format, args); return set(name, value); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java index 4a00115cee4..fdb14189656 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java @@ -18,7 +18,7 @@ class IfSection extends Section { IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset, SectionList ifSections, Optional<SectionList> elseSections) { - super(range); + super("if", range); this.negated = negated; this.name = name; this.nameOffset = nameOffset; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java index 831dc3fe5e8..a137a27f3bb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java @@ -17,7 +17,7 @@ class ListSection extends Section { private final List<Template> elements = new ArrayList<>(); ListSection(CursorRange range, String name, Cursor nameOffset, Template body) { - super(range); + super("list", range); this.name = name; this.nameOffset = new Cursor(nameOffset); this.body = body; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java index c03653253af..667c515dcec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; */ class LiteralSection extends Section { LiteralSection(CursorRange range) { - super(range); + super("literal", range); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java index dd92af14609..0869b7f181c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java @@ -1,22 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.template; -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - /** * @author hakonhall */ public class NameAlreadyExistsTemplateException extends TemplateException { - public NameAlreadyExistsTemplateException(String name, CursorRange range) { - super("Name '" + name + "' already exists in the " + describeSection(range)); - } - - public NameAlreadyExistsTemplateException(String name, Cursor firstNameLocation, - Cursor secondNameLocation) { - super("Section named '" + name + "' at " + - firstNameLocation.calculateLocation().lineAndColumnText() + - " conflicts with earlier section with the same name at " + - secondNameLocation.calculateLocation().lineAndColumnText()); + public NameAlreadyExistsTemplateException(String name, Section first, Section second) { + super("The name '" + name + "' of the " + second.type() + " section at " + + second.range().start().calculateLocation().lineAndColumnText() + + " is in conflict with the identically named " + first.type() + " section at " + + first.range().start().calculateLocation().lineAndColumnText()); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java index 2c52fd5c34e..42927cd2c04 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java @@ -12,10 +12,12 @@ import java.util.Objects; * @author hakonhall */ abstract class Section { + private final String type; private final CursorRange range; private Template template; - protected Section(CursorRange range) { + protected Section(String type, CursorRange range) { + this.type = type; this.range = range; } @@ -24,6 +26,7 @@ abstract class Section { /** Guaranteed to return non-null after TemplateBuilder::build() returns. */ protected Template template() { return Objects.requireNonNull(template); } + protected String type() { return type; } protected CursorRange range() { return range; } abstract void appendTo(StringBuilder buffer); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java index 41e8c3e65ce..6d80ac2cad9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.template; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; @@ -25,12 +26,15 @@ import java.util.Optional; * id: a valid Java identifier * </pre> * + * <p>Other directive delimiters than "%{" and "}" may be used, see {@link TemplateDescriptor}.</p> + * * <p>Fill the template with variable values ({@link #set(String, String) set()}, set if conditions * ({@link #set(String, boolean)}), add list elements ({@link #add(String) add()}, etc, and finally * render it as a String ({@link #render()}).</p> * * <p>To reuse a template, create the template and work on snapshots of that ({@link #snapshot()}).</p> * + * @see TemplateDescriptor * @author hakonhall */ public class Template implements Form { @@ -85,6 +89,11 @@ public class Template implements Form { return template; } + public FileWriter getFileWriterTo(Path path) { + String content = render(); + return new FileWriter(path, () -> content); + } + /** Must be called (if there is a parent) before any other method. */ void setParent(Template parent) { this.parent = parent; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java index 8041a17fe74..2827a9eb005 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java @@ -35,8 +35,7 @@ class TemplateBuilder { ListSection existing = lists.get(section.name()); if (existing != null) - throw new NameAlreadyExistsTemplateException(section.name(), existing.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), existing, section); sampleVariables.put(section.name(), section); allSections.add(section); @@ -48,8 +47,7 @@ class TemplateBuilder { ListSection list = lists.get(section.name()); if (list != null) - throw new NameAlreadyExistsTemplateException(section.name(), list.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), list, section); sampleIfSections.put(section.name(), section); allSections.add(section); @@ -58,18 +56,15 @@ class TemplateBuilder { void addListSection(ListSection section) { VariableSection variableSection = sampleVariables.get(section.name()); if (variableSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), variableSection.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), variableSection, section); IfSection ifSection = sampleIfSections.get(section.name()); if (ifSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), ifSection.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), ifSection, section); ListSection previous = lists.put(section.name(), section); if (previous != null) - throw new NameAlreadyExistsTemplateException(section.name(), previous.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), previous, section); allSections.add(section); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java index c2202dea4a0..41fd716a3e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java @@ -47,9 +47,7 @@ class TemplateParser { if (current.eot()) { if (!sentinels.contains(Sentinel.EOT)) { - throw new BadTemplateException(current, - "Missing end directive for section started at " + - start.calculateLocation().lineAndColumnText()); + throw new BadTemplateException(start, "Missing end directive for section started"); } return Sentinel.EOT; } @@ -71,12 +69,12 @@ class TemplateParser { switch (type) { case "else": if (!sentinels.contains(Sentinel.ELSE)) - throw new BadTemplateException(startOfType, "Extraneous 'else'"); + throw new BadTemplateException(startOfType, "Stray 'else'"); parseEndDirective(); return Optional.of(Sentinel.ELSE); case "end": if (!sentinels.contains(Sentinel.END)) - throw new BadTemplateException(startOfType, "Extraneous 'end'"); + throw new BadTemplateException(startOfType, "Stray 'end'"); parseEndDirective(); return Optional.of(Sentinel.END); case "if": diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java index 6a7bec2e485..17fedce55fa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java @@ -15,7 +15,7 @@ class VariableSection extends Section { private final Cursor nameOffset; VariableSection(CursorRange range, String name, Cursor nameOffset) { - super(range); + super("variable", range); this.name = name; this.nameOffset = nameOffset; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 5d15d4353e2..3256b16a6c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -161,20 +161,20 @@ public class RealNodeRepositoryTest { @Test public void testAddNodes() { AddNode host = AddNode.forHost("host123.domain.tld", - Optional.of("id1"), + "id1", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost, Set.of("::1"), Set.of("::2", "::3")); NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); - AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); + AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); nodeRepositoryApi.addNodes(List.of(host, node)); NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); - assertEquals("id1", hostSpec.id().orElseThrow()); + assertEquals("id1", hostSpec.id()); assertEquals("default", hostSpec.flavor()); assertEquals(123, hostSpec.diskGb(), 0); assertEquals(NodeType.confighost, hostSpec.type()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java index 452efecefe1..fcdef83e06f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -81,11 +81,11 @@ class VespaServiceDumperImplTest { verify(operations).executeCommandInContainer( context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); verify(operations).executeCommandInContainer( - context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/tmp/vespa-service-dump/perf-record.bin", + context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/tmp/vespa-service-dump-1600000000000/perf-record.bin", "--pid=12345", "sleep", "45"); verify(operations).executeCommandInContainer( - context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/tmp/vespa-service-dump/perf-record.bin" + - " > /opt/vespa/tmp/vespa-service-dump/perf-report.txt"); + context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/tmp/vespa-service-dump-1600000000000/perf-record.bin" + + " > /opt/vespa/tmp/vespa-service-dump-1600000000000/perf-report.txt"); String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + @@ -124,7 +124,7 @@ class VespaServiceDumperImplTest { context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); verify(operations).executeCommandInContainer( context, context.users().vespa(), "jcmd", "12345", "JFR.start", "name=host-admin", "path-to-gc-roots=true", "settings=profile", - "filename=/opt/vespa/tmp/vespa-service-dump/recording.jfr", "duration=30s"); + "filename=/opt/vespa/tmp/vespa-service-dump-1600000000000/recording.jfr", "duration=30s"); verify(operations).executeCommandInContainer(context, context.users().vespa(), "jcmd", "12345", "JFR.check", "name=host-admin"); String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000," + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java deleted file mode 100644 index ed410ffc1d1..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author hakonhall - */ -public class TemplarTest { - @Test - public void test() { - Templar templar = new Templar("x y <%= foo %>, some other <%=bar%> text"); - templar.set("foo", "fidelity") - .set("bar", "halimov") - .set("not", "used"); - - assertEquals("x y fidelity, some other halimov text", templar.resolve()); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java index e010b9780c6..b0260af1582 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author hakonhall @@ -147,6 +148,70 @@ public class TemplateTest { template.render()); } + @Test + void badTemplates() { + assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6", + () -> Template.from("foo\nbar%{zoo}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=¬atoken}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8", + () -> Template.from("%{list ¬atoken}")); + + assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12", + () -> Template.from("%{list foo}missing end")); + + assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3", + () -> Template.from("%{end}stray end")); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset", + () -> Template.from("%{=notset}").render()); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond", + () -> Template.from("%{if cond}%{end}").render()); + + assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false", + () -> Template.from("%{if cond}%{end}").set("cond", 1).render()); + + assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " + + "line 1 and column 1, and ending at line 1 and column 4", + () -> Template.from("foo").add("listname")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " + + "named variable section at line 1 and column 1", + () -> Template.from("%{=a}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{=a}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " + + "named if section at line 1 and column 1", + () -> Template.from("%{if a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{if a}%{end}")); + } + + private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) { + T exception = assertThrows(class_, runnable::run); + assertEquals(message, exception.getMessage()); + } + private Template getTemplate(String filename) { return Template.at(Path.of("src/test/resources/" + filename)); } diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 0227ada15af..60aa0a83107 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -81,7 +81,7 @@ <dependency> <groupId>org.questdb</groupId> <artifactId>questdb</artifactId> - <version>6.0.4</version> + <version>6.2</version> <scope>compile</scope> </dependency> <dependency> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 7f54fff5c70..a387bc28aa4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.time.Clock; import java.util.List; @@ -59,6 +60,7 @@ public class NodeRepository extends AbstractComponent { private final LoadBalancers loadBalancers; private final FlagSource flagSource; private final MetricsDb metricsDb; + private final Orchestrator orchestrator; private final int spareCount; /** @@ -72,7 +74,8 @@ public class NodeRepository extends AbstractComponent { Curator curator, Zone zone, FlagSource flagSource, - MetricsDb metricsDb) { + MetricsDb metricsDb, + Orchestrator orchestrator) { this(flavors, provisionServiceProvider, curator, @@ -83,6 +86,7 @@ public class NodeRepository extends AbstractComponent { Optional.of(config.tenantContainerImage()).filter(s -> !s.isEmpty()).map(DockerImage::fromString), flagSource, metricsDb, + orchestrator, config.useCuratorClientCache(), zone.environment().isProduction() && !zone.getCloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0, config.nodeCacheSize()); @@ -102,6 +106,7 @@ public class NodeRepository extends AbstractComponent { Optional<DockerImage> tenantContainerImage, FlagSource flagSource, MetricsDb metricsDb, + Orchestrator orchestrator, boolean useCuratorClientCache, int spareCount, long nodeCacheSize) { @@ -113,7 +118,7 @@ public class NodeRepository extends AbstractComponent { this.db = new CuratorDatabaseClient(flavors, curator, clock, useCuratorClientCache, nodeCacheSize); this.zone = zone; this.clock = clock; - this.nodes = new Nodes(db, zone, clock); + this.nodes = new Nodes(db, zone, clock, orchestrator); this.flavors = flavors; this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator(); this.nameResolver = nameResolver; @@ -127,6 +132,7 @@ public class NodeRepository extends AbstractComponent { this.loadBalancers = new LoadBalancers(db); this.flagSource = flagSource; this.metricsDb = metricsDb; + this.orchestrator = orchestrator; this.spareCount = spareCount; nodes.rewrite(); } @@ -172,6 +178,8 @@ public class NodeRepository extends AbstractComponent { public MetricsDb metricsDb() { return metricsDb; } + public Orchestrator orchestrator() { return orchestrator; } + public NodeRepoStats computeStats() { return NodeRepoStats.computeOver(this); } /** Returns the time keeper of this system */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 08a9e373085..a73b6896c2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -18,6 +18,7 @@ import io.questdb.cairo.sql.Record; import io.questdb.cairo.sql.RecordCursor; import io.questdb.cairo.sql.RecordCursorFactory; import io.questdb.griffin.CompiledQuery; +import io.questdb.griffin.QueryFuture; import io.questdb.griffin.SqlCompiler; import io.questdb.griffin.SqlException; import io.questdb.griffin.SqlExecutionContext; @@ -341,6 +342,16 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } } + /** + * Issues and wait for an SQL statement to be executed against the QuestDb engine. + * Needs to be done for some queries, e.g. 'alter table' queries, see https://github.com/questdb/questdb/issues/1846 + */ + private void issueAsync(String sql, SqlExecutionContext context) throws SqlException { + try (QueryFuture future = issue(sql, context).execute(null)) { + future.await(); + } + } + private SqlExecutionContext newContext() { return new SqlExecutionContextImpl(engine(), 1); } @@ -374,7 +385,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { void gc() { synchronized (writeLock) { try { - issue("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext()); + issueAsync("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext()); } catch (SqlException e) { log.log(Level.WARNING, "Failed to gc old metrics data in " + dir + " table " + name, e); @@ -396,7 +407,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { void ensureColumnExists(String column, String columnType) throws SqlException { if (columnNames().contains(column)) return; - issue("alter table " + name + " add column " + column + " " + columnType, newContext()); + issueAsync("alter table " + name + " add column " + column + " " + columnType, newContext()); } private Optional<Long> adjustOrDiscard(Instant at) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index ad5bf1a2962..3e7da831bc4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.collections.AbstractFilteringList; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import java.util.Collection; @@ -21,6 +23,16 @@ public class LoadBalancerList extends AbstractFilteringList<LoadBalancer, LoadBa return matching(lb -> lb.state() == state); } + /** Returns the subset of load balancers in given cluster */ + public LoadBalancerList application(ApplicationId application) { + return matching(lb -> lb.id().application().equals(application)); + } + + /** Returns the subset of load balancers in given cluster */ + public LoadBalancerList cluster(ClusterSpec.Id cluster) { + return matching(lb -> lb.id().cluster().equals(cluster)); + } + public static LoadBalancerList copyOf(Collection<LoadBalancer> loadBalancers) { return new LoadBalancerList(loadBalancers, false); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java index d2c1aab72e2..7cbb8ef2764 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java @@ -2,20 +2,9 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.NodeAcl; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; -import java.util.Comparator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Predicate; -import java.util.stream.Collectors; /** * The load balancers of this node repo. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 341ab1f785c..e6476cd7373 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.NodesAndHosts; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Nodes; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; @@ -205,7 +204,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private Map<String, Node> findSharedHosts(NodeList nodeList) { return nodeList.stream() - .filter(node -> Nodes.canAllocateTenantNodeTo(node, true)) + .filter(node -> nodeRepository().nodes().canAllocateTenantNodeTo(node, true)) .filter(node -> node.reservedTo().isEmpty()) .filter(node -> node.exclusiveToApplicationId().isEmpty()) .collect(Collectors.toMap(Node::hostname, Function.identity())); @@ -298,7 +297,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { int wantedGroups = 1; NodePrioritizer prioritizer = new NodePrioritizer(nodesAndHosts, applicationId, clusterSpec, nodeSpec, wantedGroups, - true, nodeRepository().nameResolver(), nodeRepository().resourcesCalculator(), + true, nodeRepository().nameResolver(), nodeRepository().nodes(), nodeRepository().resourcesCalculator(), nodeRepository().spareCount()); List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of()); MutableInteger index = new MutableInteger(0); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 6133705ed59..3274f12dbc6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -42,6 +42,8 @@ import java.util.stream.Collectors; public class FailedExpirer extends NodeRepositoryMaintainer { private static final Logger log = Logger.getLogger(FailedExpirer.class.getName()); + // Try recycling nodes until reaching this many failures + private static final int maxAllowedFailures = 50; private final NodeRepository nodeRepository; private final Duration statefulExpiry; // Stateful nodes: Grace period to allow recovery of data @@ -85,11 +87,11 @@ public class FailedExpirer extends NodeRepositoryMaintainer { recycle(nodesToRecycle); } - /** Move eligible nodes to dirty. This may be a subset of the given nodes */ + /** Move eligible nodes to dirty or parked. This may be a subset of the given nodes */ private void recycle(List<Node> nodes) { List<Node> nodesToRecycle = new ArrayList<>(); for (Node candidate : nodes) { - if (NodeFailer.hasHardwareIssue(candidate, nodeRepository)) { + if (broken(candidate)) { List<String> unparkedChildren = !candidate.type().isHost() ? List.of() : nodeRepository.nodes().list() .childrenOf(candidate) @@ -98,7 +100,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { if (unparkedChildren.isEmpty()) { nodeRepository.nodes().park(candidate.hostname(), false, Agent.FailedExpirer, - "Parked by FailedExpirer due to hardware issue"); + "Parked by FailedExpirer due to hardware issue or high fail count"); } else { log.info(String.format("Expired failed node %s with hardware issue was not parked because of " + "unparked children: %s", candidate.hostname(), @@ -111,4 +113,10 @@ public class FailedExpirer extends NodeRepositoryMaintainer { nodeRepository.nodes().deallocate(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer"); } + /** Returns whether node is broken and cannot be recycled */ + private boolean broken(Node node) { + return NodeFailer.hasHardwareIssue(node, nodeRepository) || + (node.type().isHost() && node.status().failCount() >= maxAllowedFailures); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java index 4f913bb55dd..a1f36a4f1a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java @@ -88,7 +88,6 @@ public class InfrastructureVersions { case controllerhost: case proxyhost: case host: - case devhost: break; default: throw new IllegalArgumentException("Target version for type " + nodeType + " is not allowed"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index ca14a1be4c4..636884cef0a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.ClusterId; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.persistence.CacheStats; -import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -47,20 +46,17 @@ public class MetricsReporter extends NodeRepositoryMaintainer { private final Set<Pair<Metric.Context, String>> nonZeroMetrics = new HashSet<>(); private final Metric metric; - private final Orchestrator orchestrator; private final ServiceMonitor serviceMonitor; private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>(); private final Supplier<Integer> pendingRedeploymentsSupplier; MetricsReporter(NodeRepository nodeRepository, Metric metric, - Orchestrator orchestrator, ServiceMonitor serviceMonitor, Supplier<Integer> pendingRedeploymentsSupplier, Duration interval) { super(nodeRepository, interval, metric); this.metric = metric; - this.orchestrator = orchestrator; this.serviceMonitor = serviceMonitor; this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier; } @@ -201,7 +197,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { metric.set("wantToRetire", node.status().wantToRetire() ? 1 : 0, context); metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); - metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context); + metric.set("failReport", NodeFailer.reasonsToFailHost(node).isEmpty() ? 0 : 1, context); if (node.type().isHost()) { metric.set("wantToEncrypt", node.reports().getReport("wantToEncrypt").isPresent() ? 1 : 0, context); @@ -212,7 +208,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { serviceModel.getApplication(hostname) .map(ApplicationInstance::reference) - .map(reference -> orchestrator.getHostInfo(reference, hostname)) + .map(reference -> nodeRepository().orchestrator().getHostInfo(reference, hostname)) .ifPresent(info -> { int suspended = info.status().isSuspended() ? 1 : 0; metric.set("suspended", suspended, context); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 00881f5e2a8..a1916d7dc20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; @@ -15,17 +14,16 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; -import com.yahoo.vespa.orchestrator.HostNameNotFoundException; -import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; -import java.util.HashMap; +import java.util.Collection; +import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -40,7 +38,6 @@ import java.util.stream.Collectors; public class NodeFailer extends NodeRepositoryMaintainer { private static final Logger log = Logger.getLogger(NodeFailer.class.getName()); - private static final Duration nodeRequestInterval = Duration.ofMinutes(10); /** Metric for number of hosts that we want to fail, but cannot due to throttling */ static final String throttledHostFailuresMetric = "throttledHostFailures"; @@ -53,20 +50,17 @@ public class NodeFailer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Duration downTimeLimit; - private final Orchestrator orchestrator; - private final Instant constructionTime; + private final Duration suspendedDownTimeLimit; private final ThrottlePolicy throttlePolicy; private final Metric metric; public NodeFailer(Deployer deployer, NodeRepository nodeRepository, - Duration downTimeLimit, Duration interval, Orchestrator orchestrator, - ThrottlePolicy throttlePolicy, Metric metric) { + Duration downTimeLimit, Duration interval, ThrottlePolicy throttlePolicy, Metric metric) { // check ping status every interval, but at least twice as often as the down time limit super(nodeRepository, min(downTimeLimit.dividedBy(2), interval), metric); this.deployer = deployer; this.downTimeLimit = downTimeLimit; - this.orchestrator = orchestrator; - this.constructionTime = nodeRepository.clock().instant(); + this.suspendedDownTimeLimit = downTimeLimit.multipliedBy(4); // Allow more downtime when a node is suspended this.throttlePolicy = throttlePolicy; this.metric = metric; } @@ -82,38 +76,34 @@ public class NodeFailer extends NodeRepositoryMaintainer { // Ready nodes try (Mutex lock = nodeRepository().nodes().lockUnallocated()) { - for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { + for (FailingNode failing : findReadyFailingNodes()) { attempts++; - Node node = entry.getKey(); - if (throttle(node)) { + if (throttle(failing.node())) { failures++; - if (node.type().isHost()) + if (failing.node().type().isHost()) throttledHostFailures++; else throttledNodeFailures++; continue; } - String reason = entry.getValue(); - nodeRepository().nodes().fail(node.hostname(), Agent.NodeFailer, reason); + nodeRepository().nodes().fail(failing.node().hostname(), Agent.NodeFailer, failing.reason()); } } // Active nodes - for (Map.Entry<Node, String> entry : getActiveNodesByFailureReason().entrySet()) { + for (FailingNode failing : findActiveFailingNodes()) { attempts++; - Node node = entry.getKey(); - if (!failAllowedFor(node.type())) continue; + if (!failAllowedFor(failing.node().type())) continue; - if (throttle(node)) { + if (throttle(failing.node())) { failures++; - if (node.type().isHost()) + if (failing.node().type().isHost()) throttledHostFailures++; else throttledNodeFailures++; continue; } - String reason = entry.getValue(); - failActive(node, reason); + failActive(failing); } // Active hosts @@ -143,60 +133,54 @@ public class NodeFailer extends NodeRepositoryMaintainer { return asSuccessFactor(attempts, failures); } - private Map<Node, String> getReadyNodesByFailureReason() { - Instant oldestAcceptableRequestTime = - // Allow requests some time to be registered in case all config servers have been down - constructionTime.isAfter(clock().instant().minus(nodeRequestInterval.multipliedBy(2))) ? - Instant.EPOCH : - - // Nodes are taken as dead if they have not made a config request since this instant. - // Add 10 minutes to the down time limit to allow nodes to make a request that infrequently. - clock().instant().minus(downTimeLimit).minus(nodeRequestInterval); - - Map<Node, String> nodesByFailureReason = new HashMap<>(); + private Collection<FailingNode> findReadyFailingNodes() { + Set<FailingNode> failingNodes = new HashSet<>(); for (Node node : nodeRepository().nodes().list(Node.State.ready)) { Node hostNode = node.parentHostname().flatMap(parent -> nodeRepository().nodes().node(parent)).orElse(node); - List<String> failureReports = reasonsToFailParentHost(hostNode); + List<String> failureReports = reasonsToFailHost(hostNode); if (failureReports.size() > 0) { if (hostNode.equals(node)) { - nodesByFailureReason.put(node, "Host has failure reports: " + failureReports); + failingNodes.add(new FailingNode(node, "Host has failure reports: " + failureReports)); } else { - nodesByFailureReason.put(node, "Parent (" + hostNode + ") has failure reports: " + failureReports); + failingNodes.add(new FailingNode(node, "Parent (" + hostNode + ") has failure reports: " + failureReports)); } } } - return nodesByFailureReason; + return failingNodes; } - private Map<Node, String> getActiveNodesByFailureReason() { + private Collection<FailingNode> findActiveFailingNodes() { + Set<FailingNode> failingNodes = new HashSet<>(); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); - Instant graceTimeEnd = clock().instant().minus(downTimeLimit); - Map<Node, String> nodesByFailureReason = new HashMap<>(); + for (Node node : activeNodes) { - if (node.history().hasEventBefore(History.Event.Type.down, graceTimeEnd) && ! applicationSuspended(node)) { + Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit); + if (node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node)) { // Allow a grace period after node re-activation - if ( ! node.history().hasEventAfter(History.Event.Type.activated, graceTimeEnd)) - nodesByFailureReason.put(node, "Node has been down longer than " + downTimeLimit); + if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart)) + failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit)); } - else if (hostSuspended(node, activeNodes)) { - Node hostNode = node.parentHostname().flatMap(parent -> nodeRepository().nodes().node(parent)).orElse(node); - if (hostNode.type().isHost()) { - List<String> failureReports = reasonsToFailParentHost(hostNode); - if (failureReports.size() > 0) { - if (hostNode.equals(node)) { - nodesByFailureReason.put(node, "Host has failure reports: " + failureReports); - } else { - nodesByFailureReason.put(node, "Parent (" + hostNode + ") has failure reports: " + failureReports); - } + } + + for (Node node : activeNodes) { + if (allSuspended(node, activeNodes)) { + Node host = node.parentHostname().flatMap(parent -> nodeRepository().nodes().node(parent)).orElse(node); + if (host.type().isHost()) { + List<String> failureReports = reasonsToFailHost(host); + if ( ! failureReports.isEmpty()) { + failingNodes.add(new FailingNode(node, host.equals(node) ? + "Host has failure reports: " + failureReports : + "Parent " + host + " has failure reports: " + failureReports)); } } } } - return nodesByFailureReason; + + return failingNodes; } - public static List<String> reasonsToFailParentHost(Node hostNode) { - return hostNode.reports().getReports().stream() + public static List<String> reasonsToFailHost(Node host) { + return host.reports().getReports().stream() .filter(report -> report.getType().hostShouldBeFailed()) // The generated string is built from the report's ID, created time, and description only. .map(report -> report.getReportId() + " reported " + report.getCreatedTime() + ": " + report.getDescription()) @@ -205,37 +189,28 @@ public class NodeFailer extends NodeRepositoryMaintainer { /** Returns whether node has any kind of hardware issue */ static boolean hasHardwareIssue(Node node, NodeRepository nodeRepository) { - Node hostNode = node.parentHostname().flatMap(parent -> nodeRepository.nodes().node(parent)).orElse(node); - return reasonsToFailParentHost(hostNode).size() > 0; + Node host = node.parentHostname().flatMap(parent -> nodeRepository.nodes().node(parent)).orElse(node); + return reasonsToFailHost(host).size() > 0; } private boolean applicationSuspended(Node node) { try { - return orchestrator.getApplicationInstanceStatus(node.allocation().get().owner()) + return nodeRepository().orchestrator().getApplicationInstanceStatus(node.allocation().get().owner()) == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; } catch (ApplicationIdNotFoundException e) { - //Treat it as not suspended and allow to fail the node anyway - return false; - } - } - - private boolean nodeSuspended(Node node) { - try { - return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); - } catch (HostNameNotFoundException e) { - // Treat it as not suspended + // Treat it as not suspended and allow to fail the node anyway return false; } } /** Is the node and all active children suspended? */ - private boolean hostSuspended(Node node, NodeList activeNodes) { - if (!nodeSuspended(node)) return false; + private boolean allSuspended(Node node, NodeList activeNodes) { + if (!nodeRepository().nodes().suspended(node)) return false; if (node.parentHostname().isPresent()) return true; // optimization return activeNodes.stream() .filter(childNode -> childNode.parentHostname().isPresent() && childNode.parentHostname().get().equals(node.hostname())) - .allMatch(this::nodeSuspended); + .allMatch(nodeRepository().nodes()::suspended); } /** @@ -264,40 +239,40 @@ public class NodeFailer extends NodeRepositoryMaintainer { * * @return whether node was successfully failed */ - private boolean failActive(Node node, String reason) { + private boolean failActive(FailingNode failing) { Optional<Deployment> deployment = - deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30)); + deployer.deployFromLocalActive(failing.node().allocation().get().owner(), Duration.ofMinutes(30)); if (deployment.isEmpty()) return false; - try (Mutex lock = nodeRepository().nodes().lock(node.allocation().get().owner())) { + try (Mutex lock = nodeRepository().nodes().lock(failing.node().allocation().get().owner())) { // If the active node that we are trying to fail is of type host, we need to successfully fail all // the children nodes running on it before we fail the host boolean allTenantNodesFailedOutSuccessfully = true; - String reasonForChildFailure = "Failing due to parent host " + node.hostname() + " failure: " + reason; - for (Node failingTenantNode : nodeRepository().nodes().list().childrenOf(node)) { + String reasonForChildFailure = "Failing due to parent host " + failing.node().hostname() + " failure: " + failing.reason(); + for (Node failingTenantNode : nodeRepository().nodes().list().childrenOf(failing.node())) { if (failingTenantNode.state() == Node.State.active) { - allTenantNodesFailedOutSuccessfully &= failActive(failingTenantNode, reasonForChildFailure); + allTenantNodesFailedOutSuccessfully &= failActive(new FailingNode(failingTenantNode, reasonForChildFailure)); } else { nodeRepository().nodes().fail(failingTenantNode.hostname(), Agent.NodeFailer, reasonForChildFailure); } } if (! allTenantNodesFailedOutSuccessfully) return false; - wantToFail(node, true, lock); + wantToFail(failing.node(), true, lock); try { deployment.get().activate(); return true; } catch (TransientException e) { - log.log(Level.INFO, "Failed to redeploy " + node.allocation().get().owner() + + log.log(Level.INFO, "Failed to redeploy " + failing.node().allocation().get().owner() + " with a transient error, will be retried by application maintainer: " + Exceptions.toMessageString(e)); return true; } catch (RuntimeException e) { // Reset want to fail: We'll retry failing unless it heals in the meantime - nodeRepository().nodes().node(node.hostname()) + nodeRepository().nodes().node(failing.node().hostname()) .ifPresent(n -> wantToFail(n, false, lock)); - log.log(Level.WARNING, "Could not fail " + node + " for " + node.allocation().get().owner() + - " for " + reason + ": " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Could not fail " + failing.node() + " for " + failing.node().allocation().get().owner() + + " for " + failing.reason() + ": " + Exceptions.toMessageString(e)); return false; } } @@ -359,4 +334,30 @@ public class NodeFailer extends NodeRepositoryMaintainer { } + private static class FailingNode { + + private final Node node; + private final String reason; + + public FailingNode(Node node, String reason) { + this.node = node; + this.reason = reason; + } + + public Node node() { return node; } + public String reason() { return reason; } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof FailingNode)) return false; + return ((FailingNode)other).node().equals(this.node()); + } + + @Override + public int hashCode() { + return node.hashCode(); + } + + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java index 57db874fb84..552db84748d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java @@ -74,6 +74,8 @@ public abstract class NodeMover<MOVE> extends NodeRepositoryMaintainer { if (deployedRecently(applicationId)) continue; for (HostWithResources toHost : hostResources) { if (toHost.node.hostname().equals(node.parentHostname().get())) continue; + if (toHost.node.reservedTo().isPresent() && + !toHost.node.reservedTo().get().equals(applicationId.tenant())) continue; // Reserved to a different tenant if (spares.contains(toHost.node)) continue; // Do not offer spares as a valid move as they are reserved for replacement of failed nodes if ( ! toHost.hasCapacity(node.resources())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 2f200032492..15decde0d7c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.MetricsFetcher; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; -import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.time.Duration; @@ -35,7 +34,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { @Inject public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, - Zone zone, Orchestrator orchestrator, Metric metric, + Zone zone, Metric metric, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, MetricsFetcher metricsFetcher) { DefaultTimes defaults = new DefaultTimes(zone, deployer); @@ -46,11 +45,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(periodicApplicationMaintainer); maintainers.add(infrastructureProvisioner); - maintainers.add(new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, orchestrator, defaults.throttlePolicy, metric)); + maintainers.add(new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, defaults.throttlePolicy, metric)); maintainers.add(new NodeHealthTracker(hostLivenessTracker, serviceMonitor, nodeRepository, defaults.nodeFailureStatusUpdateInterval, metric)); maintainers.add(new ExpeditedChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.expeditedChangeRedeployInterval)); maintainers.add(new ReservationExpirer(nodeRepository, defaults.reservationExpiry, metric)); - maintainers.add(new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry)); + maintainers.add(new RetiredExpirer(nodeRepository, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry)); maintainers.add(new InactiveExpirer(nodeRepository, defaults.inactiveExpiry, Map.of(NodeType.config, defaults.inactiveConfigServerExpiry, NodeType.controller, defaults.inactiveControllerExpiry), metric)); @@ -58,7 +57,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new DirtyExpirer(nodeRepository, defaults.dirtyExpiry, metric)); maintainers.add(new ProvisionedExpirer(nodeRepository, defaults.provisionedExpiry, metric)); maintainers.add(new NodeRebooter(nodeRepository, flagSource, metric)); - maintainers.add(new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval)); + maintainers.add(new MetricsReporter(nodeRepository, metric, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval)); maintainers.add(new SpareCapacityMaintainer(deployer, nodeRepository, metric, defaults.spareCapacityMaintenanceInterval)); maintainers.add(new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval, metric)); maintainers.add(new Rebalancer(deployer, nodeRepository, metric, defaults.rebalancerInterval)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index a7ba8b27851..73c9a1ab55a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.OrchestrationException; -import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -31,11 +30,9 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Metric metric; - private final Orchestrator orchestrator; private final Duration retiredExpiry; public RetiredExpirer(NodeRepository nodeRepository, - Orchestrator orchestrator, Deployer deployer, Metric metric, Duration maintenanceInterval, @@ -43,7 +40,6 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { super(nodeRepository, maintenanceInterval, metric); this.deployer = deployer; this.metric = metric; - this.orchestrator = orchestrator; this.retiredExpiry = retiredExpiry; } @@ -126,7 +122,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { } try { - orchestrator.acquirePermissionToRemove(new HostName(node.hostname())); + nodeRepository().orchestrator().acquirePermissionToRemove(new HostName(node.hostname())); log.info("Node " + node + " has been granted permission to be removed"); return true; } catch (UncheckedTimeoutException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java index 88a62c94f43..ac24c83e129 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java @@ -109,10 +109,9 @@ public class NodeAcl { case proxy: // Proxy nodes trust: // - config servers - // - all connections from the world on 4080 (insecure tb removed), and 4443 + // - all connections from the world on 443 (production traffic) and 4443 (health checks) trustedNodes.addAll(allNodes.nodeType(NodeType.config).asList()); trustedPorts.add(443); - trustedPorts.add(4080); trustedPorts.add(4443); break; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 7f57ec219ae..57a3b436e37 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; @@ -18,6 +19,8 @@ import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; +import com.yahoo.vespa.orchestrator.HostNameNotFoundException; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.time.Clock; import java.time.Duration; @@ -53,14 +56,16 @@ public class Nodes { private static final Logger log = Logger.getLogger(Nodes.class.getName()); + private final CuratorDatabaseClient db; private final Zone zone; private final Clock clock; - private final CuratorDatabaseClient db; + private final Orchestrator orchestrator; - public Nodes(CuratorDatabaseClient db, Zone zone, Clock clock) { + public Nodes(CuratorDatabaseClient db, Zone zone, Clock clock, Orchestrator orchestrator) { this.zone = zone; this.clock = clock; this.db = db; + this.orchestrator = orchestrator; } /** Read and write all nodes to make sure they are stored in the latest version of the serialized format */ @@ -474,7 +479,7 @@ public class Nodes { if (node.state() == Node.State.ready) return node; Node parentHost = node.parentHostname().flatMap(this::node).orElse(node); - List<String> failureReasons = NodeFailer.reasonsToFailParentHost(parentHost); + List<String> failureReasons = NodeFailer.reasonsToFailHost(parentHost); if ( ! failureReasons.isEmpty()) illegal(node + " cannot be readied because it has hard failures: " + failureReasons); @@ -728,10 +733,11 @@ public class Nodes { return canAllocateTenantNodeTo(host, zone.getCloud().dynamicProvisioning()); } - public static boolean canAllocateTenantNodeTo(Node host, boolean dynamicProvisioning) { + public boolean canAllocateTenantNodeTo(Node host, boolean dynamicProvisioning) { if ( ! host.type().canRun(NodeType.tenant)) return false; if (host.status().wantToRetire()) return false; if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false; + if (suspended(host)) return false; if (dynamicProvisioning) return EnumSet.of(Node.State.active, Node.State.ready, Node.State.provisioned).contains(host.state()); @@ -739,6 +745,15 @@ public class Nodes { return host.state() == Node.State.active; } + public boolean suspended(Node node) { + try { + return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); + } catch (HostNameNotFoundException e) { + // Treat it as not suspended + return false; + } + } + /** Create a lock which provides exclusive rights to making changes to the given application */ // TODO: Move to Applications public Mutex lock(ApplicationId application) { @@ -819,6 +834,7 @@ public class Nodes { private static boolean parkOnDeallocationOf(Node node, Agent agent) { if (node.state() == Node.State.parked) return false; if (agent == Agent.operator) return false; + if (!node.type().isHost() && node.status().wantToDeprovision()) return false; boolean retirementRequestedByOperator = node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire) .map(History.Event::agent) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index ba28d8e6b9a..379bb2566df 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.component.Version; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 543972a9cb3..cd1b786afd1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -554,7 +554,6 @@ public class NodeSerializer { case "confighost": return NodeType.confighost; case "controller": return NodeType.controller; case "controllerhost": return NodeType.controllerhost; - case "devhost": return NodeType.devhost; default : throw new IllegalArgumentException("Unknown node type '" + typeString + "'"); } } @@ -569,7 +568,6 @@ public class NodeSerializer { case confighost: return "confighost"; case controller: return "controller"; case controllerhost: return "controllerhost"; - case devhost: return "devhost"; } throw new IllegalArgumentException("Serialized form of '" + type + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 8c358301b85..3da0506f2e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -45,8 +45,8 @@ class Activator { /** Activate required resources for application guarded by given lock */ public void activate(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { - activateNodes(hosts, generation, transaction); - activateLoadBalancers(hosts, transaction); + NodeList newActive = activateNodes(hosts, generation, transaction); + activateLoadBalancers(hosts, newActive, transaction); } /** @@ -62,8 +62,9 @@ class Activator { * @param generation the application config generation that is activated * @param transaction transaction with operations to commit together with any operations done within the repository, * while holding the node repository lock on this application + * @return the nodes that will be active when transaction is committed */ - private void activateNodes(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { + private NodeList activateNodes(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { Instant activationTime = nodeRepository.clock().instant(); // Use one timestamp for all activation changes ApplicationId application = transaction.application(); Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); @@ -95,6 +96,7 @@ class Activator { oldActive.not().retired(), newActive.not().retired()); unreserveParentsOf(reserved); + return newActive; } private void deactivate(NodeList toDeactivate, ApplicationTransaction transaction) { @@ -149,8 +151,8 @@ class Activator { } /** Activate load balancers */ - private void activateLoadBalancers(Collection<HostSpec> hosts, ApplicationTransaction transaction) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(allClustersOf(hosts), transaction)); + private void activateLoadBalancers(Collection<HostSpec> hosts, NodeList newActive, ApplicationTransaction transaction) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(allClustersOf(hosts), newActive, transaction)); } private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 4088d717a67..290a3f8f947 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -72,10 +71,6 @@ public class CapacityPolicies { public NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) { - if (zone.system() == SystemName.dev) { - // Use small logserver in dev system - return new NodeResources(0.1, 1, 10, 0.3); - } return zone.getCloud().dynamicProvisioning() && ! sharedHosts.apply(clusterType) ? new NodeResources(0.5, 4, 50, 0.3) : new NodeResources(0.5, 2, 50, 0.3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 2d93763c631..ae65f367684 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -149,6 +149,7 @@ public class GroupPreparer { wantedGroups, nodeRepository.zone().getCloud().dynamicProvisioning(), nodeRepository.nameResolver(), + nodeRepository.nodes(), nodeRepository.resourcesCalculator(), nodeRepository.spareCount()); allocation.offer(prioritizer.collect(surplusActiveNodes)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 5ff78c53f8a..04f084dd079 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -100,11 +100,11 @@ public class LoadBalancerProvisioner { * * Calling this when no load balancer has been prepared for given cluster is a no-op. */ - public void activate(Set<ClusterSpec> clusters, ApplicationTransaction transaction) { + public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) { Set<ClusterSpec.Id> activatingClusters = clusters.stream() .map(LoadBalancerProvisioner::effectiveId) .collect(Collectors.toSet()); - for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { + for (var cluster : loadBalancedClustersOf(newActive).entrySet()) { if (!activatingClusters.contains(cluster.getKey())) continue; Node clusterNode = cluster.getValue().first().get(); @@ -232,12 +232,13 @@ public class LoadBalancerProvisioner { /** Returns the nodes allocated to the given load balanced cluster */ private NodeList nodesOf(ClusterSpec.Id loadBalancedCluster, ApplicationId application) { - return loadBalancedClustersOf(application).getOrDefault(loadBalancedCluster, NodeList.copyOf(List.of())); + NodeList nodes = nodeRepository.nodes().list(Node.State.reserved, Node.State.active) + .owner(application); + return loadBalancedClustersOf(nodes).getOrDefault(loadBalancedCluster, NodeList.of()); } /** Returns the load balanced clusters of given application and their nodes */ - private Map<ClusterSpec.Id, NodeList> loadBalancedClustersOf(ApplicationId application) { - NodeList nodes = nodeRepository.nodes().list(Node.State.reserved, Node.State.active).owner(application); + private Map<ClusterSpec.Id, NodeList> loadBalancedClustersOf(NodeList nodes) { if (nodes.stream().anyMatch(node -> node.type() == NodeType.config)) { nodes = nodes.nodeType(NodeType.config).type(ClusterSpec.Type.admin); } else if (nodes.stream().anyMatch(node -> node.type() == NodeType.controller)) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 85a43e38e07..fe4eb5d68c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -30,13 +30,14 @@ import java.util.stream.Collectors; */ public class NodePrioritizer { - private final List<NodeCandidate> nodes = new ArrayList<>(); + private final List<NodeCandidate> candidates = new ArrayList<>(); private final NodesAndHosts<LockedNodeList> allNodesAndHosts; private final HostCapacity capacity; private final NodeSpec requestedNodes; private final ApplicationId application; private final ClusterSpec clusterSpec; private final NameResolver nameResolver; + private final Nodes nodes; private final boolean dynamicProvisioning; /** Whether node specification allows new nodes to be allocated. */ private final boolean canAllocateNew; @@ -46,7 +47,7 @@ public class NodePrioritizer { private final Set<Node> spareHosts; public NodePrioritizer(NodesAndHosts<LockedNodeList> allNodesAndHosts, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, + int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes, HostResourcesCalculator hostResourcesCalculator, int spareCount) { this.allNodesAndHosts = allNodesAndHosts; this.capacity = new HostCapacity(this.allNodesAndHosts, hostResourcesCalculator); @@ -58,6 +59,7 @@ public class NodePrioritizer { capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodesAndHosts.nodes().asList()) : capacity.findSpareHosts(this.allNodesAndHosts.nodes().asList(), spareCount); this.nameResolver = nameResolver; + this.nodes = nodes; NodeList nodesInCluster = this.allNodesAndHosts.nodes().owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); @@ -95,12 +97,12 @@ public class NodePrioritizer { /** Returns the list of nodes sorted by {@link NodeCandidate#compareTo(NodeCandidate)} */ private List<NodeCandidate> prioritize() { // Group candidates by their switch hostname - Map<String, List<NodeCandidate>> candidatesBySwitch = this.nodes.stream() + Map<String, List<NodeCandidate>> candidatesBySwitch = this.candidates.stream() .collect(Collectors.groupingBy(candidate -> candidate.parent.orElseGet(candidate::toNode) .switchHostname() .orElse(""))); // Mark lower priority nodes on shared switch as non-exclusive - List<NodeCandidate> nodes = new ArrayList<>(this.nodes.size()); + List<NodeCandidate> nodes = new ArrayList<>(this.candidates.size()); for (var clusterSwitch : candidatesBySwitch.keySet()) { List<NodeCandidate> switchCandidates = candidatesBySwitch.get(clusterSwitch); if (clusterSwitch.isEmpty()) { @@ -126,7 +128,7 @@ public class NodePrioritizer { for (Node node : surplusNodes) { NodeCandidate candidate = candidateFrom(node, true); if (!candidate.violatesSpares || canAllocateToSpareHosts) { - nodes.add(candidate); + candidates.add(candidate); } } } @@ -136,7 +138,7 @@ public class NodePrioritizer { if ( !canAllocateNew) return; for (Node host : allNodesAndHosts.nodes()) { - if ( ! Nodes.canAllocateTenantNodeTo(host, dynamicProvisioning)) continue; + if ( ! nodes.canAllocateTenantNodeTo(host, dynamicProvisioning)) continue; if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; if (host.reservedTo().isPresent() && application.instance().isTester()) continue; if (host.exclusiveToApplicationId().isPresent()) continue; // Never allocate new nodes to exclusive hosts @@ -144,7 +146,7 @@ public class NodePrioritizer { if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requestedNodes.resources().get())) continue; if ( ! allNodesAndHosts.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; - nodes.add(NodeCandidate.createNewChild(requestedNodes.resources().get(), + candidates.add(NodeCandidate.createNewChild(requestedNodes.resources().get(), capacity.availableCapacityOf(host), host, spareHosts.contains(host), @@ -164,7 +166,7 @@ public class NodePrioritizer { .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) .filter(node -> node.state() == Node.State.active || canStillAllocate(node)) .map(node -> candidateFrom(node, false)) - .forEach(nodes::add); + .forEach(candidates::add); } /** Add nodes already provisioned, but not allocated to any application */ @@ -174,7 +176,7 @@ public class NodePrioritizer { .filter(node -> node.state() == Node.State.ready) .map(node -> candidateFrom(node, false)) .filter(n -> !n.violatesSpares || canAllocateToSpareHosts) - .forEach(nodes::add); + .forEach(candidates::add); } /** Create a candidate from given pre-existing node */ @@ -218,7 +220,7 @@ public class NodePrioritizer { private boolean canStillAllocate(Node node) { if (node.type() != NodeType.tenant || node.parentHostname().isEmpty()) return true; Optional<Node> parent = allNodesAndHosts.parentOf(node); - return parent.isPresent() ? Nodes.canAllocateTenantNodeTo(parent.get(), dynamicProvisioning) : null; + return parent.isPresent() && nodes.canAllocateTenantNodeTo(parent.get(), dynamicProvisioning); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index d5dbe08dca9..310f921367e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -71,7 +71,6 @@ public class NodeResourceLimits { } private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) { - if (zone().system() == SystemName.dev) return 1; // Allow small containers in dev system if (clusterType == ClusterSpec.Type.admin) return 1; return 4; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java index aa6209ae80d..1ba686772c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -18,7 +18,7 @@ public class LoadBalancersV1ApiHandler extends RestApiRequestHandler<LoadBalance private final NodeRepository nodeRepository; @Inject - public LoadBalancersV1ApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { + public LoadBalancersV1ApiHandler(ThreadedHttpRequestHandler.Context parentCtx, NodeRepository nodeRepository) { super(parentCtx, LoadBalancersV1ApiHandler::createRestApiDefinition); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index be011c886a5..6282c072001 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -53,7 +53,6 @@ public class NodeSerializer { case "confighost": return NodeType.confighost; case "controller": return NodeType.controller; case "controllerhost": return NodeType.controllerhost; - case "devhost": return NodeType.devhost; default: throw new IllegalArgumentException("Unknown node type '" + nodeType + "'"); } } @@ -68,7 +67,6 @@ public class NodeSerializer { case confighost: return "confighost"; case controller: return "controller"; case controllerhost: return "controllerhost"; - case devhost: return "devhost"; default: throw new IllegalArgumentException("Unknown node type '" + type.name() + "'"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 15e1061f5e1..1304b85be6b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; @@ -67,7 +67,7 @@ import static com.yahoo.slime.SlimeUtils.optionalString; * * @author bratseth */ -public class NodesV2ApiHandler extends LoggingRequestHandler { +public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; @@ -75,7 +75,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { private final NodeFlavors nodeFlavors; @Inject - public NodesV2ApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, + public NodesV2ApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, MetricsDb metricsDb, NodeFlavors flavors) { super(parentCtx); this.orchestrator = orchestrator; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 1a2d5294aa5..ff406efdc39 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -70,6 +70,7 @@ public class MockNodeRepository extends NodeRepository { Optional.empty(), new InMemoryFlagSource(), new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))), + new OrchestratorMock(), true, 0, 1000); this.flavors = flavors; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index b391292884f..65a57ebd53e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import java.util.List; import java.util.Optional; @@ -47,6 +48,7 @@ public class NodeRepositoryTester { Optional.empty(), new InMemoryFlagSource(), new MemoryMetricsDb(clock), + new OrchestratorMock(), true, 0, 1000); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index e06bdba90fb..68aea0e9056 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import java.io.IOException; import java.nio.file.Files; @@ -73,6 +74,7 @@ public class CapacityCheckerTester { Optional.empty(), new InMemoryFlagSource(), new MemoryMetricsDb(clock), + new OrchestratorMock(), true, 0, 1000); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 5e9137f8713..5211b855fff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -126,7 +126,7 @@ public class InactiveAndFailedExpirerTest { ); Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); - new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(), + new RetiredExpirer(tester.nodeRepository(), deployer, new TestMetric(), Duration.ofDays(30), Duration.ofMinutes(10)).run(); assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 8c649243d61..98d3ffa92f8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -339,7 +339,6 @@ public class MetricsReporterTest { private MetricsReporter metricsReporter(TestMetric metric, ProvisioningTester tester) { return new MetricsReporter(tester.nodeRepository(), metric, - tester.orchestrator(), serviceMonitor, () -> 42, LONG_INTERVAL); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 0ea7011a930..f67e9cd8345 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -26,10 +26,8 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub; import com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker; -import com.yahoo.vespa.orchestrator.Orchestrator; import java.time.Clock; import java.time.Duration; @@ -66,14 +64,11 @@ public class NodeFailTester { public MockDeployer deployer; public TestMetric metric; private final TestHostLivenessTracker hostLivenessTracker; - private final Orchestrator orchestrator; private final NodeRepositoryProvisioner provisioner; private final Curator curator; private NodeFailTester() { - orchestrator = new OrchestratorMock(); - tester = new ProvisioningTester.Builder().orchestrator(orchestrator) - .flavors(hostFlavors.getFlavors()) + tester = new ProvisioningTester.Builder().flavors(hostFlavors.getFlavors()) .spareCount(1).build(); clock = tester.clock(); curator = tester.getCurator(); @@ -215,7 +210,7 @@ public class NodeFailTester { public void suspend(ApplicationId app) { try { - orchestrator.suspend(app); + nodeRepository.orchestrator().suspend(app); } catch (Exception e) { throw new RuntimeException(e); } @@ -223,7 +218,7 @@ public class NodeFailTester { public void suspend(String hostName) { try { - orchestrator.suspend(new HostName(hostName)); + nodeRepository.orchestrator().suspend(new HostName(hostName)); } catch (Exception e) { throw new RuntimeException(e); } @@ -231,7 +226,7 @@ public class NodeFailTester { public NodeFailer createFailer() { return new NodeFailer(deployer, nodeRepository, downtimeLimitOneHour, - Duration.ofMinutes(5), orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); + Duration.ofMinutes(5), NodeFailer.ThrottlePolicy.hosted, metric); } public NodeHealthTracker createUpdater() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 4575f7b4355..1237ede5345 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.testutils.MockDuperModel; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.duper.ConfigServerApplication; import org.junit.Before; import org.junit.Test; @@ -60,11 +61,11 @@ public class RetiredExpirerTest { private final NodeResources hostResources = new NodeResources(64, 128, 2000, 10); private final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final Orchestrator orchestrator = mock(Orchestrator.class); + private final ProvisioningTester tester = new ProvisioningTester.Builder().orchestrator(orchestrator).build(); private final ManualClock clock = tester.clock(); private final NodeRepository nodeRepository = tester.nodeRepository(); private final NodeRepositoryProvisioner provisioner = tester.provisioner(); - private final Orchestrator orchestrator = mock(Orchestrator.class); private static final Duration RETIRED_EXPIRATION = Duration.ofHours(12); @@ -72,6 +73,7 @@ public class RetiredExpirerTest { public void setup() throws OrchestrationException { // By default, orchestrator should deny all request for suspension so we can test expiration doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); + when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS); } @Test @@ -269,7 +271,6 @@ public class RetiredExpirerTest { private RetiredExpirer createRetiredExpirer(Deployer deployer) { return new RetiredExpirer(nodeRepository, - orchestrator, deployer, new TestMetric(), Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index 373bfe20162..2fa18681ece 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvid import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.Ignore; import org.junit.Test; @@ -267,6 +268,7 @@ public class SpareCapacityMaintainerTest { Optional.empty(), new InMemoryFlagSource(), new MemoryMetricsDb(clock), + new OrchestratorMock(), true, 1, 1000); deployer = new MockDeployer(nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 23f5701f825..2346b9e2fab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -100,6 +100,7 @@ public class AclProvisioningTest { // Trusted nodes is all tenant nodes, all proxy nodes, all config servers and load balancer subnets assertAcls(List.of(tenantNodes.asList(), proxyNodes, configServers.asList()), Set.of("10.2.3.0/24", "10.4.5.0/24"), List.of(nodeAcl)); + assertEquals(Set.of(22, 4443), nodeAcl.trustedPorts()); } @Test @@ -121,6 +122,7 @@ public class AclProvisioningTest { // Trusted nodes is all config servers and all proxy nodes assertAcls(List.of(proxyNodes.asList(), configServers.asList()), List.of(nodeAcl)); + assertEquals(Set.of(22, 443, 4443), nodeAcl.trustedPorts()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index df9e8efde93..583ccdda656 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; @@ -267,6 +268,35 @@ public class DynamicAllocationTest { } @Test + public void does_not_allocate_to_suspended_hosts() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + tester.makeReadyNodes(4, "host-small", NodeType.host, 32); + tester.activateTenantHosts(); + + HostName randomHost = new HostName(tester.nodeRepository().nodes().list(State.active).first().get().hostname()); + tester.orchestrator().suspend(randomHost); + + ApplicationId application1 = ProvisioningTester.applicationId(); + ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1"); + NodeResources flavor = new NodeResources(1, 4, 100, 1); + + try { + tester.prepare(application1, clusterSpec, 4, 1, flavor); + fail("Should not be able to deploy 4 nodes on 4 hosts because 1 is suspended"); + } catch (OutOfCapacityException ignored) { } + + // Resume the host, the deployment goes through + tester.orchestrator().resume(randomHost); + tester.activate(application1, tester.prepare(application1, clusterSpec, 4, 1, flavor)); + Set<String> hostnames = tester.getNodes(application1, State.active).hostnames(); + + // Verify that previously allocated nodes are not affected by host suspension + tester.orchestrator().suspend(randomHost); + tester.activate(application1, tester.prepare(application1, clusterSpec, 4, 1, flavor)); + assertEquals(hostnames, tester.getNodes(application1, State.active).hostnames()); + } + + @Test public void non_prod_zones_do_not_have_spares() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 390df571b04..31643b2ac79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -128,26 +128,6 @@ public class HostCapacityTest { } @Test - public void devhostCapacityTest() { - // Dev host can assign both configserver and tenant containers. - - var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); - var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), createIps(2, 10)), "devhost", nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost).build(); - - var cfg = Node.reserve(Set.of("::2"), "cfg", "devhost", resources0, NodeType.config).build(); - - var nodes = new ArrayList<>(List.of(cfg)); - var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertTrue(capacity.hasCapacity(devHost, resources0)); - - var container1 = Node.reserve(Set.of("::3"), "container1", "devhost", resources0, NodeType.tenant).build(); - nodes = new ArrayList<>(List.of(cfg, container1)); - capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertFalse(capacity.hasCapacity(devHost, resources0)); - - } - - @Test public void verifyCapacityFromAddresses() { Node nodeA = Node.reserve(Set.of("::2"), "nodeA", "host1", resources0, NodeType.tenant).build(); Node nodeB = Node.reserve(Set.of("::3"), "nodeB", "host1", resources0, NodeType.tenant).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 885efed9241..b1ee156f8e8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -12,24 +12,22 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.applicationmodel.InfrastructureApplication; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Test; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.SortedSet; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -47,6 +45,7 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); + private final NodeResources nodeResources = new NodeResources(1, 4, 10, 0.3); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); @@ -55,22 +54,24 @@ public class LoadBalancerProvisionerTest { public void provision_load_balancer() { Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().list(app1).asList(); Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().list(app2).asList(); - ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); + ClusterSpec.Id containerCluster = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); // Provision a load balancer for each application var nodes = prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster)); assertEquals(1, lbApp1.get().size()); assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbApp1.get().get(0).instance().get().reals().size()); tester.activate(app1, nodes); - tester.activate(app2, prepare(app2, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + tester.activate(app2, prepare(app2, clusterRequest(ClusterSpec.Type.container, containerCluster))); assertEquals(1, lbApp2.get().size()); + assertReals(app1, containerCluster, Node.State.active); + assertReals(app2, containerCluster, Node.State.active); // Reals are configured after activation assertEquals(app1, lbApp1.get().get(0).id().application()); - assertEquals(containerCluster1, lbApp1.get().get(0).id().cluster()); + assertEquals(containerCluster, lbApp1.get().get(0).id().cluster()); assertEquals(Collections.singleton(4443), lbApp1.get().get(0).instance().get().ports()); assertEquals("127.0.0.1", get(lbApp1.get().get(0).instance().get().reals(), 0).ipAddress()); assertEquals(4443, get(lbApp1.get().get(0).instance().get().reals(), 0).port()); @@ -84,7 +85,7 @@ public class LoadBalancerProvisionerTest { // Redeploying replaces failed node and removes it from load balancer tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().list(app1).asList().get(0); assertEquals(2, loadBalancer.instance().get().reals().size()); @@ -99,31 +100,15 @@ public class LoadBalancerProvisionerTest { // Add another container cluster to first app ClusterSpec.Id containerCluster2 = ClusterSpec.Id.from("qrs2"); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.container, containerCluster2), clusterRequest(ClusterSpec.Type.content, contentCluster))); // Load balancer is provisioned for second container cluster - assertEquals(2, lbApp1.get().size()); - List<HostName> activeContainers = tester.getNodes(app1, Node.State.active) - .container().asList() - .stream() - .map(Node::hostname) - .map(HostName::from) - .sorted() - .collect(Collectors.toList()); - List<HostName> reals = lbApp1.get().stream() - .map(LoadBalancer::instance) - .flatMap(Optional::stream) - .map(LoadBalancerInstance::reals) - .flatMap(Collection::stream) - .map(Real::hostname) - .sorted() - .collect(Collectors.toList()); - assertEquals(activeContainers, reals); + assertReals(app1, containerCluster2, Node.State.active); // Cluster removal deactivates relevant load balancer - tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1))); + tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster))); assertEquals(2, lbApp1.get().size()); assertEquals("Deactivated load balancer for cluster " + containerCluster2, LoadBalancer.State.inactive, lbApp1.get().stream() @@ -131,9 +116,9 @@ public class LoadBalancerProvisionerTest { .map(LoadBalancer::state) .findFirst() .get()); - assertEquals("Load balancer for cluster " + containerCluster1 + " remains active", LoadBalancer.State.active, + assertEquals("Load balancer for cluster " + containerCluster + " remains active", LoadBalancer.State.active, lbApp1.get().stream() - .filter(lb -> lb.id().cluster().equals(containerCluster1)) + .filter(lb -> lb.id().cluster().equals(containerCluster)) .map(LoadBalancer::state) .findFirst() .get()); @@ -148,11 +133,11 @@ public class LoadBalancerProvisionerTest { // Application is redeployed with one cluster and load balancer is re-activated tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); - assertSame("Re-activated load balancer for " + containerCluster1, LoadBalancer.State.active, + assertSame("Re-activated load balancer for " + containerCluster, LoadBalancer.State.active, lbApp1.get().stream() - .filter(lb -> lb.id().cluster().equals(containerCluster1)) + .filter(lb -> lb.id().cluster().equals(containerCluster)) .map(LoadBalancer::state) .findFirst() .orElseThrow()); @@ -160,14 +145,14 @@ public class LoadBalancerProvisionerTest { // Next redeploy does not create a new load balancer instance because reals are unchanged tester.loadBalancerService().throwOnCreate(true); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); // Routing is disabled through feature flag. Reals are removed on next deployment tester.loadBalancerService().throwOnCreate(false); flagSource.withBooleanFlag(PermanentFlags.DEACTIVATE_ROUTING.id(), true); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); List<LoadBalancer> activeLoadBalancers = lbApp1.get().stream() .filter(lb -> lb.state() == LoadBalancer.State.active) @@ -235,29 +220,12 @@ public class LoadBalancerProvisionerTest { @Test public void provision_load_balancer_config_server_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(InfrastructureApplication.CONFIG_SERVER.id()).asList(); - var cluster = ClusterSpec.Id.from("zone-config-servers"); - var nodes = prepare(InfrastructureApplication.CONFIG_SERVER.id(), Capacity.fromRequiredNodeType(NodeType.config), - clusterRequest(ClusterSpec.Type.admin, cluster)); - assertEquals(1, lbs.get().size()); - assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size()); - tester.activate(InfrastructureApplication.CONFIG_SERVER.id(), nodes); - assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); - assertEquals(cluster, lbs.get().get(0).id().cluster()); + provisionInfrastructureLoadBalancer(infraApp1, NodeType.config); } @Test public void provision_load_balancer_controller_cluster() { - ApplicationId controllerApp = ApplicationId.from("hosted-vespa", "controller", "default"); - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(controllerApp).asList(); - var cluster = ClusterSpec.Id.from("zone-config-servers"); - var nodes = prepare(controllerApp, Capacity.fromRequiredNodeType(NodeType.controller), - clusterRequest(ClusterSpec.Type.container, cluster)); - assertEquals(1, lbs.get().size()); - assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size()); - tester.activate(controllerApp, nodes); - assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); - assertEquals(cluster, lbs.get().get(0).id().cluster()); + provisionInfrastructureLoadBalancer(infraApp1, NodeType.controller); } @Test @@ -320,16 +288,69 @@ public class LoadBalancerProvisionerTest { assertTrue("No load balancer provisioned", tester.nodeRepository().loadBalancers().list(app1).asList().isEmpty()); } + @Test + public void load_balancer_targets_newly_active_nodes() { + ClusterSpec.Id container1 = ClusterSpec.Id.from("c1"); + // Initial deployment + { + Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, nodeResources)); + Set<HostSpec> preparedHosts = prepare(app1, capacity1, clusterRequest(ClusterSpec.Type.container, container1)); + tester.activate(app1, preparedHosts); + } + assertReals(app1, container1, Node.State.active); + + // Next deployment removes a node + { + Capacity capacity1 = Capacity.from(new ClusterResources(2, 1, nodeResources)); + Set<HostSpec> preparedHosts = prepare(app1, capacity1, clusterRequest(ClusterSpec.Type.container, container1)); + tester.activate(app1, preparedHosts); + } + assertReals(app1, container1, Node.State.active); + } + + private void assertReals(ApplicationId application, ClusterSpec.Id cluster, Node.State... states) { + List<LoadBalancer> loadBalancers = tester.nodeRepository().loadBalancers().list(application).cluster(cluster).asList(); + assertEquals(1, loadBalancers.size()); + List<String> reals = loadBalancers.get(0).instance().get().reals().stream() + .map(real -> real.hostname().value()) + .sorted() + .collect(Collectors.toList()); + List<String> activeNodes = tester.nodeRepository().nodes().list(states) + .owner(application) + .cluster(cluster) + .hostnames().stream() + .sorted() + .collect(Collectors.toList()); + assertEquals("Load balancer targets active nodes of " + application + " in " + cluster, + activeNodes, reals); + } + + private void provisionInfrastructureLoadBalancer(ApplicationId application, NodeType nodeType) { + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(application).asList(); + var cluster = ClusterSpec.Id.from("infra-cluster"); + ClusterSpec.Type clusterType = nodeType == NodeType.config ? ClusterSpec.Type.admin : ClusterSpec.Type.container; + var nodes = prepare(application, Capacity.fromRequiredNodeType(nodeType), clusterRequest(clusterType, cluster)); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 3, lbs.get().get(0).instance().get().reals().size()); + tester.activate(application, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(cluster, lbs.get().get(0).id().cluster()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().nodes().deallocate(tester.nodeRepository().nodes().list().owner(application).asList(), Agent.system, this.getClass().getSimpleName()); } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { - return prepare(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), specs); + return prepare(application, Capacity.from(new ClusterResources(2, 1, nodeResources), false, true), specs); } private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, ClusterSpec... specs) { - tester.makeReadyNodes(specs.length * 2, new NodeResources(1, 4, 10, 0.3), capacity.type()); + int nodeCount = capacity.minResources().nodes(); + if (capacity.type().isConfigServerLike()) { + nodeCount = 3; + } + tester.makeReadyNodes(specs.length * nodeCount, nodeResources, capacity.type()); Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { allNodes.addAll(tester.prepare(application, spec, capacity)); @@ -354,6 +375,9 @@ public class LoadBalancerProvisionerTest { } private static <T> T get(Set<T> set, int position) { + if (!(set instanceof SortedSet)) { + throw new IllegalArgumentException(set + " is not a sorted set"); + } return Iterators.get(set.iterator(), position, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 482b798d736..3db8a71e4a7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -237,7 +237,6 @@ public class MultigroupProvisioningTest { new MockDeployer.ApplicationContext(application1, cluster(), Capacity.from(new ClusterResources(8, 1, large), false, true)))); new RetiredExpirer(tester.nodeRepository(), - tester.orchestrator(), deployer, new TestMetric(), Duration.ofDays(30), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 4a9707f52f8..12f1abf0cf5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -98,7 +98,6 @@ public class NodeTypeProvisioningTest { clusterSpec, capacity))); RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), - tester.orchestrator(), deployer, new TestMetric(), Duration.ofDays(30), @@ -166,7 +165,6 @@ public class NodeTypeProvisioningTest { Collections.singletonMap(application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity))); RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), - tester.orchestrator(), deployer, new TestMetric(), Duration.ofDays(30), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 95f25612dd7..c9b384b95a6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -871,19 +871,6 @@ public class ProvisioningTest { } @Test - public void devsystem_application_deployment_on_devhost() { - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.dev, Environment.dev, RegionName.from("no-central"))).build(); - - tester.makeReadyNodes(4, defaultResources, NodeType.devhost, 1); - tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.devhost); - - ApplicationId application = ProvisioningTester.applicationId(); - SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); - assertEquals(4, state.allHosts.size()); - tester.activate(application, state.allHosts); - } - - @Test public void allocates_reserved_nodes_for_type_spec_deployment() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); Function<InfraApplication, Collection<HostSpec>> prepareAndActivate = app -> tester.activate(app.getApplicationId(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c478840780f..b781f397f70 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -43,6 +43,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.duper.ConfigServerApplication; @@ -80,7 +81,6 @@ public class ProvisioningTester { private final NodeFlavors nodeFlavors; private final ManualClock clock; private final NodeRepository nodeRepository; - private final Orchestrator orchestrator; private final NodeRepositoryProvisioner provisioner; private final CapacityPolicies capacityPolicies; private final ProvisionLogger provisionLogger; @@ -114,10 +114,10 @@ public class ProvisioningTester { Optional.empty(), flagSource, new MemoryMetricsDb(clock), + orchestrator, true, spareCount, 1000); - this.orchestrator = orchestrator; this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, @@ -144,7 +144,7 @@ public class ProvisioningTester { public void advanceTime(TemporalAmount duration) { clock.advance(duration); } public NodeRepository nodeRepository() { return nodeRepository; } - public Orchestrator orchestrator() { return orchestrator; } + public Orchestrator orchestrator() { return nodeRepository.orchestrator(); } public ManualClock clock() { return clock; } public NodeRepositoryProvisioner provisioner() { return provisioner; } public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } @@ -689,20 +689,13 @@ public class ProvisioningTester { } public ProvisioningTester build() { - Orchestrator orchestrator = Optional.ofNullable(this.orchestrator) - .orElseGet(() -> { - Orchestrator orch = mock(Orchestrator.class); - doThrow(new RuntimeException()).when(orch).acquirePermissionToRemove(any()); - return orch; - }); - return new ProvisioningTester(Optional.ofNullable(curator).orElseGet(MockCurator::new), new NodeFlavors(Optional.ofNullable(flavorsConfig).orElseGet(ProvisioningTester::createConfig)), resourcesCalculator, Optional.ofNullable(zone).orElseGet(Zone::defaultZone), Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()), defaultImage, - orchestrator, + Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new), hostProvisioner, new LoadBalancerServiceMock(), Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index a7a6a06ae0f..50ca2b1ca41 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json @@ -1,7 +1,7 @@ { "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com", "id": "node55", - "state": "parked", + "state": "dirty", "type": "tenant", "hostname": "host55.yahoo.com", "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", @@ -22,7 +22,7 @@ "agent": "system" }, { - "event": "parked", + "event": "deallocated", "at": 123, "agent": "system" } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json index 03df6c8e1dc..2b650bad39b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json @@ -17,8 +17,8 @@ @include(node2.json), @include(docker-node1.json), @include(node1.json), - @include(node5.json), @include(node55.json), + @include(node5.json), @include(dockerhost6.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json index 8835945dc92..55e216f454a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json @@ -17,7 +17,7 @@ @include(node2.json), @include(docker-node1.json), @include(node1.json), - @include(node5.json), - @include(node55.json) + @include(node55.json), + @include(node5.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json index db4d0bb1682..54ff2bc232f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json @@ -52,10 +52,10 @@ "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json index 68ff9fedc00..27767be6315 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json @@ -45,6 +45,7 @@ "dirty": { "url": "http://localhost:8080/nodes/v2/state/dirty", "nodes": [ + @include(node55.json) ] }, "failed": { @@ -56,7 +57,6 @@ "parked": { "url": "http://localhost:8080/nodes/v2/state/parked", "nodes": [ - @include(node55.json) ] }, "deprovisioned": { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 2a9cdb7e0eb..ddaec86b340 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; @@ -13,7 +13,6 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java index 20b27ea7632..fd62f2b4b70 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.orchestrator.OrchestratorContext; import java.io.IOException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 7c8be703310..f4929a4f09e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java index 141687bd269..cdae68a0d06 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.jaxrs.client.JaxRsTimeouts; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index fc790089517..d393117d57c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.model; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import java.util.logging.Level; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; @@ -22,6 +21,7 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.logging.Level; import java.util.logging.Logger; public class StorageNodeImpl implements StorageNode { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java index 012018f3889..2ebe377c238 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.http.HttpResponse.Status; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -31,7 +31,7 @@ public class ApplicationSuspensionRequestHandler extends RestApiRequestHandler<A private final Orchestrator orchestrator; @Inject - public ApplicationSuspensionRequestHandler(LoggingRequestHandler.Context context, Orchestrator orchestrator) { + public ApplicationSuspensionRequestHandler(ThreadedHttpRequestHandler.Context context, Orchestrator orchestrator) { super(context, ApplicationSuspensionRequestHandler::createRestApiDefinition); this.orchestrator = orchestrator; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java index bdfd2c16df2..09311e6f3e0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; @@ -27,7 +27,7 @@ public class HealthRequestHandler extends RestApiRequestHandler<HealthRequestHan private final HealthMonitorApi healthMonitorApi; @Inject - public HealthRequestHandler(LoggingRequestHandler.Context context, + public HealthRequestHandler(ThreadedHttpRequestHandler.Context context, HealthMonitorApi healthMonitorApi) { super(context, HealthRequestHandler::createRestApiDefinition); this.healthMonitorApi = healthMonitorApi; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java index c56866fdad6..f90258d97d7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Response; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.RestApi; @@ -42,7 +42,7 @@ public class HostRequestHandler extends RestApiRequestHandler<HostRequestHandler private final Orchestrator orchestrator; @Inject - public HostRequestHandler(LoggingRequestHandler.Context context, Orchestrator orchestrator) { + public HostRequestHandler(ThreadedHttpRequestHandler.Context context, Orchestrator orchestrator) { super(context, HostRequestHandler::createRestApiDefinition); this.orchestrator = orchestrator; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java index 33391e42a93..eff4ab3364b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Response; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.RestApi; @@ -32,7 +32,7 @@ public class HostSuspensionRequestHandler extends RestApiRequestHandler<HostSusp private final Orchestrator orchestrator; @Inject - public HostSuspensionRequestHandler(LoggingRequestHandler.Context context, Orchestrator orchestrator) { + public HostSuspensionRequestHandler(ThreadedHttpRequestHandler.Context context, Orchestrator orchestrator) { super(context, HostSuspensionRequestHandler::createRestApiDefinition); this.orchestrator = orchestrator; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java index 9dd69773595..df836bf7933 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -56,7 +56,7 @@ public class InstanceRequestHandler extends RestApiRequestHandler<InstanceReques private final ServiceMonitor serviceMonitor; @Inject - public InstanceRequestHandler(LoggingRequestHandler.Context context, + public InstanceRequestHandler(ThreadedHttpRequestHandler.Context context, ServiceMonitor serviceMonitor, StatusService statusService, SlobrokApi slobrokApi, diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index 00ab1a964c8..ce4dd5ad4ef 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index a29c186d30c..ce68dca0cc5 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 698c7b544b3..a88e6766050 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -61,7 +61,7 @@ public class OrchestratorTest { var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1")); this.superModelManager = new MySuperModelProvider(); var duperModel = new DuperModel(); - this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel, flagSource, zone.system()); + this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel); this.monitorManager = mock(UnionMonitorManager.class); var metric = mock(Metric.class); var serviceMonitor = new ServiceMonitorImpl(duperModelManager, monitorManager, metric, timer, zone); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java index 970a8682c5a..6f22ff74ad8 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.test.ManualClock; import com.yahoo.time.TimeBudget; import org.junit.Before; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java index ea7566f9d51..03f2c0b5e3b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Metric; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java index 089a2dc8709..9641e045431 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.RestApiTestDriver; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java index 07c8662f656..20bfb09a53d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/parent/pom.xml b/parent/pom.xml index 8afebd281c9..cac01fc6d5e 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -172,6 +172,30 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>${maven-shade-plugin.version}</version> + <configuration> + <filters> + <filter> + <!-- Filter out files that are known to cause issues and warnings. --> + <!-- WARNING: Adding a custom filter in a module's pom will disable this filter! --> + <artifact>*:*</artifact> + <excludes> + <!-- Silence warning about "breaking its strong encapsulation" --> + <exclude>module-info.class</exclude> + <!-- Multi-release jars --> + <exclude>META-INF/versions/*/module-info.class</exclude> + <!-- License and notices files --> + <exclude>META-INF/LICENSE*</exclude> + <exclude>META-INF/NOTICE*</exclude> + <!-- E.g. bundle manifests --> + <exclude>META-INF/MANIFEST.MF</exclude> + <!-- Signature files, from e.g. bouncycastle --> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -474,7 +498,7 @@ <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> - <version>57.2</version> + <version>70.1</version> </dependency> <dependency> <groupId>com.infradna.tool</groupId> @@ -856,7 +880,7 @@ <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> - <version>2.12.1</version> + <version>2.12.2</version> </dependency> </dependencies> </dependencyManagement> @@ -111,6 +111,7 @@ <module>predicate-search</module> <module>predicate-search-core</module> <module>provided-dependencies</module> + <module>routing-generator</module> <module>searchcore</module> <module>searchlib</module> <module>searchsummary</module> diff --git a/routing-generator/CMakeLists.txt b/routing-generator/CMakeLists.txt new file mode 100644 index 00000000000..f8d34754c41 --- /dev/null +++ b/routing-generator/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definitions() diff --git a/routing-generator/OWNERS b/routing-generator/OWNERS new file mode 100644 index 00000000000..e31fe78605e --- /dev/null +++ b/routing-generator/OWNERS @@ -0,0 +1,2 @@ +mpolden +tokle diff --git a/routing-generator/pom.xml b/routing-generator/pom.xml new file mode 100644 index 00000000000..3a197b94012 --- /dev/null +++ b/routing-generator/pom.xml @@ -0,0 +1,96 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>routing-generator</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + + <dependencies> + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.jimfs</groupId> + <artifactId>jimfs</artifactId> + <version>1.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-athenz</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile --> + <dependency> + <!-- compile because this is only provided when using standalone container --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/Router.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/Router.java new file mode 100644 index 00000000000..c7cd5a75359 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/Router.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +/** + * A {@link Router} (e.g. a reverse proxy) consumes a {@link RoutingTable} by + * translating it to the router's own format and loading it. + * + * @author mpolden + */ +public interface Router { + + /** Load the given routing table */ + void load(RoutingTable table); + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java new file mode 100644 index 00000000000..a1d84873379 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java @@ -0,0 +1,166 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.jdisc.Metric; +import com.yahoo.routing.config.ZoneConfig; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.routing.nginx.Nginx; +import com.yahoo.vespa.hosted.routing.status.RoutingStatus; +import com.yahoo.yolean.Exceptions; +import com.yahoo.yolean.concurrent.Sleeper; + +import java.nio.file.FileSystems; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * The routing generator generates a routing table for a hosted Vespa zone. + * + * Config is retrieved by subscribing to {@link LbServicesConfig} for all deployments. This is then translated to a + * {@link RoutingTable}, which is loaded into a {@link Router}. + * + * @author oyving + * @author mpolden + */ +public class RoutingGenerator extends AbstractComponent { + + private static final Logger log = Logger.getLogger(RoutingGenerator.class.getName()); + private static final Duration configTimeout = Duration.ofSeconds(10); + private static final Duration shutdownTimeout = Duration.ofSeconds(10); + private static final Duration refreshInterval = Duration.ofSeconds(30); + + private final Router router; + private final Clock clock; + @SuppressWarnings("removal") // TODO Vespa 8: remove + private final ConfigSubscriber configSubscriber; + + private final ExecutorService executor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("routing-generator-config-subscriber")); + private final ScheduledExecutorService scheduledExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("routing-generator-maintenance")); + private final Object monitor = new Object(); + + private volatile RoutingTable routingTable = null; + + @Inject + @SuppressWarnings("removal") // TODO Vespa 8: remove + public RoutingGenerator(ZoneConfig zoneConfig, RoutingStatus routingStatus, Metric metric) { + this(new ConfigSourceSet(zoneConfig.configserver()), new Nginx(FileSystems.getDefault(), + new ProcessExecuter(), + Sleeper.DEFAULT, + Clock.systemUTC(), + routingStatus, + metric), + Clock.systemUTC()); + } + + @SuppressWarnings("removal") // TODO Vespa 8: remove + RoutingGenerator(ConfigSource configSource, Router router, Clock clock) { + this.router = Objects.requireNonNull(router); + this.clock = Objects.requireNonNull(clock); + this.configSubscriber = new ConfigSubscriber(configSource); + executor.execute(() -> subscribeOn(LbServicesConfig.class, this::load, configSource, executor)); + // Reload configuration periodically. The router depend on state from other sources than config, such as RoutingStatus + scheduledExecutor.scheduleAtFixedRate(this::reload, refreshInterval.toMillis(), refreshInterval.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Get the currently active routing table, if any */ + public Optional<RoutingTable> routingTable() { + synchronized (monitor) { + return Optional.ofNullable(routingTable); + } + } + + /** Reload the current routing table, if any */ + private void reload() { + synchronized (monitor) { + routingTable().ifPresent(this::load); + } + } + + /** Load the given routing table */ + private void load(RoutingTable newTable) { + synchronized (monitor) { + router.load(newTable); + routingTable = newTable; + } + } + + private void load(LbServicesConfig lbServicesConfig, long generation) { + load(RoutingTable.from(lbServicesConfig, generation)); + } + + @SuppressWarnings("removal") // TODO Vespa 8: remove + private <T extends ConfigInstance> void subscribeOn(Class<T> clazz, BiConsumer<T, Long> action, ConfigSource configSource, + ExecutorService executor) { + ConfigHandle<T> configHandle = null; + String configId = "*"; + while (!executor.isShutdown()) { + try { + boolean initializing = true; + log.log(Level.INFO, "Subscribing to configuration " + clazz + "@" + configId + " from " + configSource); + if (configHandle == null) { + configHandle = configSubscriber.subscribe(clazz, configId); + } + while (!executor.isShutdown() && !configSubscriber.isClosed()) { + Instant subscribingAt = clock.instant(); + if (configSubscriber.nextGeneration(configTimeout.toMillis(), initializing) && configHandle.isChanged()) { + log.log(Level.INFO, "Received new configuration: " + configHandle); + T configuration = configHandle.getConfig(); + log.log(Level.FINE, "Received new configuration: " + configuration); + action.accept(configuration, configSubscriber.getGeneration()); + initializing = false; + } else { + log.log(Level.FINE, "Configuration tick with no change: " + configHandle + + ", getting config took " + Duration.between(subscribingAt, clock.instant()) + + ", timeout is " + configTimeout); + } + } + } catch (Exception e) { + log.log(Level.WARNING, "Exception while subscribing to configuration: " + clazz + "@" + configId + + " from " + configSource + ": " + Exceptions.toMessageString(e)); + } + } + } + + @Override + @SuppressWarnings("removal") // TODO Vespa 8: remove + public void deconstruct() { + configSubscriber.close(); + // shutdownNow because ConfigSubscriber#nextGeneration blocks until next config, and we don't want to wait for + // that when shutting down + executor.shutdownNow(); + scheduledExecutor.shutdown(); + awaitTermination("executor", executor); + awaitTermination("scheduledExecutor", scheduledExecutor); + } + + private static void awaitTermination(String name, ExecutorService executorService) { + try { + if (!executorService.awaitTermination(shutdownTimeout.toMillis(), TimeUnit.MILLISECONDS)) { + throw new RuntimeException("Failed to shut down " + name + " within " + shutdownTimeout); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java new file mode 100644 index 00000000000..90a38da8687 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java @@ -0,0 +1,399 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +import com.google.common.hash.Hashing; +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A routing table for a hosted Vespa zone. This holds the details necessary for the routing layer to route traffic to + * deployments. + * + * This is immutable. + * + * @author mpolden + */ +public class RoutingTable { + + private static final String HOSTED_VESPA_TENANT_NAME = "hosted-vespa"; + + private final SortedMap<Endpoint, Target> table; + private final long generation; + + public RoutingTable(Map<Endpoint, Target> table, long generation) { + this.table = Collections.unmodifiableSortedMap(new TreeMap<>(Objects.requireNonNull(table))); + this.generation = generation; + } + + public SortedMap<Endpoint, Target> asMap() { + return table; + } + + /** Returns the target for given dnsName, if any */ + public Optional<Target> targetOf(String dnsName, RoutingMethod routingMethod) { + return Optional.ofNullable(table.get(new Endpoint(dnsName, routingMethod))); + } + + /** Returns a copy of this containing only endpoints using given routing method */ + public RoutingTable routingMethod(RoutingMethod method) { + Map<Endpoint, Target> copy = new TreeMap<>(table); + copy.keySet().removeIf(endpoint -> !endpoint.routingMethod().equals(method)); + return new RoutingTable(copy, generation); + } + + /** Returns the Vespa config generation this is based on */ + public long generation() { + return generation; + } + + @Override + public String toString() { + return table.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingTable that = (RoutingTable) o; + return generation == that.generation && table.equals(that.table); + } + + @Override + public int hashCode() { + return Objects.hash(table, generation); + } + + public static RoutingTable from(LbServicesConfig config, long generation) { + Map<Endpoint, Target> entries = new TreeMap<>(); + for (var tenants : config.tenants().entrySet()) { + TenantName tenantName = TenantName.from(tenants.getKey()); + if (tenantName.value().equals(HOSTED_VESPA_TENANT_NAME)) continue; + for (var applications : tenants.getValue().applications().entrySet()) { + String[] parts = applications.getKey().split(":"); + if (parts.length != 4) throw new IllegalArgumentException("Invalid deployment ID '" + applications.getKey() + "'"); + + ApplicationName application = ApplicationName.from(parts[0]); + ZoneId zone = ZoneId.from(parts[1], parts[2]); + InstanceName instance = InstanceName.from(parts[3]); + + for (var configuredEndpoint : applications.getValue().endpoints()) { + List<Real> reals = configuredEndpoint.hosts().stream() + .map(hostname -> new Real(hostname, + 4443, + configuredEndpoint.weight(), + applications.getValue().activeRotation())) + .collect(Collectors.toList()); + Endpoint endpoint = new Endpoint(configuredEndpoint.dnsName(), routingMethodFrom(configuredEndpoint)); + ClusterSpec.Id cluster = ClusterSpec.Id.from(configuredEndpoint.clusterId()); + Target target; + boolean applicationEndpoint = configuredEndpoint.scope() == LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.application; + if (applicationEndpoint) { + target = Target.create(endpoint.dnsName, tenantName, application, cluster, zone, reals); + } else { + target = Target.create(ApplicationId.from(tenantName, application, instance), cluster, zone, reals); + } + entries.merge(endpoint, target, (oldValue, value) -> { + if (applicationEndpoint) { + List<Real> merged = new ArrayList<>(oldValue.reals()); + merged.addAll(value.reals()); + return value.withReals(merged); + } + return oldValue; + }); + } + } + } + return new RoutingTable(entries, generation); + } + + private static RoutingMethod routingMethodFrom(LbServicesConfig.Tenants.Applications.Endpoints endpoint) { + switch (endpoint.routingMethod()) { + case shared: return RoutingMethod.shared; + case sharedLayer4: return RoutingMethod.sharedLayer4; + } + throw new IllegalArgumentException("Unhandled routing method: " + endpoint.routingMethod()); + } + + /** The target of an {@link Endpoint} */ + public static class Target implements Comparable<Target> { + + private final String id; + + private final TenantName tenant; + private final ApplicationName application; + private final Optional<InstanceName> instance; + private final ZoneId zone; + private final ClusterSpec.Id cluster; + private final List<Real> reals; + + private Target(String id, TenantName tenant, ApplicationName application, Optional<InstanceName> instance, + ClusterSpec.Id cluster, ZoneId zone, List<Real> reals) { + this.id = Objects.requireNonNull(id); + this.tenant = Objects.requireNonNull(tenant); + this.application = Objects.requireNonNull(application); + this.instance = Objects.requireNonNull(instance); + this.zone = Objects.requireNonNull(zone); + this.cluster = Objects.requireNonNull(cluster); + this.reals = Objects.requireNonNull(reals).stream().sorted().collect(Collectors.toUnmodifiableList()); + for (int i = 0; i < reals.size(); i++) { + for (int j = 0; j < i; j++) { + if (reals.get(i).equals(reals.get(j))) { + throw new IllegalArgumentException("Found duplicate real server: " + reals.get(i)); + } + } + } + } + + /** An unique identifier of this target (previously known as "upstreamName") */ + public String id() { + return id; + } + + /** Returns whether this is an application-level target, which points to reals of multiple instances */ + public boolean applicationLevel() { + return instance.isEmpty(); + } + + public TenantName tenant() { + return tenant; + } + + public ApplicationName application() { + return application; + } + + public Optional<InstanceName> instance() { + return instance; + } + + public ZoneId zone() { + return zone; + } + + public ClusterSpec.Id cluster() { + return cluster; + } + + /** The real servers this points to */ + public List<Real> reals() { + return reals; + } + + /** Returns whether this is active and should receive traffic either through a global or application endpoint */ + public boolean active() { + return reals.stream().anyMatch(Real::active); + } + + /** Returns a copy of this containing given reals */ + public Target withReals(List<Real> reals) { + return new Target(id, tenant, application, instance, cluster, zone, reals); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Target target = (Target) o; + return id.equals(target.id) && tenant.equals(target.tenant) && application.equals(target.application) && instance.equals(target.instance) && zone.equals(target.zone) && cluster.equals(target.cluster) && reals.equals(target.reals); + } + + @Override + public int hashCode() { + return Objects.hash(id, tenant, application, instance, zone, cluster, reals); + } + + @Override + public String toString() { + return "target " + id + " -> " + + "tenant=" + tenant + + ",application=" + application + + ",instance=" + instance + + ",zone=" + zone + + ",cluster=" + cluster + + ",reals=" + reals; + } + + @Override + public int compareTo(RoutingTable.Target other) { + return id.compareTo(other.id); + } + + /** Create an instance-level tartget */ + public static Target create(ApplicationId instance, ClusterSpec.Id cluster, ZoneId zone, List<Real> reals) { + return new Target(createId("", instance.tenant(), instance.application(), Optional.of(instance.instance()), cluster, zone), + instance.tenant(), instance.application(), Optional.of(instance.instance()), cluster, zone, reals); + } + + /** Create an application-level target */ + public static Target create(String dnsName, TenantName tenant, ApplicationName application, ClusterSpec.Id cluster, ZoneId zone, List<Real> reals) { + return new Target(createId(Objects.requireNonNull(dnsName), tenant, application, Optional.empty(), cluster, zone), + tenant, application, Optional.empty(), cluster, zone, reals); + } + + /** Create an unique identifier for given dnsName and target */ + private static String createId(String dnsName, TenantName tenant, ApplicationName application, + Optional<InstanceName> instance, ClusterSpec.Id cluster, ZoneId zone) { + if (instance.isEmpty()) { // Application-scoped endpoint + if (dnsName.isEmpty()) throw new IllegalArgumentException("dnsName must given for application-scoped endpoint"); + String endpointHash = Hashing.sha1().hashString(dnsName, StandardCharsets.UTF_8).toString(); + return "application-" + endpointHash + "." +application.value() + "." + tenant.value(); + } else { + if (!dnsName.isEmpty()) throw new IllegalArgumentException("dnsName must not be given for instance-level endpoint"); + } + return Stream.of(nullIfDefault(cluster.value()), + nullIfDefault(instance.get().value()), + application.value(), + tenant.value(), + zone.region().value(), + zone.environment().value()) + .filter(Objects::nonNull) + .map(Target::sanitize) + .collect(Collectors.joining(".")); + } + + private static String nullIfDefault(String value) { // Sublime sadness + return "default".equals(value) ? null : value; + } + + private static String sanitize(String id) { + return id.toLowerCase() + .replace('_', '-') + .replaceAll("[^a-z0-9-]*", ""); + } + + } + + /** An externally visible endpoint */ + public static class Endpoint implements Comparable<Endpoint> { + + private static final Comparator<Endpoint> COMPARATOR = Comparator.comparing(Endpoint::dnsName) + .thenComparing(Endpoint::routingMethod); + + private final String dnsName; + private final RoutingMethod routingMethod; + + public Endpoint(String dnsName, RoutingMethod routingMethod) { + this.dnsName = Objects.requireNonNull(dnsName); + this.routingMethod = Objects.requireNonNull(routingMethod); + } + + /** The DNS name of this endpoint. This does not contain a trailing dot */ + public String dnsName() { + return dnsName; + } + + public RoutingMethod routingMethod() { + return routingMethod; + } + + @Override + public String toString() { + return "endpoint " + dnsName + " (routing method: " + routingMethod + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Endpoint endpoint = (Endpoint) o; + return dnsName.equals(endpoint.dnsName) && routingMethod == endpoint.routingMethod; + } + + @Override + public int hashCode() { + return Objects.hash(dnsName, routingMethod); + } + + @Override + public int compareTo(Endpoint o) { + return COMPARATOR.compare(this, o); + } + + } + + /** A real server, i.e. a node in a Vespa cluster */ + public static class Real implements Comparable<Real> { + + private static final Comparator<Real> COMPARATOR = Comparator.comparing(Real::hostname) + .thenComparing(Real::port) + .thenComparing(Real::weight) + .thenComparing(Real::active); + + private final String hostname; + private final int port; + private final int weight; + private final boolean active; + + public Real(String hostname, int port, int weight, boolean active) { + this.hostname = Objects.requireNonNull(hostname); + this.port = port; + this.weight = weight; + this.active = active; + } + + /** The hostname of this */ + public String hostname() { + return hostname; + } + + /** The port this is listening on */ + public int port() { + return port; + } + + /** The relative weight of this. Controls the amount of traffic this should receive */ + public int weight() { + return weight; + } + + /** Returns whether this is active and should receive traffic */ + public boolean active() { + return active; + } + + @Override + public String toString() { + return "real server " + hostname + "[port=" + port + ",weight=" + weight + ",active=" + active + "]"; + } + + @Override + public int compareTo(Real other) { + return COMPARATOR.compare(this, other); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Real real = (Real) o; + return port == real.port && hostname.equals(real.hostname); + } + + @Override + public int hashCode() { + return Objects.hash(hostname, port); + } + + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/Nginx.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/Nginx.java new file mode 100644 index 00000000000..e75717d8ac1 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/Nginx.java @@ -0,0 +1,190 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.yahoo.collections.Pair; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.jdisc.Metric; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.routing.Router; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.status.RoutingStatus; +import com.yahoo.yolean.Exceptions; +import com.yahoo.yolean.concurrent.Sleeper; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Objects; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This loads a {@link RoutingTable} into a running Nginx process. + * + * @author mpolden + */ +public class Nginx implements Router { + + private static final Logger LOG = Logger.getLogger(Nginx.class.getName()); + private static final int EXEC_ATTEMPTS = 5; + + static final String GENERATED_UPSTREAMS_METRIC = "upstreams_generated"; + static final String CONFIG_RELOADS_METRIC = "upstreams_nginx_reloads"; + static final String OK_CONFIG_RELOADS_METRIC = "upstreams_nginx_reloads_succeeded"; + + private final FileSystem fileSystem; + private final ProcessExecuter processExecuter; + private final Sleeper sleeper; + private final Clock clock; + private final RoutingStatus routingStatus; + private final Metric metric; + + private final Object monitor = new Object(); + + public Nginx(FileSystem fileSystem, ProcessExecuter processExecuter, Sleeper sleeper, Clock clock, RoutingStatus routingStatus, Metric metric) { + this.fileSystem = Objects.requireNonNull(fileSystem); + this.processExecuter = Objects.requireNonNull(processExecuter); + this.sleeper = Objects.requireNonNull(sleeper); + this.clock = Objects.requireNonNull(clock); + this.routingStatus = Objects.requireNonNull(routingStatus); + this.metric = Objects.requireNonNull(metric); + } + + @Override + public void load(RoutingTable table) { + synchronized (monitor) { + try { + table = table.routingMethod(RoutingMethod.sharedLayer4); // This router only supports layer 4 endpoints + testConfig(table); + loadConfig(table.asMap().size()); + gcConfig(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + /** Write given routing table to a temporary config file and test it */ + private void testConfig(RoutingTable table) throws IOException { + String config = NginxConfig.from(table, routingStatus); + Files.createDirectories(NginxPath.root.in(fileSystem)); + atomicWriteString(NginxPath.temporaryConfig.in(fileSystem), config); + + // This retries config testing because it can fail due to external factors, such as hostnames not resolving in + // DNS. Retrying can be removed if we switch to having only IP addresses in config + retryingExec("/usr/bin/sudo /opt/vespa/bin/vespa-verify-nginx"); + } + + /** Load tested config into Nginx */ + private void loadConfig(int upstreamCount) throws IOException { + Path configPath = NginxPath.config.in(fileSystem); + Path tempConfigPath = NginxPath.temporaryConfig.in(fileSystem); + try { + String currentConfig = Files.readString(configPath); + String newConfig = Files.readString(tempConfigPath); + if (currentConfig.equals(newConfig)) { + Files.deleteIfExists(tempConfigPath); + return; + } + Path rotatedConfig = NginxPath.config.rotatedIn(fileSystem, clock.instant()); + atomicCopy(configPath, rotatedConfig); + } catch (NoSuchFileException ignored) { + // Fine, not enough files exist to compare or rotate + } + Files.move(tempConfigPath, configPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + metric.add(CONFIG_RELOADS_METRIC, 1, null); + // Retry reload. Same rationale for retrying as in testConfig() + LOG.info("Loading new configuration file from " + configPath); + retryingExec("/usr/bin/sudo /opt/vespa/bin/vespa-reload-nginx"); + metric.add(OK_CONFIG_RELOADS_METRIC, 1, null); + metric.set(GENERATED_UPSTREAMS_METRIC, upstreamCount, null); + } + + /** Remove old config files */ + private void gcConfig() throws IOException { + Instant oneWeekAgo = clock.instant().minus(Duration.ofDays(7)); + // Rotated files have the format <basename>-yyyy-MM-dd-HH:mm:ss.SSS + String configBasename = NginxPath.config.in(fileSystem).getFileName().toString(); + Files.list(NginxPath.root.in(fileSystem)) + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().startsWith(configBasename) || + // TODO(mpolden): This cleans up old layer 7 files. Remove after 2022-03-15 + path.getFileName().toString().startsWith("nginx.conf-")) + .filter(path -> rotatedAt(path).map(instant -> instant.isBefore(oneWeekAgo)) + .orElse(false)) + .forEach(path -> Exceptions.uncheck(() -> Files.deleteIfExists(path))); + } + + /** Returns the time given path was rotated */ + private Optional<Instant> rotatedAt(Path path) { + String[] parts = path.getFileName().toString().split("-", 2); + if (parts.length != 2) return Optional.empty(); + return Optional.of(LocalDateTime.from(NginxPath.ROTATED_SUFFIX_FORMAT.parse(parts[1])).toInstant(ZoneOffset.UTC)); + } + + /** Run given command. Retries after a delay on failure */ + private void retryingExec(String command) { + boolean success = false; + for (int attempt = 1; attempt <= EXEC_ATTEMPTS; attempt++) { + String errorMessage; + try { + Pair<Integer, String> result = processExecuter.exec(command); + if (result.getFirst() == 0) { + success = true; + break; + } + errorMessage = result.getSecond(); + } catch (IOException e) { + errorMessage = Exceptions.toMessageString(e); + } + Duration duration = Duration.ofSeconds((long) Math.pow(2, attempt)); + LOG.log(Level.WARNING, "Failed to run " + command + " on attempt " + attempt + ": " + errorMessage + + ". Retrying in " + duration); + sleeper.sleep(duration); + } + if (!success) { + throw new RuntimeException("Failed to run " + command + " successfully after " + EXEC_ATTEMPTS + + " attempts, giving up"); + } + } + + /** Apply pathOperation to a temporary file, then atomically move the temporary file to path */ + private void atomicWrite(Path path, PathOperation pathOperation) throws IOException { + Path tempFile = null; + try { + tempFile = Files.createTempFile(path.getParent(), "nginx", ""); + pathOperation.run(tempFile); + Files.move(tempFile, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } finally { + if (tempFile != null) { + Files.deleteIfExists(tempFile); + } + } + } + + private void atomicCopy(Path src, Path dst) throws IOException { + atomicWrite(dst, (tempFile) -> Files.copy(src, tempFile, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES)); + } + + private void atomicWriteString(Path path, String content) throws IOException { + atomicWrite(path, (tempFile) -> Files.writeString(tempFile, content)); + } + + @FunctionalInterface + private interface PathOperation { + void run(Path path) throws IOException; + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxConfig.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxConfig.java new file mode 100644 index 00000000000..ffaa2b0bb60 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxConfig.java @@ -0,0 +1,116 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.RoutingTable.Real; +import com.yahoo.vespa.hosted.routing.status.RoutingStatus; + +import java.util.HashMap; +import java.util.Map; + +/** + * Converts a {@link RoutingTable} to Nginx's own config format. + * + * @author mpolden + */ +class NginxConfig { + + private NginxConfig() { + } + + public static String from(RoutingTable routingTable, RoutingStatus routingStatus) { + StringBuilder sb = new StringBuilder(); + + // Map SNI header to upstream + sb.append("map $ssl_preread_server_name $name {\n"); + routingTable.asMap().forEach((endpoint, target) -> { + sb.append(" ").append(endpoint.dnsName()).append(" ").append(target.id()).append(";\n"); + }); + + // Forward requests without SNI header directly to Nginx (e.g. VIP health checks) + sb.append(" '' default;\n"); + sb.append("}\n\n"); + + // Render routing table targets as upstreams + renderUpstreamsTo(sb, routingTable, routingStatus); + + // Configure the default upstream, which targets Nginx itself + sb.append("upstream default {\n"); + sb.append(" server localhost:4445;\n"); + sb.append(" ").append(checkDirective(4080)).append("\n"); + sb.append(" ").append(checkHttpSendDirective("localhost")).append("\n"); + sb.append("}\n\n"); + + // Listener port + sb.append("server {\n"); + sb.append(" listen 443 reuseport;\n"); + sb.append(" listen [::]:443 reuseport;\n"); + sb.append(" proxy_pass $name;\n"); + sb.append(" ssl_preread on;\n"); + sb.append(" proxy_protocol on;\n"); + sb.append("}\n"); + + return sb.toString(); + } + + private static String checkDirective(int port) { + // nginx_http_upstream_check_module does not support health checks over https + // a different http port is used instead, which acts as a http->https proxy for /status.html requests + return String.format("check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=%d;", + port); + } + + private static String checkHttpSendDirective(String upstreamName) { + return "check_http_send \"" + + "GET /status.html HTTP/1.0\\r\\n" + + "Host: " + upstreamName + "\\r\\n" + + "\\r\\n\";"; + } + + private static void renderUpstreamsTo(StringBuilder sb, RoutingTable routingTable, RoutingStatus routingStatus) { + Map<Real, RoutingTable.Target> realTable = new HashMap<>(); + for (var target : routingTable.asMap().values()) { + if (target.applicationLevel()) continue; + for (var real : target.reals()) { + realTable.put(real, target); + } + } + routingTable.asMap().values().stream().sorted().distinct().forEach(target -> { + sb.append("upstream ").append(target.id()).append(" {").append("\n"); + + // Check if any target is active. + for (var real : target.reals()) { + boolean explicitRoutingActive = true; + // Check external status service if this is an application-level target + if (target.applicationLevel()) { + RoutingTable.Target targetOfReal = realTable.get(real); + explicitRoutingActive = routingStatus.isActive(targetOfReal.id()); + } + String serverParameter = serverParameter(target, real, explicitRoutingActive); + sb.append(" server ").append(real.hostname()).append(":4443").append(serverParameter).append(";\n"); + } + int healthCheckPort = 4082; + sb.append(" ").append(checkDirective(healthCheckPort)).append("\n"); + sb.append(" ").append(checkHttpSendDirective(target.id())).append("\n"); + sb.append(" random two;\n"); + sb.append("}\n\n"); + }); + } + + private static String serverParameter(RoutingTable.Target target, Real real, boolean routingActive) { + // For each real consider: + // * if not an application-level target -> no parameters + // * if active & routingActive = false AND the upstream contains at least one active host -> "down" + // * if weight assigned = 0 -> "backup" + // * if weight assigned > 0 -> "weight=<weight>" + if (!target.applicationLevel()) return ""; + if (!(real.active() && routingActive) && target.active()) return " down"; + int weight = real.weight(); + if (weight == 0) { + return " backup"; + } else { + return " weight=" + weight; + } + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClient.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClient.java new file mode 100644 index 00000000000..fdfd0f71e96 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClient.java @@ -0,0 +1,104 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.lang.CachedSupplier; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.routing.status.HealthStatus; +import com.yahoo.vespa.hosted.routing.status.ServerGroup; +import com.yahoo.yolean.Exceptions; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Client for the Nginx upstream health status page served at /health-status. + * + * @author oyving + * @author mpolden + */ +public class NginxHealthClient extends AbstractComponent implements HealthStatus { + + private static final URI healthStatusUrl = URI.create("http://localhost:4080/health-status/?format=json"); + private static final Duration requestTimeout = Duration.ofSeconds(5); + private static final Duration cacheTtl = Duration.ofSeconds(5); + + private final CloseableHttpClient httpClient; + private final CachedSupplier<ServerGroup> cache = new CachedSupplier<>(this::getStatus, cacheTtl); + + @Inject + public NginxHealthClient() { + this( + HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout((int) requestTimeout.toMillis()) + .setConnectionRequestTimeout((int) requestTimeout.toMillis()) + .setSocketTimeout((int) requestTimeout.toMillis()) + .build()) + .build() + ); + } + + NginxHealthClient(CloseableHttpClient client) { + this.httpClient = Objects.requireNonNull(client); + } + + @Override + public ServerGroup servers() { + return cache.get(); + } + + private ServerGroup getStatus() { + HttpGet httpGet = new HttpGet(healthStatusUrl); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + String entity = Exceptions.uncheck(() -> EntityUtils.toString(response.getEntity())); + if (response.getStatusLine().getStatusCode() / 100 != 2) { + throw new IllegalArgumentException("Got status code " + response.getStatusLine().getStatusCode() + + " for URL " + healthStatusUrl + ", with response: " + entity); + } + return parseStatus(entity); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static ServerGroup parseStatus(String json) { + Slime slime = SlimeUtils.jsonToSlime(json); + Cursor root = slime.get(); + List<ServerGroup.Server> servers = new ArrayList<>(); + Cursor serversObject = root.field("servers"); + + Cursor streamArray = serversObject.field("stream"); + Cursor serverArray = serversObject.field("server"); // TODO(mpolden): Remove after 2022-03-01 + Cursor array = streamArray.valid() ? streamArray : serverArray; + + array.traverse((ArrayTraverser) (idx, inspector) -> { + String upstreamName = inspector.field("upstream").asString(); + String hostPort = inspector.field("name").asString(); + boolean up = "up".equals(inspector.field("status").asString()); + servers.add(new ServerGroup.Server(upstreamName, hostPort, up)); + }); + return new ServerGroup(servers); + } + + @Override + public void deconstruct() { + Exceptions.uncheck(httpClient::close); + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java new file mode 100644 index 00000000000..79381b8c99e --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java @@ -0,0 +1,191 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.status.HealthStatus; +import com.yahoo.vespa.hosted.routing.status.ServerGroup; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Report Nginx metrics periodically. + * + * @author mortent + * @author mpolden + */ +public class NginxMetricsReporter extends AbstractComponent implements Runnable { + + private static final Duration interval = Duration.ofSeconds(20); + + static final String UPSTREAM_UP_METRIC = "nginx.upstreams.up"; + static final String UPSTREAM_DOWN_METRIC = "nginx.upstreams.down"; + static final String UPSTREAM_UNKNOWN_METRIC = "nginx.upstreams.unknown"; + static final String CONFIG_AGE_METRIC = "upstreams_configuration_age"; + + private final Metric metric; + private final HealthStatus healthStatus; + private final ApplicationId routingApplication; + private final FileSystem fileSystem; + private final ScheduledExecutorService service; + private final Supplier<Optional<RoutingTable>> tableSupplier; + + @Inject + public NginxMetricsReporter(ApplicationIdConfig applicationId, Metric metric, HealthStatus healthStatus, RoutingGenerator routingGenerator) { + this(new ApplicationId(applicationId), metric, healthStatus, FileSystems.getDefault(), interval, routingGenerator::routingTable); + } + + NginxMetricsReporter(ApplicationId application, Metric metric, HealthStatus healthStatus, FileSystem fileSystem, Duration interval, + Supplier<Optional<RoutingTable>> tableSupplier) { + this.metric = Objects.requireNonNull(metric); + this.healthStatus = Objects.requireNonNull(healthStatus); + this.routingApplication = Objects.requireNonNull(application); + this.fileSystem = Objects.requireNonNull(fileSystem); + this.tableSupplier = Objects.requireNonNull(tableSupplier); + this.service = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("nginx-metrics-reporter")); + this.service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public void run() { + Optional<RoutingTable> table = tableSupplier.get(); + table.ifPresent(this::reportHealth); + reportConfigAge(); + } + + private void reportConfigAge() { + Path temporaryNginxConfiguration = NginxPath.temporaryConfig.in(fileSystem); + Path nginxConfiguration = NginxPath.config.in(fileSystem); + Optional<Instant> temporaryConfigModified = lastModified(temporaryNginxConfiguration); + if (temporaryConfigModified.isEmpty()) { + metric.set(CONFIG_AGE_METRIC, 0, metric.createContext(Map.of())); + return; + } + Instant configModified = lastModified(nginxConfiguration).orElse(Instant.EPOCH); + long secondsDiff = Math.abs(Duration.between(configModified, temporaryConfigModified.get()).toSeconds()); + metric.set(CONFIG_AGE_METRIC, secondsDiff, metric.createContext(Map.of())); + } + + private void reportHealth(RoutingTable table) { + Collection<RoutingTable.Target> targets = table.asMap().values(); + Map<String, List<ServerGroup.Server>> status = healthStatus.servers().asMap(); + targets.forEach(service -> { + List<ServerGroup.Server> serversOfUpstream = status.get(service.id()); + if (serversOfUpstream != null) { + reportMetrics(service, serversOfUpstream); + } else { + reportMetricsUnknown(service); + } + }); + + Set<String> knownUpstreams = targets.stream().map(RoutingTable.Target::id).collect(Collectors.toSet()); + long unknownUpstreamCount = status.keySet().stream() + .filter(upstreamName -> !knownUpstreams.contains(upstreamName)) + .count(); + reportMetricsUnknown(unknownUpstreamCount); + } + + // We report a target as unknown if there is no trace of it in the health check yet. This might not be an issue + // (the health check status is a cache), but if it lasts for a long time it might be an error. + private void reportMetricsUnknown(RoutingTable.Target target) { + var dimensions = metricsDimensionsForService(target); + var context = metric.createContext(dimensions); + metric.set(UPSTREAM_UP_METRIC, 0L, context); + metric.set(UPSTREAM_DOWN_METRIC, 0L, context); + metric.set(UPSTREAM_UNKNOWN_METRIC, 1L, context); + } + + // This happens if an application is mentioned in the health check cache, but is not present + // in the routing table. We report this to the routing application, as we don't have anywhere + // else to put the data. + private void reportMetricsUnknown(long count) { + var dimensions = ImmutableMap.of( + "tenantName", routingApplication.tenant().value(), + "app", String.format("%s.%s", routingApplication.application().value(), routingApplication.instance().value()), + "applicationId", routingApplication.toFullString(), + "clusterid", "routing" + ); + var context = metric.createContext(dimensions); + metric.set(UPSTREAM_UNKNOWN_METRIC, count, context); + } + + private void reportMetrics(RoutingTable.Target target, List<ServerGroup.Server> servers) { + long up = countStatus(servers, true); + long down = countStatus(servers, false); + + var dimensions = metricsDimensionsForService(target); + var context = metric.createContext(dimensions); + metric.set(UPSTREAM_UP_METRIC, up, context); + metric.set(UPSTREAM_DOWN_METRIC, down, context); + metric.set(UPSTREAM_UNKNOWN_METRIC, 0L, context); + } + + private Map<String, String> metricsDimensionsForService(RoutingTable.Target target) { + String applicationId = target.tenant().value() + "." + target.application().value(); + String app = target.application().value(); + if (target.instance().isPresent()) { + app += "." + target.instance().get().value(); + applicationId += "." + target.instance().get().value(); + } + return ImmutableMap.of( + "tenantName", target.tenant().value(), + "app", app, + "applicationId", applicationId, + "clusterid", target.cluster().value() + ); + } + + private long countStatus(List<ServerGroup.Server> upstreams, boolean up) { + return upstreams.stream().filter(nginxServer -> up == nginxServer.up()).count(); + } + + private static Optional<Instant> lastModified(Path path) { + try { + return Optional.ofNullable(Files.getLastModifiedTime(path).toInstant()); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void deconstruct() { + Duration timeout = Duration.ofSeconds(10); + service.shutdown(); + try { + if (!service.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) { + throw new RuntimeException("Failed to shutdown executor within " + timeout); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxPath.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxPath.java new file mode 100644 index 00000000000..0cde7725260 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxPath.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +/** + * File system paths used by Nginx. + * + * @author mpolden + */ +enum NginxPath { + + root("/opt/vespa/var/vespa-hosted/routing", null), + config("nginxl4.conf", root), + temporaryConfig("nginxl4.conf.tmp", root); + + public static final DateTimeFormatter ROTATED_SUFFIX_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH:mm:ss.SSS"); + + private final String path; + + NginxPath(String path, NginxPath parent) { + if (parent == null) { + if (path.endsWith("/")) throw new IllegalArgumentException("Path should not end with '/', got '" + path + "'"); + this.path = path; + } else { + if (path.contains("/")) throw new IllegalArgumentException("Filename should not contain '/', got '" + path + "'"); + this.path = parent.path + "/" + path; + } + } + + /** Returns the path to this, bound to given file system */ + public Path in(FileSystem fileSystem) { + return fileSystem.getPath(path); + } + + /** Returns the rotated path of this with given instant, bound to given file system */ + public Path rotatedIn(FileSystem fileSystem, Instant instant) { + LocalDateTime dateTime = LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + return fileSystem.getPath(path + "-" + ROTATED_SUFFIX_FORMAT.format(dateTime)); + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandler.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandler.java new file mode 100644 index 00000000000..9ed05278331 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandler.java @@ -0,0 +1,109 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.restapi; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.status.HealthStatus; +import com.yahoo.vespa.hosted.routing.status.RoutingStatus; + +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * This handler implements the /akamai health check. + * + * The global routing service polls /akamai to determine if a deployment should receive requests via its global + * endpoint. + * + * @author oyving + * @author mpolden + * @author Torbjorn Smorgrav + * @author Wacek Kusnierczyk + */ +public class AkamaiHandler extends ThreadedHttpRequestHandler { + + public static final String + ROTATION_UNKNOWN_MESSAGE = "Rotation not found", + ROTATION_UNAVAILABLE_MESSAGE = "Rotation set unavailable", + ROTATION_UNHEALTHY_MESSAGE = "Rotation unhealthy", + ROTATION_INACTIVE_MESSAGE = "Rotation not available", + ROTATION_OK_MESSAGE = "Rotation OK"; + + private final RoutingStatus routingStatus; + private final HealthStatus healthStatus; + private final Supplier<Optional<RoutingTable>> tableSupplier; + + @Inject + public AkamaiHandler(Context parentCtx, + RoutingGenerator routingGenerator, + RoutingStatus routingStatus, + HealthStatus healthStatus) { + this(parentCtx, routingGenerator::routingTable, routingStatus, healthStatus); + } + + AkamaiHandler(Context parentCtx, + Supplier<Optional<RoutingTable>> tableSupplier, + RoutingStatus routingStatus, + HealthStatus healthStatus) { + super(parentCtx); + this.routingStatus = Objects.requireNonNull(routingStatus); + this.healthStatus = Objects.requireNonNull(healthStatus); + this.tableSupplier = Objects.requireNonNull(tableSupplier); + } + + @Override + public HttpResponse handle(HttpRequest request) { + Path path = new Path(request.getUri()); + if (path.matches("/akamai/v1/status")) { + return status(request); + } + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse status(HttpRequest request) { + String hostHeader = request.getHeader("host"); + String hostname = withoutPort(hostHeader); + Optional<RoutingTable.Target> target = tableSupplier.get().flatMap(table -> table.targetOf(hostname, RoutingMethod.sharedLayer4)); + + if (target.isEmpty()) + return response(404, hostHeader, "", ROTATION_UNKNOWN_MESSAGE); + + if (!target.get().active()) + return response(404, hostHeader, "", ROTATION_INACTIVE_MESSAGE); + + String upstreamName = target.get().id(); + + if (!routingStatus.isActive(upstreamName)) + return response(404, hostHeader, upstreamName, ROTATION_UNAVAILABLE_MESSAGE); + + if (!healthStatus.servers().isHealthy(upstreamName)) + return response(502, hostHeader, upstreamName, ROTATION_UNHEALTHY_MESSAGE); + + return response(200, hostHeader, upstreamName, ROTATION_OK_MESSAGE); + } + + private static HttpResponse response(int status, String hostname, String name, String message) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("hostname", hostname); + root.setString("upstream", name); + root.setString("message", message); + return new SlimeJsonResponse(status, slime); + } + + private static String withoutPort(String hostHeader) { + return hostHeader.replaceFirst("(:[\\d]+)?$", ""); + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/package-info.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/package-info.java new file mode 100644 index 00000000000..6a1d7f8234e --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/restapi/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +@ExportPackage +package com.yahoo.vespa.hosted.routing.restapi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/HealthStatus.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/HealthStatus.java new file mode 100644 index 00000000000..e5d50390011 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/HealthStatus.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.status; + +/** + * Interface for accessing the health status of servers behind a router/reverse proxy. + * +* @author oyving +*/ +// TODO(mpolden): Make this a part of the future Router interface +public interface HealthStatus { + + /** Returns status of all servers */ + ServerGroup servers(); + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatus.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatus.java new file mode 100644 index 00000000000..9c030aeb100 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatus.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.status; + +/** + * Interface for accessing the global routing status of an upstream server. + * +* @author oyving +*/ +public interface RoutingStatus { + + /** Returns whether the given upstream name is active in global routing */ + boolean isActive(String upstreamName); + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java new file mode 100644 index 00000000000..70ed84aa014 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.status; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.lang.CachedSupplier; +import com.yahoo.routing.config.ZoneConfig; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.yolean.Exceptions; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.NoopUserTokenHandler; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.time.Duration; +import java.util.Objects; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Caching client for the /routing/v1/status API on the config server. That API decides if a deployment (or entire zone) + * is explicitly disabled in any global endpoints. + * + * This caches the status for a brief period to avoid drowning config servers with requests from health check pollers. + * + * @author oyving + * @author andreer + * @author mpolden + */ +public class RoutingStatusClient extends AbstractComponent implements RoutingStatus { + + private static final Logger log = Logger.getLogger(RoutingStatusClient.class.getName()); + private static final Duration requestTimeout = Duration.ofSeconds(2); + private static final Duration cacheTtl = Duration.ofSeconds(5); + + private final CloseableHttpClient httpClient; + private final URI configServerVip; + private final CachedSupplier<Status> cache = new CachedSupplier<>(this::status, cacheTtl); + + @Inject + public RoutingStatusClient(ZoneConfig config, ServiceIdentityProvider provider) { + this( + HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout((int) requestTimeout.toMillis()) + .setConnectionRequestTimeout((int) requestTimeout.toMillis()) + .setSocketTimeout((int) requestTimeout.toMillis()) + .build()) + .setSSLContext(provider.getIdentitySslContext()) + .setSSLHostnameVerifier(createHostnameVerifier(config)) + // Required to enable connection pooling, which is disabled by default when using mTLS + .setUserTokenHandler(NoopUserTokenHandler.INSTANCE) + .setUserAgent("hosted-vespa-routing-status-client") + .build(), + URI.create(config.configserverVipUrl()) + ); + } + + public RoutingStatusClient(CloseableHttpClient httpClient, URI configServerVip) { + this.httpClient = Objects.requireNonNull(httpClient); + this.configServerVip = Objects.requireNonNull(configServerVip); + } + + @Override + public boolean isActive(String upstreamName) { + try { + return cache.get().isActive(upstreamName); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to get status for '" + upstreamName + "': " + Exceptions.toMessageString(e)); + return true; // Assume IN if cache update fails + } + } + + @Override + public void deconstruct() { + Exceptions.uncheck(httpClient::close); + } + + void invalidateCache() { + cache.invalidate(); + } + + private Status status() { + Slime slime = get("/routing/v2/status"); + Cursor root = slime.get(); + Set<String> inactiveDeployments = SlimeUtils.entriesStream(root.field("inactiveDeployments")) + .map(inspector -> inspector.field("upstreamName").asString()) + .collect(Collectors.toUnmodifiableSet()); + boolean zoneActive = root.field("zoneActive").asBool(); + return new Status(zoneActive, inactiveDeployments); + } + + private Slime get(String path) { + URI url = configServerVip.resolve(path); + HttpGet httpGet = new HttpGet(url); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + String entity = Exceptions.uncheck(() -> EntityUtils.toString(response.getEntity())); + if (response.getStatusLine().getStatusCode() / 100 != 2) { + throw new IllegalArgumentException("Got status code " + response.getStatusLine().getStatusCode() + + " for URL " + url + ", with response: " + entity); + } + return SlimeUtils.jsonToSlime(entity); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static AthenzIdentityVerifier createHostnameVerifier(ZoneConfig config) { + return new AthenzIdentityVerifier(Set.of(new AthenzService(config.configserverAthenzDomain(), + config.configserverAthenzServiceName()))); + } + + private static class Status { + + private final boolean zoneActive; + private final Set<String> inactiveDeployments; + + public Status(boolean zoneActive, Set<String> inactiveDeployments) { + this.zoneActive = zoneActive; + this.inactiveDeployments = Set.copyOf(Objects.requireNonNull(inactiveDeployments)); + } + + public boolean isActive(String upstreamName) { + return zoneActive && !inactiveDeployments.contains(upstreamName); + } + + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/ServerGroup.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/ServerGroup.java new file mode 100644 index 00000000000..f1a87aa7106 --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/ServerGroup.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.status; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A group servers behind a router/reverse proxy. + * + * @author mpolden + */ +public class ServerGroup { + + private static final double requiredUpFraction = 0.25D; + + private final Map<String, List<Server>> servers; + + public ServerGroup(List<Server> servers) { + this.servers = servers.stream().collect(Collectors.collectingAndThen(Collectors.groupingBy(Server::upstreamName), + Collections::unmodifiableMap)); + } + + public Map<String, List<Server>> asMap() { + return servers; + } + + /** Returns whether given upstream is healthy */ + public boolean isHealthy(String upstreamName) { + // TODO(mpolden): Look up key directly here once layer 4 config (and thus "-feed" upstreams) are gone + List<Server> upstreamServers = servers.values().stream() + .flatMap(Collection::stream) + .filter(server -> upstreamName.startsWith(server.upstreamName())) + .collect(Collectors.toList()); + long upCount = upstreamServers.stream() + .filter(Server::up) + .count(); + return upCount > upstreamServers.size() * requiredUpFraction; + } + + public static class Server { + + private final String upstreamName; + private final String hostport; + private final boolean up; + + public Server(String upstreamName, String hostport, boolean up) { + this.upstreamName = upstreamName; + this.hostport = hostport; + this.up = up; + } + + public String upstreamName() { + return upstreamName; + } + + public String hostport() { + return hostport; + } + + public boolean up() { + return up; + } + + } + +} diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/package-info.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/package-info.java new file mode 100644 index 00000000000..2cd9e6a141e --- /dev/null +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +@ExportPackage +package com.yahoo.vespa.hosted.routing.status; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/routing-generator/src/main/resources/configdefinitions/routing.config.zone.def b/routing-generator/src/main/resources/configdefinitions/routing.config.zone.def new file mode 100755 index 00000000000..e89cc6ba532 --- /dev/null +++ b/routing-generator/src/main/resources/configdefinitions/routing.config.zone.def @@ -0,0 +1,14 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=routing.config + +# URL to config server load balancer +configserverVipUrl string + +# Athenz domain/service name of config server +configserverAthenzDomain string +configserverAthenzServiceName string + +# Tenant config server endpoint, used to fetch tenant (mapping) info +# Auto detected if empty (from VESPA_CONFIG_SOURCES) +# E.g.: tcp/cfg1.example.com:19070 +configserver[] string diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java new file mode 100644 index 00000000000..3e8b9be572f --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.ConfigKey; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author mpolden + */ +public class RoutingGeneratorTest { + + @Test(timeout = 2000) + public void config_subscription() { + RouterMock router = new RouterMock(); + RoutingGenerator generator = new RoutingGenerator(new ConfigSetMock(), router, new ManualClock()); + try { + router.awaitLoad(); + assertNotNull("Router loads table", router.currentTable); + assertEquals("Routing generator and router has same table", + generator.routingTable().get(), + router.currentTable); + } finally { + generator.deconstruct(); + } + } + + private static class RouterMock implements Router { + + private final CountDownLatch latch = new CountDownLatch(1); + + private volatile RoutingTable currentTable = null; + + @Override + public void load(RoutingTable table) { + currentTable = table; + latch.countDown(); + } + + public void awaitLoad() { + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + } + + @SuppressWarnings("removal") // TODO Vespa 8: remove + private static class ConfigSetMock extends ConfigSet { + + private int attempt = 0; + + public ConfigSetMock() { + addBuilder("*", new LbServicesConfig.Builder()); + } + + @Override + public ConfigInstance.Builder get(ConfigKey<?> key) { + if (++attempt <= 5) { + throw new RuntimeException("Failed to get config on attempt " + attempt); + } + return super.get(key); + } + + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java new file mode 100644 index 00000000000..dc4df7d45ad --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.routing.RoutingTable.Endpoint; +import com.yahoo.vespa.hosted.routing.RoutingTable.Real; +import com.yahoo.vespa.hosted.routing.RoutingTable.Target; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class RoutingTableTest { + + @Test + public void translate_from_lb_services_config() { + RoutingTable expected = new RoutingTable(Map.of( + new Endpoint("beta.music.vespa.us-north-1.vespa.oath.cloud", RoutingMethod.sharedLayer4), + Target.create(ApplicationId.from("vespa", "music", "beta"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host3-beta", 4443, 1, true), + new Real("host4-beta", 4443, 1, true))), + + new Endpoint("music.vespa.global.vespa.oath.cloud", RoutingMethod.sharedLayer4), + Target.create(ApplicationId.from("vespa", "music", "default"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host1-default", 4443, 1, true), + new Real("host2-default", 4443, 1, true))), + + new Endpoint("music.vespa.us-north-1.vespa.oath.cloud", RoutingMethod.sharedLayer4), + Target.create(ApplicationId.from("vespa", "music", "default"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host1-default", 4443, 1, true), + new Real("host2-default", 4443, 1, true))), + + new Endpoint("rotation-02.vespa.global.routing", RoutingMethod.sharedLayer4), + Target.create(ApplicationId.from("vespa", "music", "default"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host1-default", 4443, 1, true), + new Real("host2-default", 4443, 1, true))), + + new Endpoint("rotation-02.vespa.global.routing", RoutingMethod.shared), + Target.create(ApplicationId.from("vespa", "music", "default"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host1-default", 4443, 1, true), + new Real("host2-default", 4443, 1, true))), + + new Endpoint("music--vespa.global.vespa.oath.cloud", RoutingMethod.shared), + Target.create(ApplicationId.from("vespa", "music", "default"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host1-default", 4443, 1, true), + new Real("host2-default", 4443, 1, true))), + + new Endpoint("use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud", RoutingMethod.sharedLayer4), + Target.create("use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud", TenantName.from("vespa"), ApplicationName.from("music"), + ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), + List.of(new Real("host3-beta", 4443, 1, true), + new Real("host4-beta", 4443, 1, true), + new Real("host1-default", 4443, 0, true), + new Real("host2-default", 4443, 0, true))) + ), 42); + + RoutingTable actual = TestUtil.readRoutingTable("lbservices-config"); + assertEquals(expected, actual); + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java new file mode 100644 index 00000000000..09440cfaac7 --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.yolean.Exceptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +/** + * @author mpolden + */ +public class TestUtil { + + private static final Path testData = Paths.get("src/test/resources/"); + + @SuppressWarnings("removal") // TODO Vespa 8: remove + public static RoutingTable readRoutingTable(String filename) { + List<String> lines = Exceptions.uncheck(() -> Files.readAllLines(testFile(filename), + StandardCharsets.UTF_8)); + LbServicesConfig lbServicesConfig = new CfgConfigPayloadBuilder().deserialize(lines) + .toInstance(LbServicesConfig.class, "*"); + return RoutingTable.from(lbServicesConfig, 42); + } + + public static Path testFile(String filename) { + return testData.resolve(filename); + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HealthStatusMock.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HealthStatusMock.java new file mode 100644 index 00000000000..66aff350b8b --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HealthStatusMock.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.mock; + +import com.yahoo.vespa.hosted.routing.status.HealthStatus; +import com.yahoo.vespa.hosted.routing.status.ServerGroup; + +import java.util.List; + +/** + * @author mpolden + */ +public class HealthStatusMock implements HealthStatus { + + private ServerGroup status = new ServerGroup(List.of()); + + public HealthStatusMock setStatus(ServerGroup newStatus) { + this.status = newStatus; + return this; + } + + @Override + public ServerGroup servers() { + return status; + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HttpClientMock.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HttpClientMock.java new file mode 100644 index 00000000000..025eac90b8d --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/HttpClientMock.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.mock; + +import com.yahoo.yolean.Exceptions; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpVersion; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.params.HttpParams; +import org.apache.http.protocol.HttpContext; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** + * @author mpolden + */ +@SuppressWarnings("deprecation") // Deprecations in third-party interface +public class HttpClientMock extends CloseableHttpClient { + + private final Map<String, CloseableHttpResponse> responses = new HashMap<>(); + + public HttpClientMock setResponse(String method, String url, CloseableHttpResponse response) { + responses.put(requestKey(method, url), response); + return this; + } + + @Override + protected CloseableHttpResponse doExecute(HttpHost httpHost, HttpRequest httpRequest, HttpContext httpContext) { + String key = requestKey(httpRequest.getRequestLine().getMethod(), httpRequest.getRequestLine().getUri()); + CloseableHttpResponse response = responses.get(key); + if (response == null) { + throw new IllegalArgumentException("No response defined for " + key); + } + return response; + } + + @Override + public void close() {} + + @Override + public HttpParams getParams() { + throw new UnsupportedOperationException(); + } + + @Override + public ClientConnectionManager getConnectionManager() { + throw new UnsupportedOperationException(); + } + + private static String requestKey(String method, String url) { + return method.toUpperCase(Locale.ENGLISH) + " " + url; + } + + public static class JsonResponse extends BasicHttpResponse implements CloseableHttpResponse { + + public JsonResponse(Path jsonFile, int code) { + this(Exceptions.uncheck(() -> Files.readString(jsonFile)), code); + } + + public JsonResponse(String json, int code) { + super(HttpVersion.HTTP_1_1, code, null); + setEntity(new StringEntity(json, ContentType.APPLICATION_JSON)); + } + + @Override + public void close() {} + + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/RoutingStatusMock.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/RoutingStatusMock.java new file mode 100644 index 00000000000..931627cd7c4 --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/mock/RoutingStatusMock.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.mock; + +import com.yahoo.vespa.hosted.routing.status.RoutingStatus; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author mortent + */ +public class RoutingStatusMock implements RoutingStatus { + + private final Set<String> outOfRotation = new HashSet<>(); + + @Override + public boolean isActive(String upstreamName) { + return !outOfRotation.contains(upstreamName); + } + + public RoutingStatusMock setStatus(String upstreamName, boolean active) { + if (active) { + outOfRotation.remove(upstreamName); + } else { + outOfRotation.add(upstreamName); + } + return this; + } +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClientTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClientTest.java new file mode 100644 index 00000000000..722bc55437f --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxHealthClientTest.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.yahoo.vespa.hosted.routing.mock.HttpClientMock; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author oyving + * @author mpolden + */ +public class NginxHealthClientTest { + + @Test + public void unknown_endpoint_is_down() { + NginxHealthClient client = createClient("nginx-health-output.json"); + assertFalse(client.servers().isHealthy("no.such.endpoint")); + } + + @Test + public void all_down_endpoint_is_down() { + NginxHealthClient service = createClient("nginx-health-output-all-down.json"); + assertFalse(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + } + + @Test + public void all_up_endpoint_is_up() { + NginxHealthClient service = createClient("nginx-health-output-all-up.json"); + assertTrue(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + } + + @Test + public void two_down_endpoint_is_down() { + NginxHealthClient service = createClient("nginx-health-output-policy-down.json"); + assertFalse(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + } + + @Test + public void one_down_endpoint_is_up() { + NginxHealthClient service = createClient("nginx-health-output-policy-up.json"); + assertTrue(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + } + + @Test + public void all_up_but_other_endpoint_down() { + NginxHealthClient service = createClient("nginx-health-output-all-up-but-other-down.json"); + assertTrue(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + assertFalse(service.servers().isHealthy("frog.prod.music.vespa.us-east-2.prod")); + } + + @Test + public void all_up_but_other_endpoint_down_stream() { + NginxHealthClient service = createClient("nginx-health-output-stream.json"); + assertTrue(service.servers().isHealthy("gateway.prod.music.vespa.us-east-2.prod")); + assertFalse(service.servers().isHealthy("frog.prod.music.vespa.us-east-2.prod")); + } + + private static NginxHealthClient createClient(String file) { + HttpClientMock httpClient = new HttpClientMock().setResponse("GET", + "http://localhost:4080/health-status/?format=json", + response(file)); + return new NginxHealthClient(httpClient); + } + + private static HttpClientMock.JsonResponse response(String file) { + return new HttpClientMock.JsonResponse(Paths.get("src/test/resources/", file), 200); + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporterTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporterTest.java new file mode 100644 index 00000000000..eea3724e7e9 --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporterTest.java @@ -0,0 +1,163 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.google.common.jimfs.Jimfs; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.RoutingTable.Endpoint; +import com.yahoo.vespa.hosted.routing.RoutingTable.Target; +import com.yahoo.vespa.hosted.routing.mock.HealthStatusMock; +import com.yahoo.vespa.hosted.routing.status.ServerGroup; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author mortent + * @author mpolden + */ +public class NginxMetricsReporterTest { + + private static final ApplicationId routingApp = ApplicationId.from("hosted-vespa", "routing", "default"); + + private static final Target target0 = createTarget("vespa", "music", "prod", "gateway"); + private static final Target target1 = createTarget("vespa", "music", "prod", "qrs"); + private static final Target target2 = createTarget("vespa", "donbot", "default", "default"); + private static final Target target3 = createTarget("notchecked", "notchecked", "default", "default"); + private static final Target target4 = createTarget("not", "appearing-in-routing", "default", "default"); + private static final Target target5 = createTarget(routingApp.tenant().value(), routingApp.application().value(), routingApp.instance().value(), "routing"); + + private final MockMetric metrics = new MockMetric(); + private final RoutingTable routingTable = createRoutingTable(); + private final HealthStatusMock healthService = new HealthStatusMock(); + private final FileSystem fileSystem = Jimfs.newFileSystem(); + private final NginxMetricsReporter reporter = new NginxMetricsReporter(routingApp, metrics, healthService, + fileSystem, Duration.ofDays(1), + () -> Optional.of(routingTable)); + + @Test + public void upstream_metrics() { + List<ServerGroup.Server> servers = List.of( + new ServerGroup.Server("gateway.prod.music.vespa.us-east-2.prod", "10.78.114.166:4080", true), + new ServerGroup.Server("gateway.prod.music.vespa.us-east-2.prod", "10.78.115.68:4080", true), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.114.166:4080", true), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.115.68:4080", true), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.114.166:4080", false), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.115.68:4080", false), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.114.166:4080", false), + new ServerGroup.Server("qrs.prod.music.vespa.us-east-2.prod", "10.78.115.68:4080", false), + new ServerGroup.Server("donbot.vespa.us-east-2.prod", "10.201.8.47:4080", true), + new ServerGroup.Server("donbot.vespa.us-east-2.prod", "10.201.14.46:4080", false), + new ServerGroup.Server("appearing-in-routing.not.us-east-2.prod", "10.201.14.50:4080", false) + ); + healthService.setStatus(new ServerGroup(servers)); + reporter.run(); + + assertEquals(2D, getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target0)), Double.MIN_VALUE); + assertEquals(0D, getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target0)), Double.MIN_VALUE); + assertEquals(0D, getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target0)), Double.MIN_VALUE); + + assertEquals(2L, getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target1)), Double.MIN_VALUE); + assertEquals(4L, getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target1)), Double.MIN_VALUE); + assertEquals(0L, getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target1)), Double.MIN_VALUE); + + assertEquals(1D, getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target2)), Double.MIN_VALUE); + assertEquals(1D, getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target2)), Double.MIN_VALUE); + assertEquals(0D, getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target2)), Double.MIN_VALUE); + + // If the application appears in routing table - but not in health check cache yet + assertEquals(0D, getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target3)), Double.MIN_VALUE); + assertEquals(0D, getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target3)), Double.MIN_VALUE); + assertEquals(1D, getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target3)), Double.MIN_VALUE); + + // If the application does not appear in routing table - but still appears in cache + assertNull(getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target4))); + assertNull(getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target4))); + assertNull(getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target4))); + + assertNull(getMetric(NginxMetricsReporter.UPSTREAM_UP_METRIC, dimensionsOf(target5))); + assertNull(getMetric(NginxMetricsReporter.UPSTREAM_DOWN_METRIC, dimensionsOf(target5))); + assertEquals(1D, getMetric(NginxMetricsReporter.UPSTREAM_UNKNOWN_METRIC, dimensionsOf(target5)), Double.MIN_VALUE); + } + + @Test + public void config_age_metric() throws Exception { + reporter.run(); + // No files exist + assertEquals(0D, getMetric(NginxMetricsReporter.CONFIG_AGE_METRIC), Double.MIN_VALUE); + + // Only temporary file exists + Path configRoot = fileSystem.getPath("/opt/vespa/var/vespa-hosted/routing/"); + Path tempFile = configRoot.resolve("nginxl4.conf.tmp"); + createFile(tempFile, Instant.ofEpochSecond(123)); + reporter.run(); + assertEquals(123D, getMetric(NginxMetricsReporter.CONFIG_AGE_METRIC), Double.MIN_VALUE); + + // Only main file exists + Files.delete(tempFile); + createFile(configRoot.resolve("nginxl4.conf"), Instant.ofEpochSecond(456)); + reporter.run(); + assertEquals(0D, getMetric(NginxMetricsReporter.CONFIG_AGE_METRIC), Double.MIN_VALUE); + + // Both files exist + createFile(tempFile, Instant.ofEpochSecond(123)); + reporter.run(); + assertEquals(333D, getMetric(NginxMetricsReporter.CONFIG_AGE_METRIC), Double.MIN_VALUE); + } + + private double getMetric(String name) { + return getMetric(name, Map.of()); + } + + private Double getMetric(String name, Map<String, ?> dimensions) { + Map<Map<String, ?>, Double> metric = metrics.metrics().get(name); + if (metric == null) throw new IllegalArgumentException("Metric '" + name + "' not found"); + return metric.get(dimensions); + } + + private void createFile(Path path, Instant lastModified) throws IOException { + Files.createDirectories(path.getParent()); + Files.createFile(path); + Files.setLastModifiedTime(path, FileTime.from(lastModified)); + } + + private Map<String, ?> dimensionsOf(Target target) { + return Map.of( + "tenantName", target.tenant().value(), + "app", String.format("%s.%s", target.application().value(), target.instance().get().value()), + "applicationId", String.format("%s.%s.%s", target.tenant().value(), target.application().value(), target.instance().get().value()), + "clusterid", target.cluster().value() + ); + } + + private static Target createTarget(String tenantName, String applicationName, String instanceName, String clusterName) { + ZoneId zone = ZoneId.from("prod", "us-east-2"); + ClusterSpec.Id cluster = ClusterSpec.Id.from(clusterName); + return Target.create(ApplicationId.from(tenantName, applicationName, instanceName), cluster, zone, List.of()); + } + + private static RoutingTable createRoutingTable() { + return new RoutingTable(Map.of(new Endpoint("endpoint0", RoutingMethod.sharedLayer4), target0, + new Endpoint("endpoint1", RoutingMethod.sharedLayer4), target1, + new Endpoint("endpoint2", RoutingMethod.sharedLayer4), target2, + new Endpoint("endpoint3", RoutingMethod.sharedLayer4), target3), + 42); + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxTest.java new file mode 100644 index 00000000000..e79b3e8787e --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/nginx/NginxTest.java @@ -0,0 +1,217 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.nginx; + +import com.google.common.jimfs.Jimfs; +import com.yahoo.collections.Pair; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.TestUtil; +import com.yahoo.vespa.hosted.routing.mock.RoutingStatusMock; +import com.yahoo.yolean.Exceptions; +import com.yahoo.yolean.concurrent.Sleeper; +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author mpolden + */ +public class NginxTest { + + @Test + public void load_routing_table() { + NginxTester tester = new NginxTester(); + tester.clock.setInstant(Instant.parse("2022-01-01T15:00:00Z")); + + // Load routing table + RoutingTable table0 = TestUtil.readRoutingTable("lbservices-config"); + tester.load(table0) + .assertVerifiedConfig(1) + .assertLoadedConfig(true) + .assertConfigContents("nginx.conf") + .assertTemporaryConfigRemoved(true) + .assertMetric(Nginx.CONFIG_RELOADS_METRIC, 1) + .assertMetric(Nginx.OK_CONFIG_RELOADS_METRIC, 1) + .assertMetric(Nginx.GENERATED_UPSTREAMS_METRIC, 5); + + // Loading the same table again does nothing + tester.load(table0) + .assertVerifiedConfig(1) + .assertLoadedConfig(false) + .assertConfigContents("nginx.conf") + .assertTemporaryConfigRemoved(true) + .assertMetric(Nginx.CONFIG_RELOADS_METRIC, 1) + .assertMetric(Nginx.OK_CONFIG_RELOADS_METRIC, 1) + .assertMetric(Nginx.GENERATED_UPSTREAMS_METRIC, 5); + + // A new table is loaded + Map<RoutingTable.Endpoint, RoutingTable.Target> newEntries = new HashMap<>(table0.asMap()); + newEntries.put(new RoutingTable.Endpoint("endpoint1", RoutingMethod.sharedLayer4), + RoutingTable.Target.create(ApplicationId.from("t1", "a1", "i1"), + ClusterSpec.Id.from("default"), + ZoneId.from("prod", "us-north-1"), + List.of(new RoutingTable.Real("host42", 4443, 1, true)))); + RoutingTable table1 = new RoutingTable(newEntries, 43); + + // Verification of new table fails enough times to exhaust retries + tester.processExecuter.withFailCount(10); + try { + tester.load(table1); + fail("Expected exception"); + } catch (Exception ignored) {} + tester.assertVerifiedConfig(5) + .assertLoadedConfig(false) + .assertConfigContents("nginx.conf") + .assertTemporaryConfigRemoved(false) + .assertMetric(Nginx.CONFIG_RELOADS_METRIC, 1) + .assertMetric(Nginx.OK_CONFIG_RELOADS_METRIC, 1); + + // Verification succeeds, with few enough failures + tester.processExecuter.withFailCount(3); + tester.load(table1) + .assertVerifiedConfig(3) + .assertLoadedConfig(true) + .assertConfigContents("nginx-updated.conf") + .assertTemporaryConfigRemoved(true) + .assertRotatedFiles("nginxl4.conf-2022-01-01-15:00:00.000") + .assertMetric(Nginx.CONFIG_RELOADS_METRIC, 2) + .assertMetric(Nginx.OK_CONFIG_RELOADS_METRIC, 2); + + // Some time passes and new tables are loaded. Old rotated files are removed + tester.clock.advance(Duration.ofDays(3)); + // Simulate old rotated layer 7 config file, which should be removed + Exceptions.uncheck(() -> Files.createFile(NginxPath.root.in(tester.fileSystem).resolve("nginx.conf-2021-12-15-15:00:00.000"))); + tester.load(table0); + tester.clock.advance(Duration.ofDays(4).plusSeconds(1)); + tester.load(table1) + .assertRotatedFiles("nginxl4.conf-2022-01-04-15:00:00.000", + "nginxl4.conf-2022-01-08-15:00:01.000"); + tester.clock.advance(Duration.ofDays(4)); + tester.load(table1) // Same table is loaded again, which is a no-op, but old rotated files are still removed + .assertRotatedFiles("nginxl4.conf-2022-01-08-15:00:01.000"); + } + + private static class NginxTester { + + private final FileSystem fileSystem = Jimfs.newFileSystem(); + private final ManualClock clock = new ManualClock(); + private final RoutingStatusMock routingStatus = new RoutingStatusMock(); + private final ProcessExecuterMock processExecuter = new ProcessExecuterMock(); + private final MockMetric metric = new MockMetric(); + private final Nginx nginx = new Nginx(fileSystem, processExecuter, Sleeper.NOOP, clock, routingStatus, metric); + + public NginxTester load(RoutingTable table) { + processExecuter.clearHistory(); + nginx.load(table); + return this; + } + + public NginxTester assertMetric(String name, double expected) { + assertEquals("Metric " + name + " has expected value", expected, metric.metrics().get(name).get(Map.of()), Double.MIN_VALUE); + return this; + } + + public NginxTester assertConfigContents(String expectedConfig) { + String expected = Exceptions.uncheck(() -> Files.readString(TestUtil.testFile(expectedConfig))); + String actual = Exceptions.uncheck(() -> Files.readString(NginxPath.config.in(fileSystem))); + assertEquals(expected, actual); + return this; + } + + public NginxTester assertTemporaryConfigRemoved(boolean removed) { + Path path = NginxPath.temporaryConfig.in(fileSystem); + assertEquals(path + (removed ? " does not exist" : " exists"), removed, !Files.exists(path)); + return this; + } + + public NginxTester assertRotatedFiles(String... expectedRotatedFiles) { + List<String> rotatedFiles = Exceptions.uncheck(() -> Files.list(NginxPath.root.in(fileSystem)) + .map(path -> path.getFileName().toString()) + .filter(filename -> filename.contains(".conf-")) + .collect(Collectors.toList())); + assertEquals(List.of(expectedRotatedFiles), rotatedFiles); + return this; + } + + public NginxTester assertVerifiedConfig(int times) { + for (int i = 0; i < times; i++) { + assertEquals("/usr/bin/sudo /opt/vespa/bin/vespa-verify-nginx", processExecuter.history().get(i)); + } + return this; + } + + public NginxTester assertLoadedConfig(boolean loaded) { + String reloadCommand = "/usr/bin/sudo /opt/vespa/bin/vespa-reload-nginx"; + if (loaded) { + assertEquals(reloadCommand, processExecuter.history().get(processExecuter.history().size() - 1)); + } else { + assertTrue("Config is not loaded", + processExecuter.history.stream().noneMatch(command -> command.equals(reloadCommand))); + } + return this; + } + + } + + private static class ProcessExecuterMock extends ProcessExecuter { + + private final List<String> history = new ArrayList<>(); + + private int wantedFailCount = 0; + private int currentFailCount = 0; + + public List<String> history() { + return Collections.unmodifiableList(history); + } + + public ProcessExecuterMock clearHistory() { + history.clear(); + return this; + } + + public ProcessExecuterMock withFailCount(int count) { + this.wantedFailCount = count; + this.currentFailCount = 0; + return this; + } + + @Override + public Pair<Integer, String> exec(String command) { + history.add(command); + int exitCode = 0; + String out = ""; + if (++currentFailCount <= wantedFailCount) { + exitCode = 1; + out = "failing to unit test"; + } + return new Pair<>(exitCode, out); + } + + @Override + public Pair<Integer, String> exec(String[] command) { + return exec(String.join(" ", command)); + } + + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandlerTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandlerTest.java new file mode 100644 index 00000000000..2814fcff8f7 --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/restapi/AkamaiHandlerTest.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.restapi; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpRequestBuilder; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.vespa.hosted.routing.RoutingTable; +import com.yahoo.vespa.hosted.routing.RoutingTable.Endpoint; +import com.yahoo.vespa.hosted.routing.mock.HealthStatusMock; +import com.yahoo.vespa.hosted.routing.mock.RoutingStatusMock; +import com.yahoo.vespa.hosted.routing.status.HealthStatus; +import com.yahoo.vespa.hosted.routing.status.ServerGroup; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author oyving + * @author mpolden + */ +public class AkamaiHandlerTest { + + private static final String + ENDPOINT_OK = "ok.vespa.yahooapis.com", + ENDPOINT_UNKNOWN = "unknown.vespa.yahooapis.com", + ENDPOINT_UNAVAILABLE = "out.vespa.yahooapis.com", + ENDPOINT_UNHEALTHY = "unhealthy.vespa.yahooapis.com", + ENDPOINT_INACTIVE = "inactive.vespa.yahooapis.com"; + + private static final String ENDPOINT_WITH_PORT_OK = ENDPOINT_OK + ":4080"; + + private final RoutingStatusMock statusService = new RoutingStatusMock().setStatus("i3.a3.t3.us-north-1.prod", false); + + private final HealthStatus healthStatus = new HealthStatusMock().setStatus(new ServerGroup(List.of( + new ServerGroup.Server("i1.a1.t1.us-north-1.prod", "hostport", true), + new ServerGroup.Server("i2.a2.t2.us-north-1.prod", "hostport", false)))); + + private final AkamaiHandler handler = new AkamaiHandler(ThreadedHttpRequestHandler.testContext(), + () -> Optional.of(makeRoutingTable()), + statusService, + healthStatus); + + @Test + public void ok_endpoint() { + assertResponse(ENDPOINT_OK, 200, AkamaiHandler.ROTATION_OK_MESSAGE); + assertResponse(ENDPOINT_WITH_PORT_OK, 200, AkamaiHandler.ROTATION_OK_MESSAGE); + } + + @Test + public void unknown_endpoint() { + assertResponse(ENDPOINT_UNKNOWN, 404, AkamaiHandler.ROTATION_UNKNOWN_MESSAGE); + } + + @Test + public void out_of_rotation_endpoint() { + assertResponse(ENDPOINT_UNAVAILABLE, 404, AkamaiHandler.ROTATION_UNAVAILABLE_MESSAGE); + } + + @Test + public void unhealthy_endpoint() { + assertResponse(ENDPOINT_UNHEALTHY, 502, AkamaiHandler.ROTATION_UNHEALTHY_MESSAGE); + } + + @Test + public void inactive_endpoint() { + assertResponse(ENDPOINT_INACTIVE, 404, AkamaiHandler.ROTATION_INACTIVE_MESSAGE); + } + + private void assertResponse(String rotation, int status, String message) { + HttpRequest req = HttpRequestBuilder.create(com.yahoo.jdisc.http.HttpRequest.Method.GET, "/akamai/v1/status") + .withHeader("Host", rotation) + .build(); + HttpResponse response = handler.handle(req); + assertEquals(status, response.getStatus()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Exceptions.uncheck(() -> response.render(out)); + String responseBody = out.toString(); + + String expected = "\"message\":\"" + message + "\""; + assertTrue("Contains expected message", responseBody.contains(expected)); + } + + private static RoutingTable makeRoutingTable() { + return new RoutingTable(Map.of( + new Endpoint(ENDPOINT_OK, RoutingMethod.sharedLayer4), createTarget("t1", "a1", "i1", "default", true), + new Endpoint(ENDPOINT_UNAVAILABLE, RoutingMethod.sharedLayer4), createTarget("t3", "a3", "i3", "default", true), + new Endpoint(ENDPOINT_UNHEALTHY, RoutingMethod.sharedLayer4), createTarget("t2", "a2", "i2", "default", true), + new Endpoint(ENDPOINT_INACTIVE, RoutingMethod.sharedLayer4), createTarget("t1", "a1", "i1", "default", false) + ), 42); + } + + private static RoutingTable.Target createTarget(String tenantName, String applicationName, String instanceName, String clusterName, boolean routingActive) { + ZoneId zone = ZoneId.from("prod", "us-north-1"); + ClusterSpec.Id cluster = ClusterSpec.Id.from(clusterName); + return RoutingTable.Target.create(ApplicationId.from(tenantName, applicationName, instanceName), cluster, zone, + List.of(new RoutingTable.Real("host", 8080, 1, routingActive))); + } + +} diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClientTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClientTest.java new file mode 100644 index 00000000000..2e923292280 --- /dev/null +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClientTest.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.routing.status; + +import com.yahoo.vespa.hosted.routing.mock.HttpClientMock; +import com.yahoo.vespa.hosted.routing.mock.HttpClientMock.JsonResponse; +import org.junit.Test; + +import java.net.URI; +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class RoutingStatusClientTest { + + @Test + public void client() { + String statusUrl = "http://host/routing/v2/status"; + HttpClientMock httpClient = new HttpClientMock(); + RoutingStatusClient client = new RoutingStatusClient(httpClient, URI.create("http://host")); + + // Nothing is inactive + httpClient.setResponse("GET", statusUrl, response(true)); + assertTrue(client.isActive("foo")); + + // Two upstreams are set inactive + httpClient.setResponse("GET", statusUrl, response(true, "bar", "foo")); + client.invalidateCache(); + assertFalse(client.isActive("foo")); + assertFalse(client.isActive("bar")); + assertTrue(client.isActive("baz")); + + // Bad response results in active status + client.invalidateCache(); + httpClient.setResponse("GET", statusUrl, badRequest("something went wrong")); + assertTrue(client.isActive("foo")); + + // Inactive zone overrides deployment status + client.invalidateCache(); + httpClient.setResponse("GET", statusUrl, response(false, "bar")); + assertFalse(client.isActive("foo")); + assertFalse(client.isActive("bar")); + + // Zone is active again. Fall back to reading deployment status + httpClient.setResponse("GET", statusUrl, response(true, "bar")); + client.invalidateCache(); + assertTrue(client.isActive("foo")); + assertFalse(client.isActive("bar")); + } + + private static JsonResponse badRequest(String message) { + return new JsonResponse("{\"message\":\"" + message + "\"}", 400); + } + + private static JsonResponse response(boolean zoneActive, String... inactiveUpstreams) { + String inactiveDeployments = "[" + Arrays.stream(inactiveUpstreams) + .map(d -> "{\"upstreamName\":\"" + d + "\"}") + .collect(Collectors.joining(",")) + "]"; + String json = "{\"inactiveDeployments\":" + inactiveDeployments + ",\"zoneActive\":" + zoneActive + "}"; + return new JsonResponse(json, 200); + } + +} diff --git a/routing-generator/src/test/resources/lbservices-config b/routing-generator/src/test/resources/lbservices-config new file mode 100644 index 00000000000..9c79a9b063f --- /dev/null +++ b/routing-generator/src/test/resources/lbservices-config @@ -0,0 +1,66 @@ +tenants.vespa.applications.music:prod:us-north-1:default.activeRotation true +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].dnsName "music.vespa.us-north-1.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].scope "zone" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[0].weight 1 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].dnsName "music.vespa.global.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].scope "global" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].weight 1 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].dnsName "music--vespa.global.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].scope "global" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].routingMethod "shared" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].weight 1 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].dnsName "rotation-02.vespa.global.routing" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].scope "global" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].weight 1 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].dnsName "rotation-02.vespa.global.routing" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].scope "global" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].routingMethod "shared" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].weight 1 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].hosts[0] "host1-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].hosts[1] "host2-default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].dnsName "use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].scope "application" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].weight 0 +tenants.vespa.applications.music:prod:us-north-1:beta.activeRotation true +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].hosts[0] "host3-beta" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].hosts[1] "host4-beta" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].dnsName "beta.music.vespa.us-north-1.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].scope "zone" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].weight 1 +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].hosts[0] "host3-beta" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].hosts[1] "host4-beta" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].dnsName "use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].clusterId "default" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].scope "application" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].routingMethod "sharedLayer4" +tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[1].weight 1 +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.activeRotation true +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].hosts[0] "routing-host1" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].hosts[1] "routing-host2" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].dnsName "routing.hosted-vespa.cd-us-west-1.hosted-vespa.oath.cloud" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].clusterId "default" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].scope "zone" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].routingMethod "sharedLayer4" +tenants.hosted-vespa.applications.routing:prod:cd-us-west-1:default.endpoints[0].weight 1 diff --git a/routing-generator/src/test/resources/nginx-health-multiple-tenants-application-metrics.json b/routing-generator/src/test/resources/nginx-health-multiple-tenants-application-metrics.json new file mode 100644 index 00000000000..4f537336849 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-multiple-tenants-application-metrics.json @@ -0,0 +1,18 @@ + +{"servers": { + "total": 9, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 4, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 5, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 6, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 7, "upstream": "qrs.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 8, "upstream": "donbot.vespa.us-east-2.prod", "name": "10.201.8.47:4080", "status": "up", "rise": 50604, "fall": 0, "type": "http", "port": 0}, + {"index": 9, "upstream": "donbot.vespa.us-east-2.prod", "name": "10.201.14.46:4080", "status": "down", "rise": 50834, "fall": 0, "type": "http", "port": 0}, + {"index": 10, "upstream": "appearing-in-routing.not.us-east-2.prod", "name": "10.201.14.50:4080", "status": "down", "rise": 50834, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-all-down.json b/routing-generator/src/test/resources/nginx-health-output-all-down.json new file mode 100644 index 00000000000..634bcf33148 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-all-down.json @@ -0,0 +1,11 @@ + +{"servers": { + "total": 4, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.115.68:4080", "status": "down", "rise": 1, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-all-up-but-other-down.json b/routing-generator/src/test/resources/nginx-health-output-all-up-but-other-down.json new file mode 100644 index 00000000000..c8e15bffb25 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-all-up-but-other-down.json @@ -0,0 +1,15 @@ + +{"servers": { + "total": 2, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 0, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 4, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 5, "upstream": "frog.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-all-up.json b/routing-generator/src/test/resources/nginx-health-output-all-up.json new file mode 100644 index 00000000000..a7a635d9ae3 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-all-up.json @@ -0,0 +1,9 @@ + +{"servers": { + "total": 2, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-policy-down.json b/routing-generator/src/test/resources/nginx-health-output-policy-down.json new file mode 100644 index 00000000000..347042b034a --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-policy-down.json @@ -0,0 +1,12 @@ + +{"servers": { + "total": 5, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 4, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-policy-up.json b/routing-generator/src/test/resources/nginx-health-output-policy-up.json new file mode 100644 index 00000000000..7dd015a7667 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-policy-up.json @@ -0,0 +1,12 @@ + +{"servers": { + "total": 5, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "up", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 4, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "down", "rise": 2, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output-stream.json b/routing-generator/src/test/resources/nginx-health-output-stream.json new file mode 100644 index 00000000000..9439e3f1ac4 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output-stream.json @@ -0,0 +1,12 @@ + +{"servers": { + "total": 4, + "generation": 1, + "http": [], + "stream": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.115.68:4080", "status": "down", "rise": 1, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-health-output.json b/routing-generator/src/test/resources/nginx-health-output.json new file mode 100644 index 00000000000..9c27a906a68 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-health-output.json @@ -0,0 +1,11 @@ + +{"servers": { + "total": 4, + "generation": 1, + "server": [ + {"index": 0, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 1, "upstream": "gateway.prod.music.vespa.us-east-2.prod", "name": "10.78.115.68:4080", "status": "up", "rise": 2, "fall": 0, "type": "http", "port": 0}, + {"index": 2, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.114.166:4080", "status": "down", "rise": 0, "fall": 1, "type": "http", "port": 0}, + {"index": 3, "upstream": "gateway.prod.music.vespa.us-east-2.prod-feed", "name": "10.78.115.68:4080", "status": "down", "rise": 1, "fall": 0, "type": "http", "port": 0} + ] +}} diff --git a/routing-generator/src/test/resources/nginx-updated.conf b/routing-generator/src/test/resources/nginx-updated.conf new file mode 100644 index 00000000000..5c90226cdb2 --- /dev/null +++ b/routing-generator/src/test/resources/nginx-updated.conf @@ -0,0 +1,56 @@ +map $ssl_preread_server_name $name { + beta.music.vespa.us-north-1.vespa.oath.cloud beta.music.vespa.us-north-1.prod; + endpoint1 i1.a1.t1.us-north-1.prod; + music.vespa.global.vespa.oath.cloud music.vespa.us-north-1.prod; + music.vespa.us-north-1.vespa.oath.cloud music.vespa.us-north-1.prod; + rotation-02.vespa.global.routing music.vespa.us-north-1.prod; + use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa; + '' default; +} + +upstream application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa { + server host1-default:4443 backup; + server host2-default:4443 backup; + server host3-beta:4443 weight=1; + server host4-beta:4443 weight=1; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa\r\n\r\n"; + random two; +} + +upstream beta.music.vespa.us-north-1.prod { + server host3-beta:4443; + server host4-beta:4443; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: beta.music.vespa.us-north-1.prod\r\n\r\n"; + random two; +} + +upstream i1.a1.t1.us-north-1.prod { + server host42:4443; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: i1.a1.t1.us-north-1.prod\r\n\r\n"; + random two; +} + +upstream music.vespa.us-north-1.prod { + server host1-default:4443; + server host2-default:4443; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: music.vespa.us-north-1.prod\r\n\r\n"; + random two; +} + +upstream default { + server localhost:4445; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4080; + check_http_send "GET /status.html HTTP/1.0\r\nHost: localhost\r\n\r\n"; +} + +server { + listen 443 reuseport; + listen [::]:443 reuseport; + proxy_pass $name; + ssl_preread on; + proxy_protocol on; +} diff --git a/routing-generator/src/test/resources/nginx.conf b/routing-generator/src/test/resources/nginx.conf new file mode 100644 index 00000000000..3064bde480a --- /dev/null +++ b/routing-generator/src/test/resources/nginx.conf @@ -0,0 +1,48 @@ +map $ssl_preread_server_name $name { + beta.music.vespa.us-north-1.vespa.oath.cloud beta.music.vespa.us-north-1.prod; + music.vespa.global.vespa.oath.cloud music.vespa.us-north-1.prod; + music.vespa.us-north-1.vespa.oath.cloud music.vespa.us-north-1.prod; + rotation-02.vespa.global.routing music.vespa.us-north-1.prod; + use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa; + '' default; +} + +upstream application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa { + server host1-default:4443 backup; + server host2-default:4443 backup; + server host3-beta:4443 weight=1; + server host4-beta:4443 weight=1; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: application-b53398a37399e67cf8c12017e0db764d145f9660.music.vespa\r\n\r\n"; + random two; +} + +upstream beta.music.vespa.us-north-1.prod { + server host3-beta:4443; + server host4-beta:4443; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: beta.music.vespa.us-north-1.prod\r\n\r\n"; + random two; +} + +upstream music.vespa.us-north-1.prod { + server host1-default:4443; + server host2-default:4443; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4082; + check_http_send "GET /status.html HTTP/1.0\r\nHost: music.vespa.us-north-1.prod\r\n\r\n"; + random two; +} + +upstream default { + server localhost:4445; + check interval=2000 fall=5 rise=2 timeout=3000 default_down=true type=http port=4080; + check_http_send "GET /status.html HTTP/1.0\r\nHost: localhost\r\n\r\n"; +} + +server { + listen 443 reuseport; + listen [::]:443 reuseport; + proxy_pass $name; + ssl_preread on; + proxy_protocol on; +} diff --git a/screwdriver.yaml b/screwdriver.yaml index 14d9902d335..57b7a3c05ae 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -202,6 +202,7 @@ jobs: - SAMPLE_APPS_DEPLOY_KEY - VESPA_DEPLOY_KEY - DOCKER_HUB_DEPLOY_KEY + - GHCR_DEPLOY_KEY environment: GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index f3d345a3552..87f06762169 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -10,15 +10,14 @@ fi readonly VESPA_VERSION=$1 -if curl -fsSL https://index.docker.io/v1/repositories/vespaengine/vespa/tags/$VESPA_VERSION &> /dev/null; then - echo "Container image docker.io/vespaengine/vespa:$VESPA_VERSION aldready exists." - exit 0 -fi - if [[ -z "$DOCKER_HUB_DEPLOY_KEY" ]]; then echo "Environment variable DOCKER_HUB_DEPLOY_KEY must be set, but is empty." exit 1 fi +if [[ -z "$GHCR_DEPLOY_KEY" ]]; then + echo "Environment variable GHCR_DEPLOY_KEY must be set, but is empty." + exit 1 +fi BUILD_DIR=$(mktemp -d) trap "rm -rf $BUILD_DIR" EXIT @@ -27,8 +26,27 @@ cd $BUILD_DIR git clone --depth 1 https://github.com/vespa-engine/docker-image cd docker-image +docker build --build-arg VESPA_VERSION=$VESPA_VERSION \ + --tag docker.io/vespaengine/vespa:$VESPA_VERSION --tag docker.io/vespaengine/vespa:latest \ + --tag ghcr.io/vespa-engine/vespa:$VESPA_VERSION --tag ghcr.io/vespa-engine/vespa:latest . + +# Push to Docker Hub docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" -docker build --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/vespaengine/vespa:$VESPA_VERSION --tag docker.io/vespaengine/vespa:latest . +if curl -fsSL https://index.docker.io/v1/repositories/vespaengine/vespa/tags/$VESPA_VERSION &> /dev/null; then + echo "Container image docker.io/vespaengine/vespa:$VESPA_VERSION aldready exists." +else + docker push docker.io/vespaengine/vespa:$VESPA_VERSION + docker push docker.io/vespaengine/vespa:latest +fi + +# Push to GitHub Container Registry +docker login --username aressem --password "$GHCR_DEPLOY_KEY" ghcr.io +JWT=$(curl -sSL -u aressem:$GHCR_DEPLOY_KEY "https://ghcr.io/token?service=ghcr.io&scope=repository:vespa-engine/vespa:pull" | jq -re '.token') +IMAGE_TAGS=$(curl -sSL -H "Authorization: Bearer $JWT" https://ghcr.io/v2/vespa-engine/vespa/tags/list | jq -re '.tags[]') +if grep $VESPA_VERSION <<< "$IMAGE_TAGS" &> /dev/null; then + echo "Container image ghcr.io/vespa-engine/vespa:$VESPA_VERSION aldready exists." +else + docker push ghcr.io/vespa-engine/vespa:$VESPA_VERSION + docker push ghcr.io/vespa-engine/vespa:latest +fi -docker push docker.io/vespaengine/vespa:$VESPA_VERSION -docker push docker.io/vespaengine/vespa:latest diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 7aaa821998a..3efa4360266 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -65,10 +65,10 @@ mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClo mvn $COMMON_MAVEN_OPTS --threads 1C -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy # Close with checks -mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO +mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.8:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO # Release if ok -mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO +mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.8:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO # Delete the GPG rings rm -rf $SD_SOURCE_DIR/screwdriver/deploy diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index af730f38018..c6a2a4532a3 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -48,7 +48,7 @@ writeFieldSets(vespalib::asciistream &os, struct FieldName { vespalib::string name; - FieldName(const std::vector<vespalib::string> & lines) + FieldName(const config::StringVector & lines) : name(ConfigParser::parse<vespalib::string>("name", lines)) { } @@ -64,8 +64,7 @@ getFieldId(vespalib::stringref name, const T &map) } // namespace -namespace search { -namespace index { +namespace search::index { const uint32_t Schema::UNKNOWN_FIELD_ID(std::numeric_limits<uint32_t>::max()); @@ -88,7 +87,7 @@ Schema::Field::Field(vespalib::stringref n, DataType dt, CollectionType ct, vesp } // XXX: Resource leak if exception is thrown. -Schema::Field::Field(const std::vector<vespalib::string> & lines) +Schema::Field::Field(const config::StringVector & lines) : _name(ConfigParser::parse<vespalib::string>("name", lines)), _dataType(schema::dataTypeFromName(ConfigParser::parse<vespalib::string>("datatype", lines))), _collectionType(schema::collectionTypeFromName(ConfigParser::parse<vespalib::string>("collectiontype", lines))) @@ -140,7 +139,7 @@ Schema::IndexField::IndexField(vespalib::stringref name, DataType dt, { } -Schema::IndexField::IndexField(const std::vector<vespalib::string> &lines) +Schema::IndexField::IndexField(const config::StringVector &lines) : Field(lines), _avgElemLen(ConfigParser::parse<int32_t>("averageelementlen", lines, 512)), _interleaved_features(ConfigParser::parse<bool>("interleavedfeatures", lines, false)) @@ -181,11 +180,11 @@ Schema::IndexField::operator!=(const IndexField &rhs) const _interleaved_features != rhs._interleaved_features; } -Schema::FieldSet::FieldSet(const std::vector<vespalib::string> & lines) : +Schema::FieldSet::FieldSet(const config::StringVector & lines) : _name(ConfigParser::parse<vespalib::string>("name", lines)), _fields() { - std::vector<FieldName> fn = ConfigParser::parseArray<FieldName>("field", lines); + std::vector<FieldName> fn = ConfigParser::parseArray<std::vector<FieldName>>("field", lines); for (size_t i = 0; i < fn.size(); ++i) { _fields.push_back(fn[i].name); } @@ -238,16 +237,16 @@ Schema::loadFromFile(const vespalib::string & fileName) LOG(warning, "Could not open input file '%s' as part of loadFromFile()", fileName.c_str()); return false; } - std::vector<vespalib::string> lines; + config::StringVector lines; std::string tmpLine; while (file) { getline(file, tmpLine); lines.push_back(tmpLine); } - _indexFields = ConfigParser::parseArray<IndexField>("indexfield", lines); - _attributeFields = ConfigParser::parseArray<AttributeField>("attributefield", lines); - _summaryFields = ConfigParser::parseArray<SummaryField>("summaryfield", lines); - _fieldSets = ConfigParser::parseArray<FieldSet>("fieldset", lines); + _indexFields = ConfigParser::parseArray<std::vector<IndexField>>("indexfield", lines); + _attributeFields = ConfigParser::parseArray<std::vector<AttributeField>>("attributefield", lines); + _summaryFields = ConfigParser::parseArray<std::vector<SummaryField>>("summaryfield", lines); + _fieldSets = ConfigParser::parseArray<std::vector<FieldSet>>("fieldset", lines); _importedAttributeFields.clear(); // NOTE: these are not persisted to disk _indexIds.clear(); for (size_t i(0), m(_indexFields.size()); i < m; i++) { @@ -290,18 +289,13 @@ Schema::saveToFile(const vespalib::string & fileName) const FastOS_File s; s.OpenReadWrite(fileName.c_str()); if (!s.IsOpened()) { - LOG(warning, - "Could not open schema file '%s' for fsync", - fileName.c_str()); + LOG(warning, "Could not open schema file '%s' for fsync", fileName.c_str()); return false; } else { if (!s.Sync()) { - LOG(warning, - "Could not fsync schema file '%s'", - fileName.c_str()); + LOG(warning, "Could not fsync schema file '%s'", fileName.c_str()); return false; } - s.Close(); } return true; } @@ -584,5 +578,4 @@ Schema::empty() const _importedAttributeFields.empty(); } -} // namespace search::index -} // namespace search +} diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index f86bb3068c5..3a9bcbdd904 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -3,10 +3,9 @@ #pragma once #include "datatype.h" -#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/types.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/ptrholder.h> -#include <vector> namespace vespalib { class asciistream; } namespace search::index { @@ -20,7 +19,6 @@ class Schema public: using UP = std::unique_ptr<Schema>; using SP = std::shared_ptr<Schema>; - using PH = vespalib::PtrHolder<Schema>; using DataType = schema::DataType; using CollectionType = schema::CollectionType; @@ -45,7 +43,7 @@ public: /** * Create this field based on the given config lines. **/ - Field(const std::vector<vespalib::string> & lines); + Field(const config::StringVector & lines); Field(const Field &) noexcept; Field & operator = (const Field &) noexcept; Field(Field &&) noexcept; @@ -92,7 +90,7 @@ public: /** * Create this index field based on the given config lines. **/ - IndexField(const std::vector<vespalib::string> &lines); + IndexField(const config::StringVector &lines); IndexField &setAvgElemLen(uint32_t avgElemLen) { _avgElemLen = avgElemLen; return *this; } IndexField &set_interleaved_features(bool value) { @@ -133,7 +131,7 @@ public: /** * Create this field collection based on the given config lines. **/ - FieldSet(const std::vector<vespalib::string> & lines); + FieldSet(const config::StringVector & lines); ~FieldSet(); @@ -143,8 +141,9 @@ public: } const vespalib::string &getName() const { return _name; } - const std::vector<vespalib::string> &getFields() const - { return _fields; } + const std::vector<vespalib::string> &getFields() const { + return _fields; + } bool operator==(const FieldSet &rhs) const; bool operator!=(const FieldSet &rhs) const; diff --git a/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h b/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h index 7683ca771b8..dd24480f689 100644 --- a/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h +++ b/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/config/helper/legacysubscriber.h> -#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/helper/legacysubscriber.hpp> namespace search { @@ -14,7 +13,7 @@ class SubscriptionProxyNg : public config::IFetcherCallback<CFG> private: ME &_target; Method _method; - config::LegacySubscriber *_subscriber; + std::unique_ptr<config::LegacySubscriber> _subscriber; vespalib::string _cfgId; SubscriptionProxyNg(const SubscriptionProxyNg&); @@ -24,7 +23,7 @@ public: SubscriptionProxyNg(ME &target, Method method) : _target(target), _method(method), - _subscriber(NULL), + _subscriber(), _cfgId("") { } @@ -35,26 +34,25 @@ public: return _cfgId.c_str(); } void subscribe(const char *configId) { - if (_subscriber != NULL) { - if (configId != NULL && strcmp(configId, _subscriber->id().c_str()) == 0) + if (_subscriber) { + if (configId != nullptr && strcmp(configId, _subscriber->id().c_str()) == 0) { return; // same id; ignore } else { unsubscribe(); } } - if (configId != NULL && configId[0] != '\0') { + if (configId != nullptr && configId[0] != '\0') { _cfgId = configId; - _subscriber = new config::LegacySubscriber(); + _subscriber = std::make_unique<config::LegacySubscriber>(); _subscriber->subscribe<CFG>(configId, this); } } void unsubscribe() { - delete _subscriber; - _subscriber = NULL; + _subscriber.reset(); _cfgId = ""; } - virtual void configure(std::unique_ptr<CFG> cfg) override { + void configure(std::unique_ptr<CFG> cfg) override { (_target.*_method)(*cfg); } }; diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 214c57557bc..483cc3f2792 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -39,6 +39,7 @@ #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/config/subscription/sourcespec.h> #include <vespa/log/log.h> LOG_SETUP("persistenceconformance_test"); @@ -178,6 +179,12 @@ private: MockSharedThreadingService _shared_service; storage::spi::dummy::DummyBucketExecutor _bucketExecutor; + static std::shared_ptr<ProtonConfig> make_proton_config() { + ProtonConfigBuilder proton_config; + proton_config.indexing.optimize = ProtonConfigBuilder::Indexing::Optimize::LATENCY; + return std::make_shared<ProtonConfig>(proton_config); + } + public: DocumentDBFactory(const vespalib::string &baseDir, int tlsListenPort); ~DocumentDBFactory() override; @@ -193,10 +200,10 @@ public: fileCfg.saveConfig(*snapshot, 1); } config::DirSpec spec(inputCfg + "/config-1"); - TuneFileDocumentDB::SP tuneFileDocDB(new TuneFileDocumentDB()); + auto tuneFileDocDB = std::make_shared<TuneFileDocumentDB>(); DocumentDBConfigHelper mgr(spec, docType.getName()); auto b = std::make_shared<BootstrapConfig>(1, factory.getTypeCfg(), factory.getTypeRepo(), - std::make_shared<ProtonConfig>(), + make_proton_config(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), tuneFileDocDB, HwInfo()); diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index a52c9ec2fb6..5ec1794c1b0 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -4,7 +4,6 @@ #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> -#include <vespa/config/config.h> #include <vespa/config/helper/legacy.h> #include <vespa/config/common/configcontext.h> #include <vespa/config/common/exceptions.h> @@ -21,6 +20,7 @@ #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/plugin/setup.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/fastos/app.h> #include <optional> diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index 9819a1d50af..9f6254f9baa 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/config.h> #include <vespa/config/print/fileconfigwriter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/document.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/destinationsession.h> -#include <vespa/messagebus/protocolset.h> #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 6e8223399e6..c5c50ff596d 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -12,13 +12,13 @@ #include <openssl/evp.h> #include <cassert> #include <getopt.h> +#include <vector> #include <vespa/log/log.h> LOG_SETUP("vespa-gen-testdocs"); typedef vespalib::hash_set<vespalib::string> StringSet; typedef std::vector<vespalib::string> StringArray; -typedef std::shared_ptr<StringArray> StringArraySP; using namespace vespalib::alloc; using vespalib::string; @@ -38,10 +38,7 @@ void usageHeader() { using std::cerr; - cerr << - "vespa-gen-testdocs version 0.0\n" - "\n" - "USAGE:\n"; + cerr << "vespa-gen-testdocs version 0.0\n\nUSAGE:\n"; } string @@ -71,8 +68,7 @@ splitArg(const string &arg) } void -shafile(const string &baseDir, - const string &file) +shafile(const string &baseDir, const string &file) { unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int digest_len = 0; @@ -98,7 +94,6 @@ shafile(const string &baseDir, EVP_DigestUpdate(md_ctx.get(), buf.get(), thistime); remainder -= thistime; } - f.Close(); EVP_DigestFinal_ex(md_ctx.get(), &digest[0], &digest_len); assert(digest_len > 0u && digest_len <= EVP_MAX_MD_SIZE); for (unsigned int i = 0; i < digest_len; ++i) { @@ -106,10 +101,7 @@ shafile(const string &baseDir, os.fill('0'); os << std::hex << static_cast<unsigned int>(digest[i]); } - LOG(info, - "SHA256(%s)= %s", - file.c_str(), - os.str().c_str()); + LOG(info, "SHA256(%s)= %s", file.c_str(), os.str().c_str()); } class StringGenerator @@ -119,14 +111,9 @@ class StringGenerator public: StringGenerator(vespalib::Rand48 &rnd); - void - rand_string(string &res, uint32_t minLen, uint32_t maxLen); + void rand_string(string &res, uint32_t minLen, uint32_t maxLen); - void - rand_unique_array(StringArray &res, - uint32_t minLen, - uint32_t maxLen, - uint32_t size); + void rand_unique_array(StringArray &res, uint32_t minLen, uint32_t maxLen, uint32_t size); }; @@ -590,7 +577,8 @@ DocumentGenerator::generate(uint32_t docMin, uint32_t docIdLimit, } } f.Flush(); - f.Close(); + bool close_ok = f.Close(); + assert(close_ok); LOG(info, "Calculating sha256 for %s", feedFileName.c_str()); shafile(baseDir, feedFileName); } diff --git a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp index c1dbe9b2bd2..beb1e0bd6bc 100644 --- a/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp +++ b/searchcore/src/apps/vespa-redistribute-bm/vespa_redistribute_bm.cpp @@ -465,7 +465,6 @@ App::usage() "USAGE:\n"; std::cerr << "vespa-redistribute-bm\n" - "[--async-apply-bucket-diff]\n" "[--bucket-db-stripe-bits bits]\n" "[--client-threads threads]\n" "[--distributor-merge-busy-wait distributor-merge-busy-wait]\n" @@ -502,7 +501,6 @@ App::get_options() const char *opt_argument = nullptr; int long_opt_index = 0; static struct option long_opts[] = { - { "async-apply-bucket-diff", 0, nullptr, 0 }, { "bucket-db-stripe-bits", 1, nullptr, 0 }, { "client-threads", 1, nullptr, 0 }, { "distributor-merge-busy-wait", 1, nullptr, 0 }, @@ -533,7 +531,6 @@ App::get_options() { nullptr, 0, nullptr, 0 } }; enum longopts_enum { - LONGOPT_ASYNC_APPLY_BUCKET_DIFF, LONGOPT_BUCKET_DB_STRIPE_BITS, LONGOPT_CLIENT_THREADS, LONGOPT_DISTRIBUTOR_MERGE_BUSY_WAIT, @@ -568,9 +565,6 @@ App::get_options() switch (c) { case 0: switch(long_opt_index) { - case LONGOPT_ASYNC_APPLY_BUCKET_DIFF: - _bm_params.set_async_apply_bucket_diff(true); - break; case LONGOPT_BUCKET_DB_STRIPE_BITS: _bm_params.set_bucket_db_stripe_bits(atoi(opt_argument)); break; diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 1851455e321..b9e3549053a 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -35,6 +35,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/config/subscription/sourcespec.h> using namespace cloud::config::filedistribution; using namespace document; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 77f7cf4d8ed..7ba3e0b8240 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -6,7 +6,6 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/fastos/file.h> #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> #include <vespa/searchcore/proton/common/statusreport.h> @@ -35,6 +34,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/config/subscription/sourcespec.h> #include <iostream> using namespace cloud::config::filedistribution; diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index a59b3e9bc6f..fc8bd474813 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -14,11 +14,11 @@ using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; struct Fixture { ProtonConfig cfg; - Fixture(uint32_t baseLineIndexingThreads = 2, uint32_t master_task_limit = 2000, uint32_t task_limit = 500) + Fixture(uint32_t baseLineIndexingThreads = 2, uint32_t master_task_limit = 2000, int32_t task_limit = 500) : cfg(makeConfig(baseLineIndexingThreads, master_task_limit, task_limit)) { } - ProtonConfig makeConfig(uint32_t baseLineIndexingThreads, uint32_t master_task_limit, uint32_t task_limit) { + ProtonConfig makeConfig(uint32_t baseLineIndexingThreads, uint32_t master_task_limit, int32_t task_limit) { ProtonConfigBuilder builder; builder.indexing.threads = baseLineIndexingThreads; builder.indexing.tasklimit = task_limit; @@ -56,6 +56,15 @@ TEST_F("require that task limits are set", Fixture) auto tcfg = f.make(24); EXPECT_EQUAL(2000u, tcfg.master_task_limit()); EXPECT_EQUAL(500u, tcfg.defaultTaskLimit()); + EXPECT_TRUE(tcfg.is_task_limit_hard()); +} + +TEST_F("require that negative task limit makes it soft", Fixture(2, 3000, -700)) +{ + auto tcfg = f.make(24); + EXPECT_EQUAL(3000u, tcfg.master_task_limit()); + EXPECT_EQUAL(700u, tcfg.defaultTaskLimit()); + EXPECT_FALSE(tcfg.is_task_limit_hard()); } namespace { diff --git a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp index 6eab2c5bf3d..c814bdf3f37 100644 --- a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp +++ b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for diskindexcleaner. -#include <vespa/searchcorespi/index/activediskindexes.h> +#include <vespa/searchcorespi/index/disk_indexes.h> #include <vespa/searchcorespi/index/diskindexcleaner.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/fastos/file.h> @@ -90,8 +90,8 @@ void createIndexes() { void Test::requireThatAllIndexesOlderThanLastFusionIsRemoved() { createIndexes(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(3u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); @@ -101,17 +101,17 @@ void Test::requireThatAllIndexesOlderThanLastFusionIsRemoved() { void Test::requireThatIndexesInUseAreNotRemoved() { createIndexes(); - ActiveDiskIndexes active_indexes; - active_indexes.setActive(index_dir + "/index.fusion.1"); - active_indexes.setActive(index_dir + "/index.flush.2"); - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + disk_indexes.setActive(index_dir + "/index.fusion.1", 0); + disk_indexes.setActive(index_dir + "/index.flush.2", 0); + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_TRUE(contains(indexes, "index.fusion.1")); EXPECT_TRUE(contains(indexes, "index.flush.2")); - active_indexes.notActive(index_dir + "/index.fusion.1"); - active_indexes.notActive(index_dir + "/index.flush.2"); - DiskIndexCleaner::clean(index_dir, active_indexes); + disk_indexes.notActive(index_dir + "/index.fusion.1"); + disk_indexes.notActive(index_dir + "/index.flush.2"); + DiskIndexCleaner::clean(index_dir, disk_indexes); indexes = readIndexes(); EXPECT_TRUE(!contains(indexes, "index.fusion.1")); EXPECT_TRUE(!contains(indexes, "index.flush.2")); @@ -120,8 +120,8 @@ void Test::requireThatIndexesInUseAreNotRemoved() { void Test::requireThatInvalidFlushIndexesAreRemoved() { createIndexes(); FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(2u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); @@ -131,8 +131,8 @@ void Test::requireThatInvalidFlushIndexesAreRemoved() { void Test::requireThatInvalidFusionIndexesAreRemoved() { createIndexes(); FastOS_File((index_dir + "/index.fusion.2/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(4u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.1")); @@ -144,8 +144,8 @@ void Test::requireThatInvalidFusionIndexesAreRemoved() { void Test::requireThatRemoveDontTouchNewIndexes() { createIndexes(); FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::removeOldIndexes(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::removeOldIndexes(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(3u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); diff --git a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp index 313f3e9a270..c0d94ba6376 100644 --- a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp @@ -57,9 +57,10 @@ public: ~ResourceUsageTrackerTest(); - void notify(double disk_usage, double memory_usage) + void notify(double disk_usage, double memory_usage, double transient_disk_usage = 0.0, double transient_memory_usage = 0.0) { - _notifier.notify(DiskMemUsageState({ 0.8, disk_usage }, { 0.8, memory_usage })); + _notifier.notify(DiskMemUsageState({ 0.8, disk_usage }, { 0.8, memory_usage }, + transient_disk_usage, transient_memory_usage)); } ResourceUsage get_usage() { return _listener->get_usage(); } @@ -77,6 +78,15 @@ TEST_F(ResourceUsageTrackerTest, resource_usage_is_forwarded_to_listener) EXPECT_EQ(ResourceUsage(0.75, 0.25), get_usage()); } +TEST_F(ResourceUsageTrackerTest, transient_resource_usage_is_subtracted_from_absolute_usage) +{ + auto register_guard = _tracker->set_listener(*_listener); + notify(0.8, 0.5, 0.4, 0.2); + EXPECT_EQ(ResourceUsage(0.4, 0.3), get_usage()); + notify(0.8, 0.5, 0.9, 0.6); + EXPECT_EQ(ResourceUsage(0.0, 0.0), get_usage()); +} + TEST_F(ResourceUsageTrackerTest, forwarding_depends_on_register_guard) { auto register_guard = _tracker->set_listener(*_listener); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 6d3eaa30263..3c9e9fc3a64 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -69,7 +69,7 @@ struct ConfigTestFixture { BucketspacesConfigBuilder bucketspacesBuilder; map<std::string, DoctypeFixture::UP> dbConfig; ConfigSet set; - IConfigContext::SP context; + std::shared_ptr<IConfigContext> context; int idcounter; ConfigTestFixture(const std::string & id) diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index aa819d08b58..85f8e8171a8 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -7,12 +7,12 @@ #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> #include <vespa/config-summarymap.h> +#include <vespa/config-bucketspaces.h> #include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> -#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/document_db_config_owner.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> @@ -21,14 +21,12 @@ #include <vespa/searchcore/proton/server/i_proton_disk_layout.h> #include <vespa/searchcore/proton/server/threading_service_config.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/searchcore/config/config-onnx-models.h> -#include <vespa/vespalib/gtest/gtest.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/config-bucketspaces.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace config; using namespace proton; diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index ce85517ee09..db32c1e77c4 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -121,18 +121,22 @@ TEST_F(DiskMemUsageFilterTest, both_disk_limit_and_memory_limit_can_be_reached) "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } -TEST_F(DiskMemUsageFilterTest, transient_disk_usage_is_tracked_in_usage_state_and_metrics) +TEST_F(DiskMemUsageFilterTest, transient_and_non_transient_disk_usage_tracked_in_usage_state_and_metrics) { - _filter.set_transient_resource_usage({40, 0}); - EXPECT_EQ(0.4, _filter.usageState().transient_disk_usage()); - EXPECT_EQ(0.4, _filter.get_metrics().get_transient_disk_usage()); + _filter.set_transient_resource_usage({15, 0}); + EXPECT_DOUBLE_EQ(0.15, _filter.usageState().transient_disk_usage()); + EXPECT_DOUBLE_EQ(0.15, _filter.get_metrics().transient_disk_usage()); + EXPECT_DOUBLE_EQ(0.05, _filter.usageState().non_transient_disk_usage()); + EXPECT_DOUBLE_EQ(0.05, _filter.get_metrics().non_transient_disk_usage()); } -TEST_F(DiskMemUsageFilterTest, transient_memory_usage_is_tracked_in_usage_state_and_metrics) +TEST_F(DiskMemUsageFilterTest, transient_and_non_transient_memory_usage_tracked_in_usage_state_and_metrics) { - _filter.set_transient_resource_usage({0, 200}); - EXPECT_EQ(0.2, _filter.usageState().transient_memory_usage()); - EXPECT_EQ(0.2, _filter.get_metrics().get_transient_memory_usage()); + _filter.set_transient_resource_usage({0, 100}); + EXPECT_DOUBLE_EQ(0.1, _filter.usageState().transient_memory_usage()); + EXPECT_DOUBLE_EQ(0.1, _filter.get_metrics().transient_memory_usage()); + EXPECT_DOUBLE_EQ(0.2, _filter.usageState().non_transient_memory_usage()); + EXPECT_DOUBLE_EQ(0.2, _filter.get_metrics().non_transient_memory_usage()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp index 60240e06fc4..8ba02875dc0 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp @@ -10,30 +10,46 @@ using proton::DiskMemUsageState; using proton::ResourceUsageState; bool -expect_metrics(double disk_usage, double disk_utilization, double memory_usage, double memory_utilization, const DiskMemUsageMetrics &dm_metrics) +expect_metrics(double disk_usage, double disk_utilization, double transient_disk, double non_transient_disk, + double memory_usage, double memory_utilization, double transient_memory, double non_transient_memory, + const DiskMemUsageMetrics &dm_metrics) { bool result = true; - EXPECT_DOUBLE_EQ(disk_usage, dm_metrics.get_disk_usage()) << (result = false, ""); - EXPECT_DOUBLE_EQ(disk_utilization, dm_metrics.get_disk_utilization()) << (result = false, ""); - EXPECT_DOUBLE_EQ(memory_usage, dm_metrics.get_memory_usage()) << (result = false, ""); - EXPECT_DOUBLE_EQ(memory_utilization, dm_metrics.get_memory_utilization()) << (result = false, ""); + EXPECT_DOUBLE_EQ(disk_usage, dm_metrics.total_disk_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(disk_utilization, dm_metrics.total_disk_utilization()) << (result = false, ""); + EXPECT_DOUBLE_EQ(transient_disk, dm_metrics.transient_disk_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(non_transient_disk, dm_metrics.non_transient_disk_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_usage, dm_metrics.total_memory_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_utilization, dm_metrics.total_memory_utilization()) << (result = false, ""); + EXPECT_DOUBLE_EQ(transient_memory, dm_metrics.transient_memory_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(non_transient_memory, dm_metrics.non_transient_memory_usage()) << (result = false, ""); return result; } TEST(DiskMemUsageMetricsTest, default_value_is_zero) { DiskMemUsageMetrics dm_metrics; - EXPECT_TRUE(expect_metrics(0.0, 0.0, 0.0, 0.0, dm_metrics)); + EXPECT_TRUE(expect_metrics(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, dm_metrics)); } TEST(DiskMemUsageMetricsTest, merging_uses_max) { - DiskMemUsageMetrics dm_metrics({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); - EXPECT_TRUE(expect_metrics(0.4, 0.8, 0.3, 0.6, dm_metrics)); - dm_metrics.merge({ResourceUsageState(0.4, 0.4), ResourceUsageState(0.5, 0.4)}); - EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); - dm_metrics.merge({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); - EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); + DiskMemUsageMetrics dm_metrics({ResourceUsageState(0.5, 0.4), + ResourceUsageState(0.5, 0.3), 0.1, 0.05}); + EXPECT_TRUE(expect_metrics(0.4, 0.8, 0.1, 0.3, + 0.3, 0.6, 0.05, 0.25, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.4, 0.4), + ResourceUsageState(0.3, 0.3), 0.1, 0.05}); + EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.1, 0.3, + 0.3, 1.0, 0.05, 0.25, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.5, 0.45), + ResourceUsageState(0.5, 0.35), 0.1, 0.05}); + EXPECT_TRUE(expect_metrics(0.45, 1.0, 0.1, 0.35, + 0.35, 1.0, 0.05, 0.3, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.5, 0.4), + ResourceUsageState(0.5, 0.3), 0.15, 0.1}); + EXPECT_TRUE(expect_metrics(0.45, 1.0, 0.15, 0.35, + 0.35, 1.0, 0.10, 0.3, dm_metrics)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index bb0c4eeb282..e01d90f5eed 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchcore/proton/server/feedstates.h> #include <vespa/searchcore/proton/server/ireplayconfig.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> +#include <vespa/searchcore/proton/server/replay_throttling_policy.h> #include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/test/dummy_feed_view.h> #include <vespa/searchlib/common/serialnum.h> @@ -71,6 +72,7 @@ struct Fixture MemoryConfigStore config_store; bucketdb::BucketDBOwner _bucketDB; bucketdb::BucketDBHandler _bucketDBHandler; + ReplayThrottlingPolicy _replay_throttling_policy; MyIncSerialNum _inc_serial_num; ReplayTransactionLogState state; @@ -86,8 +88,9 @@ Fixture::Fixture() config_store(), _bucketDB(), _bucketDBHandler(_bucketDB), + _replay_throttling_policy({}), _inc_serial_num(9u), - state("doctypename", feed_view_ptr, _bucketDBHandler, replay_config, config_store, _inc_serial_num) + state("doctypename", feed_view_ptr, _bucketDBHandler, replay_config, config_store, _replay_throttling_policy, _inc_serial_num) { } Fixture::~Fixture() = default; diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index 7ff59a9f41a..d544f1cb1ab 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -11,6 +11,7 @@ ProtonConfig::Flush::Memory getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize, double conservativeMemoryLimitFactor = 0.5, double conservativeDiskLimitFactor = 0.6, + double high_watermark_factor = 0.9, double lowWatermarkFactor = 0.8) { ProtonConfig::Flush::Memory result; @@ -19,6 +20,7 @@ getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize, result.maxtlssize = maxTlsSize; result.conservative.memorylimitfactor = conservativeMemoryLimitFactor; result.conservative.disklimitfactor = conservativeDiskLimitFactor; + result.conservative.highwatermarkfactor = high_watermark_factor; result.conservative.lowwatermarkfactor = lowWatermarkFactor; return result; } @@ -32,14 +34,16 @@ getDefaultConfig() ResourceUsageState aboveLimit() { - return ResourceUsageState(0.7, 0.8); + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + return ResourceUsageState(0.7, 0.64); } ResourceUsageState belowLimit() { - // This is still over default low watermark of 0.56 (0.7 * 0.8) - return ResourceUsageState(0.7, 0.6); + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + // This is still over the low watermark limit of 0.56 (0.7 * 0.8 (factor)). + return ResourceUsageState(0.7, 0.62); } const HwInfo::Memory defaultMemory(8_Gi); @@ -175,6 +179,24 @@ TEST_F("require that last config if remembered when setting new disk/memory usag TEST_DO(f.assertStrategyConfig(6, 3, 18)); } +TEST_F("Use conservative settings when above high watermark for disk usage", Fixture) +{ + // The high watermark limit is 0.63 (0.7 * 0.9 (factor)). + f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.62), belowLimit()); + TEST_DO(f.assertStrategyConfig(4, 1, 20)); + f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.64), belowLimit()); + TEST_DO(f.assertStrategyConfig(4, 1, 12)); +} + +TEST_F("Use conservative settings when above high watermark for memory usage", Fixture) +{ + // The high watermark limit is 0.54 (0.6 * 0.9 (factor)). + f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.53)); + TEST_DO(f.assertStrategyConfig(4, 1, 20)); + f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.55)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); +} + TEST_F("require that we must go below low watermark for disk usage before using normal tls size value again", Fixture) { f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.8), belowLimit()); diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp index 4a3466f1a51..3ff10b19164 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.cpp @@ -6,8 +6,7 @@ namespace search::bmcluster { BmClusterParams::BmClusterParams() - : _async_apply_bucket_diff(), - _bucket_db_stripe_bits(4), + : _bucket_db_stripe_bits(4), _disable_queue_limits_for_chained_merges(false), // Same default as in stor-server.def _distributor_merge_busy_wait(10), // Same default as stor_distributormanager.def _distributor_stripes(0), diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h index 36b4c22f6a8..d365a28b0b6 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster_params.h @@ -13,7 +13,6 @@ namespace search::bmcluster { */ class BmClusterParams { - std::optional<bool> _async_apply_bucket_diff; uint32_t _bucket_db_stripe_bits; bool _disable_queue_limits_for_chained_merges; uint32_t _distributor_merge_busy_wait; @@ -45,7 +44,6 @@ class BmClusterParams public: BmClusterParams(); ~BmClusterParams(); - const std::optional<bool>& get_async_apply_bucket_diff() const noexcept { return _async_apply_bucket_diff; } uint32_t get_bucket_db_stripe_bits() const { return _bucket_db_stripe_bits; } bool get_disable_queue_limits_for_chained_merges() const noexcept { return _disable_queue_limits_for_chained_merges; } uint32_t get_distributor_merge_busy_wait() const { return _distributor_merge_busy_wait; } @@ -75,7 +73,6 @@ public: bool needs_distributor() const { return _enable_distributor || _use_document_api; } bool needs_message_bus() const { return _use_message_bus || _use_document_api; } bool needs_service_layer() const { return _enable_service_layer || _enable_distributor || _use_storage_chain || _use_message_bus || _use_document_api; } - void set_async_apply_bucket_diff(bool value) { _async_apply_bucket_diff = value; } void set_bucket_db_stripe_bits(uint32_t value) { _bucket_db_stripe_bits = value; } void set_disable_queue_limits_for_chained_merges(bool value) { _disable_queue_limits_for_chained_merges = value; } void set_distributor_merge_busy_wait(uint32_t value) { _distributor_merge_busy_wait = value; } diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index db2060bacf7..3587e8008f2 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -382,9 +382,6 @@ struct ServiceLayerConfigSet : public StorageConfigSet stor_bucket_init(), stor_visitor() { - if (params.get_async_apply_bucket_diff().has_value()) { - stor_filestor.asyncApplyBucketDiff = params.get_async_apply_bucket_diff().value(); - } stor_filestor.numResponseThreads = params.get_response_threads(); stor_filestor.numNetworkThreads = params.get_rpc_network_threads(); stor_filestor.useAsyncMessageHandlingOnSchedule = params.get_use_async_message_handling_on_schedule(); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 34530afe9f9..808535924f1 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -81,6 +81,10 @@ flush.memory.conservative.memorylimitfactor double default=0.5 ## In this case this factor is multiplied with 'maxtlssize' to calculate a conservative value to use instead. flush.memory.conservative.disklimitfactor double default=0.5 +## The factor used to multiply with the resource limits for disk / memory to find the high +## watermark indicating when to from normal into conservative mode for the flush strategy. +flush.memory.conservative.highwatermarkfactor double default=0.95 + ## The factor used to multiply with the resource limits for disk / memory to find the low ## watermark indicating when to go back from conservative to normal mode for the flush strategy. flush.memory.conservative.lowwatermarkfactor double default=0.9 @@ -442,7 +446,7 @@ initialize.threads int default = 0 ## Portion of max address space used in components in attribute vectors ## before put and update operations in feed is blocked. -writefilter.attribute.address_space_limit double default = 0.9 +writefilter.attribute.address_space_limit double default = 0.92 ## Portion of physical memory that can be resident memory in anonymous mapping ## by the proton process before put and update portion of feed is blocked. @@ -543,3 +547,11 @@ tensor_implementation enum {TENSOR_ENGINE, FAST_VALUE} default = FAST_VALUE ## Whether to report issues back to the container via protobuf field forward_issues bool default = true + +## Chooses the throttling policy used to control the window size +## of the SharedOperationThrottler component used by the transaction log replay feed state. +replay_throttling_policy.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED +## Only used if replay_throttling_policy.type == DYNAMIC: +replay_throttling_policy.min_window_size int default=100 +replay_throttling_policy.max_window_size int default=10000 +replay_throttling_policy.window_size_increment int default=20 diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 66be0737fe9..713582bf1ce 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -193,7 +193,7 @@ AttributeInitializer::loadAttribute(const AttributeVectorSP &attr, assert(attr->hasLoadData()); vespalib::Timer timer; EventLogger::loadAttributeStart(_documentSubDbName, attr->getName()); - if (!attr->load(&_executor)) { + if (!attr->load(&_shared_executor)) { LOG(warning, "Could not load attribute vector '%s' from disk. Returning empty attribute vector", attr->getBaseFileName().c_str()); return false; @@ -235,13 +235,13 @@ AttributeInitializer::AttributeInitializer(const std::shared_ptr<AttributeDirect const AttributeSpec &spec, uint64_t currentSerialNum, const IAttributeFactory &factory, - vespalib::Executor & executor) + vespalib::Executor& shared_executor) : _attrDir(attrDir), _documentSubDbName(documentSubDbName), _spec(spec), _currentSerialNum(currentSerialNum), _factory(factory), - _executor(executor), + _shared_executor(shared_executor), _header(), _header_ok(false) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h index 7b5d968353a..78a798c929e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h @@ -30,7 +30,7 @@ private: const AttributeSpec _spec; const uint64_t _currentSerialNum; const IAttributeFactory &_factory; - vespalib::Executor &_executor; + vespalib::Executor &_shared_executor; std::unique_ptr<const search::attribute::AttributeHeader> _header; bool _header_ok; @@ -48,7 +48,7 @@ private: public: AttributeInitializer(const std::shared_ptr<AttributeDirectory> &attrDir, const vespalib::string &documentSubDbName, const AttributeSpec &spec, uint64_t currentSerialNum, const IAttributeFactory &factory, - vespalib::Executor & executor); + vespalib::Executor& shared_executor); ~AttributeInitializer(); AttributeInitializerResult init() const; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 52b367fd14b..f0a23f1038e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -14,10 +14,10 @@ #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/tensor/prepare_result.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/idestructorcallback.h> -#include <vespa/vespalib/util/threadexecutor.h> #include <future> #include <vespa/log/log.h> @@ -29,6 +29,7 @@ using namespace search; using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; using search::attribute::ImportedAttributeVector; using search::tensor::PrepareResult; +using vespalib::CpuUsage; using vespalib::GateCallback; using vespalib::ISequencedTaskExecutor; @@ -631,7 +632,7 @@ AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentI wc.consider_build_field_paths(doc); auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc, doc); auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, onWriteDone); - _shared_executor.execute(std::move(prepare_task)); + _shared_executor.execute(CpuUsage::wrap(std::move(prepare_task), CpuUsage::Category::WRITE)); _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(complete_task)); } else { if (allAttributes || wc.hasStructFieldAttribute()) { @@ -781,7 +782,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, onWriteDone); LOG(debug, "About to handle assign update as two phase put for docid %u in attribute vector '%s'", lid, attrp->getName().c_str()); - _shared_executor.execute(std::move(prepare_task)); + _shared_executor.execute(CpuUsage::wrap(std::move(prepare_task), CpuUsage::Category::WRITE)); _attributeFieldWriter.executeTask(itr->second.executor_id, std::move(complete_task)); } else { args[itr->second.executor_id.getId()]->_updates.emplace_back(attrp, &fupd); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index f43aab0f385..a5907233c4b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -25,7 +25,7 @@ private: using FieldValue = document::FieldValue; const IAttributeManager::SP _mgr; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; - vespalib::ThreadExecutor& _shared_executor; + vespalib::Executor& _shared_executor; using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; public: /** diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index eee6264b9f4..ef7e422c722 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -240,7 +240,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, - vespalib::ThreadExecutor& shared_executor, + vespalib::Executor& shared_executor, const HwInfo &hwInfo) : proton::IAttributeManager(), _attributes(), @@ -264,7 +264,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext &fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, - vespalib::ThreadExecutor& shared_executor, + vespalib::Executor& shared_executor, const IAttributeFactory::SP &factory, const HwInfo &hwInfo) : proton::IAttributeManager(), @@ -558,12 +558,6 @@ AttributeManager::getAttributeFieldWriter() const return _attributeFieldWriter; } -vespalib::ThreadExecutor& -AttributeManager::get_shared_executor() const -{ - return _shared_executor; -} - AttributeVector * AttributeManager::getWritableAttribute(const vespalib::string &name) const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index 08e2d511d70..beea59b5350 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -31,14 +31,14 @@ class ShrinkLidSpaceFlushTarget; class AttributeManager : public proton::IAttributeManager { private: - typedef search::attribute::Config Config; - typedef search::SerialNum SerialNum; - typedef AttributeCollectionSpec Spec; + using AttributeReadGuard = search::attribute::AttributeReadGuard; + using AttributeVectorSP = std::shared_ptr<search::AttributeVector>; + using Config = search::attribute::Config; using FlushableAttributeSP = std::shared_ptr<FlushableAttribute>; - using ShrinkerSP = std::shared_ptr<ShrinkLidSpaceFlushTarget>; using IFlushTargetSP = std::shared_ptr<searchcorespi::IFlushTarget>; - using AttributeVectorSP = std::shared_ptr<search::AttributeVector>; - using AttributeReadGuard = search::attribute::AttributeReadGuard; + using SerialNum = search::SerialNum; + using ShrinkerSP = std::shared_ptr<ShrinkLidSpaceFlushTarget>; + using Spec = AttributeCollectionSpec; class AttributeWrap { @@ -67,8 +67,8 @@ private: const ShrinkerSP &getShrinker() const { return _shrinker; } }; - typedef vespalib::hash_map<vespalib::string, AttributeWrap> AttributeMap; - typedef vespalib::hash_map<vespalib::string, FlushableWrap> FlushableMap; + using AttributeMap = vespalib::hash_map<vespalib::string, AttributeWrap>; + using FlushableMap = vespalib::hash_map<vespalib::string, FlushableWrap>; AttributeMap _attributes; FlushableMap _flushables; @@ -80,7 +80,7 @@ private: IAttributeFactory::SP _factory; std::shared_ptr<search::attribute::Interlock> _interlock; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; - vespalib::ThreadExecutor& _shared_executor; + vespalib::Executor& _shared_executor; HwInfo _hwInfo; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; @@ -100,14 +100,14 @@ private: void transferExtraAttributes(const AttributeManager &currMgr); public: - typedef std::shared_ptr<AttributeManager> SP; + using SP = std::shared_ptr<AttributeManager>; AttributeManager(const vespalib::string &baseDir, const vespalib::string &documentSubDbName, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, - vespalib::ThreadExecutor& shared_executor, + vespalib::Executor& shared_executor, const HwInfo &hwInfo); AttributeManager(const vespalib::string &baseDir, @@ -115,7 +115,7 @@ public: const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, - vespalib::ThreadExecutor& shared_executor, + vespalib::Executor& shared_executor, const IAttributeFactory::SP &factory, const HwInfo &hwInfo); @@ -171,7 +171,7 @@ public: vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override; - vespalib::ThreadExecutor& get_shared_executor() const override; + vespalib::Executor& get_shared_executor() const override { return _shared_executor; } search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index 5f162281d96..d0caf92be17 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -167,12 +167,6 @@ FilterAttributeManager::getAttributeFieldWriter() const return _mgr->getAttributeFieldWriter(); } -vespalib::ThreadExecutor& -FilterAttributeManager::get_shared_executor() const -{ - return _mgr->get_shared_executor(); -} - search::AttributeVector * FilterAttributeManager::getWritableAttribute(const vespalib::string &name) const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 1512ab32d62..e291aca6922 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -20,15 +20,14 @@ public: typedef std::set<vespalib::string> AttributeSet; private: - AttributeSet _acceptedAttributes; - IAttributeManager::SP _mgr; + AttributeSet _acceptedAttributes; + IAttributeManager::SP _mgr; std::vector<search::AttributeVector *> _acceptedWritableAttributes; bool acceptAttribute(const vespalib::string &name) const; public: - FilterAttributeManager(const AttributeSet &acceptedAttributes, - IAttributeManager::SP mgr); + FilterAttributeManager(const AttributeSet &acceptedAttributes, IAttributeManager::SP mgr); ~FilterAttributeManager() override; // Implements search::IAttributeManager @@ -47,7 +46,7 @@ public: void pruneRemovedFields(search::SerialNum serialNum) override; const IAttributeFactory::SP &getFactory() const override; vespalib::ISequencedTaskExecutor & getAttributeFieldWriter() const override; - vespalib::ThreadExecutor& get_shared_executor() const override; + vespalib::Executor& get_shared_executor() const override { return _mgr->get_shared_executor(); } search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override; const std::vector<search::AttributeVector *> & getWritableAttributes() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index b8968ba9d2e..d32052fe4fa 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -14,7 +14,7 @@ namespace search::attribute { class IAttributeFunctor; } namespace vespalib { class ISequencedTaskExecutor; - class ThreadExecutor; + class Executor; class IDestructorCallback; } @@ -76,7 +76,7 @@ struct IAttributeManager : public search::IAttributeManager virtual vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const = 0; - virtual vespalib::ThreadExecutor& get_shared_executor() const = 0; + virtual vespalib::Executor& get_shared_executor() const = 0; /* * Get pointer to named writable attribute. If attribute isn't diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 421e602a9cf..03bad4c7eaf 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_add_library(searchcore_pcommon STATIC ipendinglidtracker.cpp operation_rate_tracker.cpp pendinglidtracker.cpp + replay_feedtoken_state.cpp select_utils.cpp selectcontext.cpp selectpruner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp index 675dce10be4..c74819577e9 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp @@ -33,6 +33,12 @@ State::setResult(ResultUP result, bool documentWasFound) { _result = std::move(result); } +bool +State::is_replay() const noexcept +{ + return false; +} + void State::fail() { diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h index 8ccb4863878..53f551999d2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h @@ -22,19 +22,33 @@ public: virtual void send(ResultUP result, bool documentWasFound) = 0; }; + +/* + * Interface class for feed token state. + */ +class IState : public vespalib::IDestructorCallback { +public: + virtual bool is_replay() const noexcept = 0; + virtual void fail() = 0; + virtual void setResult(ResultUP result, bool documentWasFound) = 0; + virtual const storage::spi::Result &getResult() = 0; +}; + + /** * This holds the result of the feed operation until it is either failed or acked. * Guarantees that the result is propagated back to the invoker via ITransport interface. */ -class State : public vespalib::IDestructorCallback { +class State : public IState { public: State(const State &) = delete; State & operator = (const State &) = delete; State(ITransport & transport); ~State() override; - void fail(); - void setResult(ResultUP result, bool documentWasFound); - const storage::spi::Result &getResult() { return *_result; } + bool is_replay() const noexcept override; + void fail() override; + void setResult(ResultUP result, bool documentWasFound) override; + const storage::spi::Result &getResult() override { return *_result; } protected: void ack(); private: @@ -71,7 +85,7 @@ make(std::shared_ptr<ITransport> transport) { } -using FeedToken = std::shared_ptr<feedtoken::State>; +using FeedToken = std::shared_ptr<feedtoken::IState>; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index 2f0db6083c7..6d69c884c90 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "hw_info_sampler.h" -#include <vespa/config/config.h> #include <vespa/config/print/fileconfigwriter.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/fastos/file.h> #include <vespa/searchcore/config/config-hwinfo.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/alloc.h> #include <filesystem> #include <thread> + #include <vespa/log/log.h> LOG_SETUP(".proton.common.hw_info_sampler"); @@ -55,7 +57,8 @@ sampleCpuCores(const HwInfoSampler::Config &cfg) return std::thread::hardware_concurrency(); } -std::unique_ptr<HwinfoConfig> readConfig(const vespalib::string &path) { +std::unique_ptr<HwinfoConfig> +readConfig(const vespalib::string &path) { FileSpec spec(path + "/" + "hwinfo.cfg"); ConfigSubscriber s(spec); std::unique_ptr<ConfigHandle<HwinfoConfig>> handle = s.subscribe<HwinfoConfig>("hwinfo"); @@ -79,29 +82,31 @@ void writeConfig(const vespalib::string &path, double measureDiskWriteSpeed(const vespalib::string &path, size_t diskWriteLen) { - FastOS_File testFile; vespalib::string fileName = path + "/hwinfo-writespeed"; size_t bufferLen = 1_Mi; Alloc buffer(Alloc::allocMMap(bufferLen)); memset(buffer.get(), 0, buffer.size()); - testFile.EnableDirectIO(); - testFile.OpenWriteOnlyTruncate(fileName.c_str()); - sync(); - sleep(1); - sync(); - sleep(1); - Clock::time_point before = Clock::now(); - size_t residue = diskWriteLen; - while (residue > 0) { - size_t writeNow = std::min(residue, bufferLen); - testFile.WriteBuf(buffer.get(), writeNow); - residue -= writeNow; + double diskWriteSpeed; + { + FastOS_File testFile; + testFile.EnableDirectIO(); + testFile.OpenWriteOnlyTruncate(fileName.c_str()); + sync(); + sleep(1); + sync(); + sleep(1); + Clock::time_point before = Clock::now(); + size_t residue = diskWriteLen; + while (residue > 0) { + size_t writeNow = std::min(residue, bufferLen); + testFile.WriteBuf(buffer.get(), writeNow); + residue -= writeNow; + } + Clock::time_point after = Clock::now(); + double elapsed = vespalib::to_s(after - before); + diskWriteSpeed = diskWriteLen / elapsed / 1_Mi; } - Clock::time_point after = Clock::now(); - testFile.Close(); vespalib::unlink(fileName); - double elapsed = vespalib::to_s(after - before); - double diskWriteSpeed = diskWriteLen / elapsed / 1_Mi; return diskWriteSpeed; } diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp new file mode 100644 index 00000000000..a3a473c9548 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "replay_feedtoken_state.h" + +namespace proton::feedtoken { + +ReplayState::ReplayState(vespalib::SharedOperationThrottler::Token throttler_token) + : IState(), + _throttler_token(std::move(throttler_token)) +{ +} + +ReplayState::~ReplayState() = default; + +bool +ReplayState::is_replay() const noexcept +{ + return true; +} + +void +ReplayState::fail() +{ +} + +void +ReplayState::setResult(ResultUP, bool) +{ +} + +const storage::spi::Result& +ReplayState::getResult() +{ + abort(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h new file mode 100644 index 00000000000..512f12a50af --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "feedtoken.h" +#include <vespa/vespalib/util/shared_operation_throttler.h> + +namespace proton::feedtoken { + +/* + * Feed token state used during replay. It contains a throttler token + * which allows the related shared operation throttler to track the completion + * of the feed operation. + */ +class ReplayState : public IState { + vespalib::SharedOperationThrottler::Token _throttler_token; +public: + ~ReplayState() override; + ReplayState(vespalib::SharedOperationThrottler::Token throttler_token); + bool is_replay() const noexcept override; + void fail() override; + void setResult(ResultUP result, bool documentWasFound) override; + const storage::spi::Result &getResult() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 28a91e1444d..a04f5bfa651 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -145,7 +145,7 @@ SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, const Summa juniperCfg, attributeMgr, _docStore, repo); } -SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const LogDocumentStore::Config & storeConfig, +SummaryManager::SummaryManager(vespalib::Executor &shared_executor, const LogDocumentStore::Config & storeConfig, const search::GrowStrategy & growStrategy, const vespalib::string &baseDir, const DocTypeName &docTypeName, const TuneFileSummary &tuneFileSummary, const FileHeaderContext &fileHeaderContext, search::transactionlog::SyncProxy &tlSyncer, @@ -154,7 +154,7 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const LogDoc _docTypeName(docTypeName), _docStore() { - _docStore = std::make_shared<LogDocumentStore>(executor, baseDir, storeConfig, growStrategy, tuneFileSummary, + _docStore = std::make_shared<LogDocumentStore>(shared_executor, baseDir, storeConfig, growStrategy, tuneFileSummary, fileHeaderContext, tlSyncer, std::move(bucketizer)); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index b3cbd399262..ba55761d091 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -10,7 +10,6 @@ #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/util/threadexecutor.h> namespace search { class IBucketizer; } namespace search::common { class FileHeaderContext; } @@ -60,7 +59,7 @@ private: public: typedef std::shared_ptr<SummaryManager> SP; - SummaryManager(vespalib::ThreadExecutor & executor, + SummaryManager(vespalib::Executor &shared_executor, const search::LogDocumentStore::Config & summary, const search::GrowStrategy & growStrategy, const vespalib::string &baseDir, diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp index 21506c3014f..568fa740d6f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp @@ -11,7 +11,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow, const vespalib::string & baseDir, const vespalib::string &subDbName, const DocTypeName &docTypeName, - vespalib::ThreadExecutor &summaryExecutor, + vespalib::Executor &shared_executor, const search::LogDocumentStore::Config & storeCfg, const search::TuneFileSummary &tuneFile, const search::common::FileHeaderContext &fileHeaderContext, @@ -23,7 +23,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow, _baseDir(baseDir), _subDbName(subDbName), _docTypeName(docTypeName), - _summaryExecutor(summaryExecutor), + _shared_executor(shared_executor), _storeCfg(storeCfg), _tuneFile(tuneFile), _fileHeaderContext(fileHeaderContext), @@ -41,7 +41,7 @@ SummaryManagerInitializer::run() vespalib::Timer timer; EventLogger::loadDocumentStoreStart(_subDbName); *_result = std::make_shared<SummaryManager> - (_summaryExecutor, _storeCfg, _grow, _baseDir, _docTypeName, + (_shared_executor, _storeCfg, _grow, _baseDir, _docTypeName, _tuneFile, _fileHeaderContext, _tlSyncer, _bucketizer); EventLogger::loadDocumentStoreComplete(_subDbName, timer.elapsed()); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h index 7075560ed56..318edc425a7 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h @@ -5,7 +5,6 @@ #include "summarymanager.h" #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcommon/common/growstrategy.h> -#include <vespa/vespalib/stllike/string.h> namespace proton { @@ -20,7 +19,7 @@ class SummaryManagerInitializer : public initializer::InitializerTask const vespalib::string _baseDir; const vespalib::string _subDbName; const DocTypeName _docTypeName; - vespalib::ThreadExecutor &_summaryExecutor; + vespalib::Executor &_shared_executor; const search::LogDocumentStore::Config _storeCfg; const search::TuneFileSummary _tuneFile; const search::common::FileHeaderContext &_fileHeaderContext; @@ -36,7 +35,7 @@ public: const vespalib::string & baseDir, const vespalib::string &subDbName, const DocTypeName &docTypeName, - vespalib::ThreadExecutor & summaryExecutor, + vespalib::Executor &shared_executor, const search::LogDocumentStore::Config & storeCfg, const search::TuneFileSummary &tuneFile, const search::common::FileHeaderContext & fileHeaderContext, diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 2b76faa8d7f..011d97d4609 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -4,20 +4,21 @@ #include "flush_all_strategy.h" #include "flushengine.h" #include "flushtask.h" -#include "tls_stats_map.h" #include "tls_stats_factory.h" +#include "tls_stats_map.h" #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/flush_token.h> -#include <vespa/vespalib/util/jsonwriter.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> #include <vespa/log/log.h> LOG_SETUP(".proton.flushengine.flushengine"); -typedef vespalib::Executor::Task Task; -using searchcorespi::IFlushTarget; +using Task = vespalib::Executor::Task; using searchcorespi::FlushStats; +using searchcorespi::IFlushTarget; +using vespalib::CpuUsage; using namespace std::chrono_literals; namespace proton { @@ -86,7 +87,7 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStats _threadPool(128_Ki), _strategy(std::move(strategy)), _priorityStrategy(), - _executor(numThreads, 128_Ki, flush_engine_executor), + _executor(numThreads, 128_Ki, CpuUsage::wrap(flush_engine_executor, CpuUsage::Category::COMPACT)), _lock(), _cond(), _handlers(), diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 58dc473b85e..8489a68af15 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/log/log.h> @@ -34,12 +35,14 @@ public: }; VESPA_THREAD_STACK_TAG(match_engine_executor) +VESPA_THREAD_STACK_TAG(match_engine_thread_bundle) } // namespace anon namespace proton { using namespace vespalib::slime; +using vespalib::CpuUsage; MatchEngine::MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey, bool async) : _lock(), @@ -48,8 +51,10 @@ MatchEngine::MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t di _closed(false), _forward_issues(true), _handlers(), - _executor(std::max(size_t(1), numThreads / threadsPerSearch), 256_Ki, match_engine_executor), - _threadBundlePool(std::max(size_t(1), threadsPerSearch)), + _executor(std::max(size_t(1), numThreads / threadsPerSearch), 256_Ki, + CpuUsage::wrap(match_engine_executor, CpuUsage::Category::READ)), + _threadBundlePool(std::max(size_t(1), threadsPerSearch), + CpuUsage::wrap(match_engine_thread_bundle, CpuUsage::Category::READ)), _nodeUp(false), _nodeMaintenance(false) { @@ -147,6 +152,9 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req) } } _threadBundlePool.release(std::move(threadBundle)); + if (searchRequest->expired()) { + vespalib::Issue::report("search request timed out; results may be incomplete"); + } } ret->request = req.release(); if (_forward_issues) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index ce04464a851..032991cbf6a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -47,7 +47,8 @@ get_feature_set(const MatchToolsFactory &mtf, } else { matchTools->setup_dump(); } - auto retval = ExtractFeatures::get_feature_set(matchTools->search(), matchTools->rank_program(), docs, mtf.get_feature_rename_map()); + auto retval = ExtractFeatures::get_feature_set(matchTools->search(), matchTools->rank_program(), docs, + matchTools->getDoom(), mtf.get_feature_rename_map()); if (auto onSummaryTask = mtf.createOnSummaryTask()) { onSummaryTask->run(docs); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index d74885bd0be..9f09255795a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -2,6 +2,7 @@ #include "extract_features.h" #include "match_tools.h" +#include <vespa/vespalib/util/doom.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/runnable.h> @@ -10,6 +11,7 @@ #include <vespa/searchlib/fef/rank_program.h> #include <vespa/searchlib/queryeval/searchiterator.h> +using vespalib::Doom; using vespalib::Runnable; using vespalib::ThreadBundle; using search::FeatureSet; @@ -59,15 +61,19 @@ struct MyChunk : Runnable { const std::pair<uint32_t,uint32_t> *begin; const std::pair<uint32_t,uint32_t> *end; FeatureValues &result; + const Doom &doom; MyChunk(const std::pair<uint32_t,uint32_t> *begin_in, - const std::pair<uint32_t,uint32_t> *end_in, - FeatureValues &result_in) - : begin(begin_in), end(end_in), result(result_in) {} + const std::pair<uint32_t,uint32_t> *end_in, + FeatureValues &result_in, const Doom &doom_in) + : begin(begin_in), end(end_in), result(result_in), doom(doom_in) {} void calculate_features(SearchIterator &search, const FeatureResolver &resolver) { assert(end > begin); assert(resolver.num_features() == result.names.size()); search.initRange(begin[0].first, end[-1].first + 1); for (auto pos = begin; pos != end; ++pos) { + if (doom.hard_doom()) { + return; + } search.unpack(pos->first); auto *dst = &result.values[pos->second * resolver.num_features()]; extract_values(resolver, pos->first, dst); @@ -81,9 +87,10 @@ struct FirstChunk : MyChunk { FirstChunk(const std::pair<uint32_t,uint32_t> *begin_in, const std::pair<uint32_t,uint32_t> *end_in, FeatureValues &result_in, + const Doom &doom_in, SearchIterator &search_in, const FeatureResolver &resolver_in) - : MyChunk(begin_in, end_in, result_in), + : MyChunk(begin_in, end_in, result_in, doom_in), search(search_in), resolver(resolver_in) {} void run() override { calculate_features(search, resolver); } @@ -94,8 +101,9 @@ struct LaterChunk : MyChunk { LaterChunk(const std::pair<uint32_t,uint32_t> *begin_in, const std::pair<uint32_t,uint32_t> *end_in, FeatureValues &result_in, + const Doom &doom_in, const MatchToolsFactory &mtf_in) - : MyChunk(begin_in, end_in, result_in), + : MyChunk(begin_in, end_in, result_in, doom_in), mtf(mtf_in) {} void run() override { auto tools = mtf.createMatchTools(); @@ -125,13 +133,16 @@ struct MyWork { FeatureSet::UP ExtractFeatures::get_feature_set(SearchIterator &search, RankProgram &rank_program, const std::vector<uint32_t> &docs, - const MatchToolsFactory::StringStringMap &renames) + const Doom &doom, const MatchToolsFactory::StringStringMap &renames) { FeatureResolver resolver(rank_program.get_seeds(false)); auto result = std::make_unique<FeatureSet>(extract_names(resolver, renames), docs.size()); if (!docs.empty()) { search.initRange(docs.front(), docs.back()+1); for (uint32_t docid: docs) { + if (doom.hard_doom()) { + return result; + } search.unpack(docid); auto *dst = result->getFeaturesByIndex(result->addDocId(docid)); extract_values(resolver, docid, dst); @@ -159,9 +170,9 @@ ExtractFeatures::get_match_features(const MatchToolsFactory &mtf, const OrderedD break; } if (i == 0) { - work.chunks.push_back(std::make_unique<FirstChunk>(&docs[idx], &docs[idx + chunk_size], result, tools->search(), resolver)); + work.chunks.push_back(std::make_unique<FirstChunk>(&docs[idx], &docs[idx + chunk_size], result, tools->getDoom(), tools->search(), resolver)); } else { - work.chunks.push_back(std::make_unique<LaterChunk>(&docs[idx], &docs[idx + chunk_size], result, mtf)); + work.chunks.push_back(std::make_unique<LaterChunk>(&docs[idx], &docs[idx + chunk_size], result, tools->getDoom(), mtf)); } idx += chunk_size; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h index 44bff08df2e..48c3476f164 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/common/stringmap.h> #include <vector> +namespace vespalib { class Doom; }; namespace vespalib { struct ThreadBundle; }; namespace search::queryeval { class SearchIterator; } namespace search::fef { class RankProgram; } @@ -27,8 +28,7 @@ struct ExtractFeatures { * documents (must be in ascending order) using unpack information * from a search. **/ - static FeatureSet::UP get_feature_set(SearchIterator &search, RankProgram &rank_program, const std::vector<uint32_t> &docs, const StringStringMap &renames); - + static FeatureSet::UP get_feature_set(SearchIterator &search, RankProgram &rank_program, const std::vector<uint32_t> &docs, const vespalib::Doom &doom, const StringStringMap &renames); // first: docid, second: result index (must be sorted on docid) using OrderedDocs = std::vector<std::pair<uint32_t,uint32_t>>; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp index af250b61d03..2e56b7010f9 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp @@ -28,15 +28,15 @@ DocumentDBJobTrackers::DocumentDBJobTrackers() { } -DocumentDBJobTrackers::~DocumentDBJobTrackers() {} +DocumentDBJobTrackers::~DocumentDBJobTrackers() = default; namespace { IFlushTarget::SP -trackFlushTarget(const IJobTracker::SP &tracker, - const IFlushTarget::SP &target) +trackFlushTarget(std::shared_ptr<IJobTracker> tracker, + std::shared_ptr<IFlushTarget> target) { - return std::make_shared<JobTrackedFlushTarget>(tracker, target); + return std::make_shared<JobTrackedFlushTarget>(std::move(tracker), std::move(target)); } } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h index a05fbb49a41..cd7ec612d98 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h @@ -4,8 +4,6 @@ #include "documentdb_tagged_metrics.h" #include "job_tracker.h" #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <chrono> -#include <mutex> namespace proton { @@ -16,20 +14,23 @@ namespace proton { class DocumentDBJobTrackers { private: - std::mutex _lock; using time_point = std::chrono::time_point<std::chrono::steady_clock>; - time_point _now; - JobTracker::SP _attributeFlush; - JobTracker::SP _memoryIndexFlush; - JobTracker::SP _diskIndexFusion; - JobTracker::SP _documentStoreFlush; - JobTracker::SP _documentStoreCompact; - JobTracker::SP _bucketMove; - JobTracker::SP _lidSpaceCompact; - JobTracker::SP _removedDocumentsPrune; + using JobTrackerSP = std::shared_ptr<JobTracker>; + std::mutex _lock; + time_point _now; + JobTrackerSP _attributeFlush; + JobTrackerSP _memoryIndexFlush; + JobTrackerSP _diskIndexFusion; + JobTrackerSP _documentStoreFlush; + JobTrackerSP _documentStoreCompact; + JobTrackerSP _bucketMove; + JobTrackerSP _lidSpaceCompact; + JobTrackerSP _removedDocumentsPrune; public: DocumentDBJobTrackers(); + DocumentDBJobTrackers(const DocumentDBJobTrackers &) = delete; + DocumentDBJobTrackers & operator = (const DocumentDBJobTrackers &) = delete; ~DocumentDBJobTrackers(); IJobTracker &getAttributeFlush() { return *_attributeFlush; } @@ -37,9 +38,9 @@ public: IJobTracker &getDiskIndexFusion() { return *_diskIndexFusion; } IJobTracker &getDocumentStoreFlush() { return *_documentStoreFlush; } IJobTracker &getDocumentStoreCompact() { return *_documentStoreCompact; } - IJobTracker::SP getBucketMove() { return _bucketMove; } - IJobTracker::SP getLidSpaceCompact() { return _lidSpaceCompact; } - IJobTracker::SP getRemovedDocumentsPrune() { return _removedDocumentsPrune; } + std::shared_ptr<IJobTracker> getBucketMove() { return _bucketMove; } + std::shared_ptr<IJobTracker> getLidSpaceCompact() { return _lidSpaceCompact; } + std::shared_ptr<IJobTracker> getRemovedDocumentsPrune() { return _removedDocumentsPrune; } searchcorespi::IFlushTarget::List trackFlushTargets(const searchcorespi::IFlushTarget::List &flushTargets); @@ -47,5 +48,4 @@ public: void updateMetrics(DocumentDBTaggedMetrics::JobMetrics &metrics); }; -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h b/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h index 646a9f07672..eec8cfbc44a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h @@ -2,8 +2,6 @@ #pragma once -#include <memory> - namespace proton { /** @@ -11,9 +9,7 @@ namespace proton { */ struct IJobTracker { - typedef std::shared_ptr<IJobTracker> SP; - - virtual ~IJobTracker() {} + virtual ~IJobTracker() = default; virtual void start() = 0; virtual void end() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp index 8de3c29d02b..5b49c724c6f 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp @@ -8,11 +8,11 @@ using searchcorespi::FlushTask; namespace proton { -JobTrackedFlushTarget::JobTrackedFlushTarget(const IJobTracker::SP &tracker, - const IFlushTarget::SP &target) +JobTrackedFlushTarget::JobTrackedFlushTarget(std::shared_ptr<IJobTracker> tracker, + std::shared_ptr<IFlushTarget> target) : IFlushTarget(target->getName(), target->getType(), target->getComponent()), - _tracker(tracker), - _target(target) + _tracker(std::move(tracker)), + _target(std::move(target)) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h index 7a9bae7f662..35d1b0b0b12 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h @@ -13,12 +13,12 @@ namespace proton { class JobTrackedFlushTarget : public searchcorespi::IFlushTarget { private: - IJobTracker::SP _tracker; - searchcorespi::IFlushTarget::SP _target; + std::shared_ptr<IJobTracker> _tracker; + std::shared_ptr<searchcorespi::IFlushTarget> _target; public: - JobTrackedFlushTarget(const IJobTracker::SP &tracker, - const searchcorespi::IFlushTarget::SP &target); + JobTrackedFlushTarget(std::shared_ptr<IJobTracker> tracker, + std::shared_ptr<searchcorespi::IFlushTarget> target); ~JobTrackedFlushTarget(); const IJobTracker &getTracker() const { return *_tracker; } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp index f5cf0b1afff..3e06c8321fd 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp @@ -6,13 +6,14 @@ using searchcorespi::FlushTask; namespace proton { -JobTrackedFlushTask::JobTrackedFlushTask(const IJobTracker::SP &tracker, - FlushTask::UP task) - : _tracker(tracker), +JobTrackedFlushTask::JobTrackedFlushTask(std::shared_ptr<IJobTracker> tracker, FlushTask::UP task) + : _tracker(std::move(tracker)), _task(std::move(task)) { } +JobTrackedFlushTask::~JobTrackedFlushTask() = default; + void JobTrackedFlushTask::run() { @@ -21,4 +22,4 @@ JobTrackedFlushTask::run() _tracker->end(); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h index fe8d59b9dd6..a10ccbdffd6 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h @@ -12,16 +12,18 @@ namespace proton { class JobTrackedFlushTask : public searchcorespi::FlushTask { private: - IJobTracker::SP _tracker; + std::shared_ptr<IJobTracker> _tracker; searchcorespi::FlushTask::UP _task; public: - JobTrackedFlushTask(const IJobTracker::SP &tracker, + JobTrackedFlushTask(std::shared_ptr<IJobTracker> tracker, searchcorespi::FlushTask::UP task); + JobTrackedFlushTask(const JobTrackedFlushTask &) = delete; + JobTrackedFlushTask & operator = (const JobTrackedFlushTask &) = delete; + ~JobTrackedFlushTask() override; - // Implements searchcorespi::FlushTask - virtual void run() override; - virtual search::SerialNum getFlushSerial() const override { + void run() override; + search::SerialNum getFlushSerial() const override { return _task->getFlushSerial(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h index 49b63a6d018..dee974f8b56 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h @@ -19,8 +19,6 @@ private: std::mutex &_lock; public: - typedef std::shared_ptr<JobTracker> SP; - JobTracker(time_point now, std::mutex &lock); /** @@ -29,10 +27,8 @@ public: */ double sampleLoad(time_point now, const std::lock_guard<std::mutex> &guard); - // Implements IJobTracker - virtual void start() override; - virtual void end() override; + void start() override; + void end() override; }; -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp index 9bc20f95d13..d026ef549d4 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp @@ -1,21 +1,53 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resource_usage_metrics.h" +#include <vespa/vespalib/util/stringfmt.h> + +using vespalib::make_string; namespace proton { +ResourceUsageMetrics::CpuUtilMetrics::CpuUtilMetrics(metrics::MetricSet *parent) + : MetricSet("cpu_util", {}, "Unnormalized cpu utilization for various categories", parent), + setup("setup", {}, "cpu used by system init and (re-)configuration", this), + read("read", {}, "cpu used by reading data from the system", this), + write("write", {}, "cpu used by writing data to the system", this), + compact("compact", {}, "cpu used by internal data re-structuring", this), + other("other", {}, "cpu used by work not classified as a specific category", this) +{ +} + +ResourceUsageMetrics::CpuUtilMetrics::~CpuUtilMetrics() = default; + +ResourceUsageMetrics::DetailedResourceMetrics::DetailedResourceMetrics(const vespalib::string& resource_type, metrics::MetricSet* parent) + : MetricSet(make_string("%s_usage", resource_type.c_str()), {}, make_string("Detailed resource usage metrics for %s", + resource_type.c_str()), parent), + total("total", {}, make_string("The total relative amount of %s used by this content node (value in the range [0, 1])", + resource_type.c_str()), this), + total_util("total_utilization", {}, make_string("The relative amount of %s used compared to the content node %s resource limit", + resource_type.c_str(), resource_type.c_str()), this), + transient("transient", {}, make_string("The relative amount of transient %s used by this content node (value in the range [0, 1])", + resource_type.c_str()), this) +{ +} + +ResourceUsageMetrics::DetailedResourceMetrics::~DetailedResourceMetrics() = default; + ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent) - : MetricSet("resource_usage", {}, "Usage metrics for various resources in this search engine", parent), - disk("disk", {}, "The relative amount of disk space used on this machine (value in the range [0, 1])", this), + : MetricSet("resource_usage", {}, "Usage metrics for various resources in this content node", parent), + disk("disk", {}, "The relative amount of disk used by this content node (transient usage not included, value in the range [0, 1]). Same value as reported to the cluster controller", this), diskUtilization("disk_utilization", {}, "The relative amount of disk used compared to the disk resource limit", this), - memory("memory", {}, "The relative amount of memory used by this process (value in the range [0, 1])", this), + memory("memory", {}, "The relative amount of memory used by this content node (transient usage not included, value in the range [0, 1]). Same value as reported to the cluster controller", this), memoryUtilization("memory_utilization", {}, "The relative amount of memory used compared to the memory resource limit", this), transient_memory("transient_memory", {}, "The relative amount of transient memory needed for loading attributes. Max value among all attributes (value in the range [0, 1])", this), transient_disk("transient_disk", {}, "The relative amount of transient disk needed for running disk index fusion. Max value among all disk indexes (value in the range [0, 1])", this), + disk_usage("disk", this), + memory_usage("memory", this), memoryMappings("memory_mappings", {}, "The number of mapped memory areas", this), openFileDescriptors("open_file_descriptors", {}, "The number of open files", this), feedingBlocked("feeding_blocked", {}, "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this), - mallocArena("malloc_arena", {}, "Size of malloc arena", this) + mallocArena("malloc_arena", {}, "Size of malloc arena", this), + cpu_util(this) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h index 774bb645c84..97cad935dba 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h @@ -12,16 +12,42 @@ namespace proton { */ struct ResourceUsageMetrics : metrics::MetricSet { + struct CpuUtilMetrics : metrics::MetricSet { + metrics::DoubleValueMetric setup; + metrics::DoubleValueMetric read; + metrics::DoubleValueMetric write; + metrics::DoubleValueMetric compact; + metrics::DoubleValueMetric other; + + CpuUtilMetrics(metrics::MetricSet *parent); + ~CpuUtilMetrics(); + }; + + struct DetailedResourceMetrics : metrics::MetricSet { + metrics::DoubleValueMetric total; + metrics::DoubleValueMetric total_util; + metrics::DoubleValueMetric transient; + + DetailedResourceMetrics(const vespalib::string& resource_type, metrics::MetricSet* parent); + ~DetailedResourceMetrics(); + }; + + // TODO Vespa 8: Remove diskUtilization, memoryUtilization, transient_memory, transient_disk. + // These are now included in disk_usage and memory_usage. + metrics::DoubleValueMetric disk; metrics::DoubleValueMetric diskUtilization; metrics::DoubleValueMetric memory; metrics::DoubleValueMetric memoryUtilization; metrics::DoubleValueMetric transient_memory; metrics::DoubleValueMetric transient_disk; + DetailedResourceMetrics disk_usage; + DetailedResourceMetrics memory_usage; metrics::LongValueMetric memoryMappings; metrics::LongValueMetric openFileDescriptors; metrics::LongValueMetric feedingBlocked; metrics::LongValueMetric mallocArena; + CpuUtilMetrics cpu_util; ResourceUsageMetrics(metrics::MetricSet *parent); ~ResourceUsageMetrics(); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp index 6307604598d..9c8e6591730 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp @@ -94,8 +94,13 @@ void ResourceUsageTracker::notifyDiskMemUsage(DiskMemUsageState state) { std::lock_guard guard(_lock); - // TODO: Subtract transient resource (memory and disk) usage from the absolute numbers. - _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_address_space_usage()); + // The transient resource usage is subtracted from the total resource usage + // before it eventually is reported to the cluster controller (to decide whether to block client feed). + // This ensures that the transient resource usage is covered by the resource headroom on the content node, + // instead of leading to feed blocked due to natural fluctuations. + _resource_usage = ResourceUsage(state.non_transient_disk_usage(), + state.non_transient_memory_usage(), + _resource_usage.get_attribute_address_space_usage()); if (_listener != nullptr) { _listener->update_resource_usage(_resource_usage); } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 8f99eb5e8a7..f0cb123b49f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -8,6 +8,7 @@ #include <vespa/config-bucketspaces.h> #include <vespa/searchlib/common/tunefileinfo.hpp> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/config/retriever/configsnapshot.hpp> #include <cassert> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp index 230593c2c1d..d740d5b129d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -11,25 +11,29 @@ DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept { } -DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept - : _disk_usage(usage_state.diskState().usage()), - _disk_utilization(usage_state.diskState().utilization()), - _transient_disk_usage(usage_state.transient_disk_usage()), - _memory_usage(usage_state.memoryState().usage()), - _memory_utilization(usage_state.memoryState().utilization()), - _transient_memory_usage(usage_state.transient_memory_usage()) +DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState& usage) noexcept + : _total_disk_usage(usage.diskState().usage()), + _total_disk_utilization(usage.diskState().utilization()), + _transient_disk_usage(usage.transient_disk_usage()), + _non_transient_disk_usage(usage.non_transient_disk_usage()), + _total_memory_usage(usage.memoryState().usage()), + _total_memory_utilization(usage.memoryState().utilization()), + _transient_memory_usage(usage.transient_memory_usage()), + _non_transient_memory_usage(usage.non_transient_memory_usage()) { } void -DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept +DiskMemUsageMetrics::merge(const DiskMemUsageState& usage) noexcept { - _disk_usage = std::max(_disk_usage, usage_state.diskState().usage()); - _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); - _transient_disk_usage = std::max(_transient_disk_usage, usage_state.transient_disk_usage()); - _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); - _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); - _transient_memory_usage = std::max(_transient_memory_usage, usage_state.transient_memory_usage()); + _total_disk_usage = std::max(_total_disk_usage, usage.diskState().usage()); + _total_disk_utilization = std::max(_total_disk_utilization, usage.diskState().utilization()); + _transient_disk_usage = std::max(_transient_disk_usage, usage.transient_disk_usage()); + _non_transient_disk_usage = std::max(_non_transient_disk_usage, usage.non_transient_disk_usage()); + _total_memory_usage = std::max(_total_memory_usage, usage.memoryState().usage()); + _total_memory_utilization = std::max(_total_memory_utilization, usage.memoryState().utilization()); + _transient_memory_usage = std::max(_transient_memory_usage, usage.transient_memory_usage()); + _non_transient_memory_usage = std::max(_non_transient_memory_usage, usage.non_transient_memory_usage()); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h index cb97eb4c891..3e3d6fdc752 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -12,23 +12,27 @@ class DiskMemUsageState; */ class DiskMemUsageMetrics { - double _disk_usage; - double _disk_utilization; + double _total_disk_usage; + double _total_disk_utilization; double _transient_disk_usage; - double _memory_usage; - double _memory_utilization; + double _non_transient_disk_usage; + double _total_memory_usage; + double _total_memory_utilization; double _transient_memory_usage; + double _non_transient_memory_usage; public: DiskMemUsageMetrics() noexcept; - DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept; - void merge(const DiskMemUsageState &usage_state) noexcept; - double get_disk_usage() const noexcept { return _disk_usage; } - double get_disk_utilization() const noexcept { return _disk_utilization; } - double get_transient_disk_usage() const noexcept { return _transient_disk_usage; } - double get_memory_usage() const noexcept { return _memory_usage; } - double get_memory_utilization() const noexcept { return _memory_utilization; } - double get_transient_memory_usage() const noexcept { return _transient_memory_usage; } + DiskMemUsageMetrics(const DiskMemUsageState& usage) noexcept; + void merge(const DiskMemUsageState& usage) noexcept; + double total_disk_usage() const noexcept { return _total_disk_usage; } + double total_disk_utilization() const noexcept { return _total_disk_utilization; } + double transient_disk_usage() const noexcept { return _transient_disk_usage; } + double non_transient_disk_usage() const noexcept { return _non_transient_disk_usage; } + double total_memory_usage() const noexcept { return _total_memory_usage; } + double total_memory_utilization() const noexcept { return _total_memory_utilization; } + double transient_memory_usage() const noexcept { return _transient_memory_usage; } + double non_transient_memory_usage() const noexcept { return _non_transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index b205b441bcf..2730388de9a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -3,6 +3,7 @@ #pragma once #include "resource_usage_state.h" +#include <algorithm> namespace proton { @@ -42,6 +43,8 @@ public: const ResourceUsageState &memoryState() const { return _memoryState; } double transient_disk_usage() const { return _transient_disk_usage; } double transient_memory_usage() const { return _transient_memory_usage; } + double non_transient_disk_usage() const { return std::max(0.0, _diskState.usage() - _transient_disk_usage); } + double non_transient_memory_usage() const { return std::max(0.0, _memoryState.usage() - _transient_memory_usage); } bool aboveDiskLimit(double resourceLimitFactor) const { return diskState().aboveLimit(resourceLimitFactor); } bool aboveMemoryLimit(double resourceLimitFactor) const { return memoryState().aboveLimit(resourceLimitFactor); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index f052d663ba6..a30aa916896 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -12,6 +12,7 @@ #include "idocumentsubdb.h" #include "maintenance_jobs_injector.h" #include "reconfig_params.h" +#include "replay_throttling_policy.h" #include <vespa/document/repo/documenttyperepo.h> #include <vespa/metrics/updatehook.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> @@ -77,6 +78,18 @@ makeIndexConfig(const ProtonConfig::Index & cfg) { return index::IndexConfig(WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size); } +ReplayThrottlingPolicy +make_replay_throttling_policy(const ProtonConfig::ReplayThrottlingPolicy& cfg) { + if (cfg.type == ProtonConfig::ReplayThrottlingPolicy::Type::UNLIMITED) { + return ReplayThrottlingPolicy({}); + } + vespalib::SharedOperationThrottler::DynamicThrottleParams params; + params.min_window_size = cfg.minWindowSize; + params.max_window_size = cfg.maxWindowSize; + params.window_size_increment = cfg.windowSizeIncrement; + return ReplayThrottlingPolicy(params); +} + class MetricsUpdateHook : public metrics::UpdateHook { DocumentDB &_db; public: @@ -190,6 +203,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _clusterStateHandler(_writeService.master()), _bucketHandler(_writeService.master()), _indexCfg(makeIndexConfig(protonCfg.index)), + _replay_throttling_policy(std::make_unique<ReplayThrottlingPolicy>(make_replay_throttling_policy(protonCfg.replayThrottlingPolicy))), _config_store(std::move(config_store)), _sessionManager(std::make_shared<matching::SessionManager>(protonCfg.grouping.sessionmanager.maxentries)), _metricsWireService(metricsWireService), @@ -486,7 +500,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum _writeServiceConfig.defaultTaskLimit()); if (params.shouldSubDbsChange()) { applySubDBConfig(*configSnapshot, serialNum, params); - if (serialNum < _feedHandler->getSerialNum()) { + if (serialNum < _feedHandler->get_replay_end_serial_num()) { // Not last entry in tls. Reprocessing should already be done. _subDBs.getReprocessingRunner().reset(); } @@ -720,7 +734,8 @@ DocumentDB::startTransactionLogReplay() getBackingStore().lastSyncToken(), oldestFlushedSerial, newestFlushedSerial, - *_config_store); + *_config_store, + *_replay_throttling_policy); _initGate.countDown(); LOG(debug, "DocumentDB(%s): Database started.", _docTypeName.toString().c_str()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 8e8391b2f31..2030e6ffac9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -51,6 +51,7 @@ class ExecutorThreadingServiceStats; class IDocumentDBOwner; class ISharedThreadingService; class ITransientResourceUsageProvider; +class ReplayThrottlingPolicy; class StatusReport; struct MetricsWireService; @@ -104,6 +105,7 @@ private: ClusterStateHandler _clusterStateHandler; BucketHandler _bucketHandler; index::IndexConfig _indexCfg; + std::unique_ptr<ReplayThrottlingPolicy> _replay_throttling_policy; ConfigStore::UP _config_store; std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer. MetricsWireService &_metricsWireService; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 06132803414..b7c319d46f5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -14,6 +14,7 @@ #include <vespa/config-summarymap.h> #include <vespa/config/file_acquirer/file_acquirer.h> #include <vespa/config/common/configcontext.h> +#include <vespa/config/retriever/configretriever.h> #include <vespa/config/helper/legacy.h> #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> @@ -23,6 +24,7 @@ #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/vespalib/time/time_box.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/config/retriever/configsnapshot.hpp> #include <thread> #include <cassert> diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index a3c29212c66..ad5959d551b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -3,9 +3,12 @@ #pragma once #include "documentdbconfig.h" -#include <vespa/config/config.h> #include <mutex> +namespace config { + class ConfigRetriever; + class DirSpec; +} namespace proton { class BootstrapConfig; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index 6285f4ce70f..d3778b4d745 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -127,6 +127,7 @@ DocumentRetriever::needFetchFromDocStore(const FieldSet & fieldSet) const { case FieldSet::Type::NONE: case FieldSet::Type::DOCID: return false; + case FieldSet::Type::DOCUMENT_ONLY: case FieldSet::Type::ALL: return ! _areAllFieldsAttributes; case FieldSet::Type::FIELD: { @@ -257,6 +258,14 @@ DocumentRetriever::getPartialDocument(search::DocumentIdT lid, const document::D populate(lid, *doc, set.getFields()); break; } + case FieldSet::Type::DOCUMENT_ONLY: { + const auto * actual = getDocumentType().getFieldSet(document::DocumentOnly::NAME); + if (actual != nullptr) { + const auto &set = actual->asCollection(); + populate(lid, *doc, set.getFields()); + } + break; + } case FieldSet::Type::NONE: case FieldSet::Type::DOCID: break; diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 36c8070f140..ca6b3d9ba0f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -3,14 +3,16 @@ #include "executorthreadingservice.h" #include "threading_service_config.h" #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> -using vespalib::SyncableThreadExecutor; using vespalib::BlockingThreadStackExecutor; -using vespalib::SingleExecutor; +using vespalib::CpuUsage; using vespalib::SequencedTaskExecutor; +using vespalib::SingleExecutor; +using vespalib::SyncableThreadExecutor; using OptimizeFor = vespalib::Executor::OptimizeFor; using SharedFieldWriterExecutor = proton::ThreadingServiceConfig::SharedFieldWriterExecutor; @@ -38,22 +40,24 @@ VESPA_THREAD_STACK_TAG(field_writer_executor) } -ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor &sharedExecutor, uint32_t num_treads) +ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor &sharedExecutor, uint32_t num_treads) : ExecutorThreadingService(sharedExecutor, nullptr, nullptr, ThreadingServiceConfig::make(num_treads)) {} -ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sharedExecutor, +ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor& sharedExecutor, vespalib::ISequencedTaskExecutor* field_writer, vespalib::InvokeService * invokerService, const ThreadingServiceConfig& cfg, uint32_t stackSize) : _sharedExecutor(sharedExecutor), - _masterExecutor(1, stackSize, master_executor), + _masterExecutor(1, stackSize, CpuUsage::wrap(master_executor, CpuUsage::Category::WRITE)), _shared_field_writer(cfg.shared_field_writer()), _master_task_limit(cfg.master_task_limit()), - _indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), index_executor)), - _summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), summary_executor)), + _indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), + CpuUsage::wrap(index_executor, CpuUsage::Category::WRITE))), + _summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), + CpuUsage::wrap(summary_executor, CpuUsage::Category::WRITE))), _masterService(_masterExecutor), _indexService(*_indexExecutor), _indexFieldInverter(), @@ -70,9 +74,11 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_summaryExecutor.get()](){ executor->wakeup();})); } if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { - _field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 2, cfg.defaultTaskLimit()); - _attributeFieldWriter = SequencedTaskExecutor::create(attribute_field_writer_executor, cfg.indexingThreads(), cfg.defaultTaskLimit(), - cfg.optimize(), cfg.kindOfwatermark()); + _field_writer = SequencedTaskExecutor::create(CpuUsage::wrap(field_writer_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads() * 2, cfg.defaultTaskLimit()); + _attributeFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(attribute_field_writer_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads(), cfg.defaultTaskLimit(), + cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_attributeFieldWriter.get()](){ executor->wakeup();})); } @@ -81,8 +87,9 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _attribute_field_writer_ptr = _attributeFieldWriter.get(); } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { - _field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 3, cfg.defaultTaskLimit(), - cfg.optimize(), cfg.kindOfwatermark()); + _field_writer = SequencedTaskExecutor::create(CpuUsage::wrap(field_writer_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads() * 3, cfg.defaultTaskLimit(), + cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_field_writer.get()](){ executor->wakeup();})); } @@ -95,10 +102,13 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _index_field_writer_ptr = field_writer; _attribute_field_writer_ptr = field_writer; } else { - _indexFieldInverter = SequencedTaskExecutor::create(index_field_inverter_executor, cfg.indexingThreads(), cfg.defaultTaskLimit()); - _indexFieldWriter = SequencedTaskExecutor::create(index_field_writer_executor, cfg.indexingThreads(), cfg.defaultTaskLimit()); - _attributeFieldWriter = SequencedTaskExecutor::create(attribute_field_writer_executor, cfg.indexingThreads(), cfg.defaultTaskLimit(), - cfg.optimize(), cfg.kindOfwatermark()); + _indexFieldInverter = SequencedTaskExecutor::create(CpuUsage::wrap(index_field_inverter_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads(), cfg.defaultTaskLimit()); + _indexFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(index_field_writer_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads(), cfg.defaultTaskLimit()); + _attributeFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(attribute_field_writer_executor, CpuUsage::Category::WRITE), + cfg.indexingThreads(), cfg.defaultTaskLimit(), + cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_attributeFieldWriter.get()](){ executor->wakeup();})); } diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 8572f7126d6..43d546927c2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -20,7 +20,7 @@ class ExecutorThreadingService : public searchcorespi::index::IThreadingService { private: using Registration = std::unique_ptr<vespalib::IDestructorCallback>; - vespalib::ThreadExecutor & _sharedExecutor; + vespalib::Executor & _sharedExecutor; vespalib::ThreadStackExecutor _masterExecutor; ThreadingServiceConfig::SharedFieldWriterExecutor _shared_field_writer; std::atomic<uint32_t> _master_task_limit; @@ -42,9 +42,9 @@ public: /** * Convenience constructor used in unit tests. */ - ExecutorThreadingService(vespalib::ThreadExecutor& sharedExecutor, uint32_t num_treads = 1); + ExecutorThreadingService(vespalib::Executor& sharedExecutor, uint32_t num_treads = 1); - ExecutorThreadingService(vespalib::ThreadExecutor& sharedExecutor, + ExecutorThreadingService(vespalib::Executor& sharedExecutor, vespalib::ISequencedTaskExecutor* field_writer, vespalib::InvokeService * invokeService, const ThreadingServiceConfig& cfg, @@ -72,7 +72,7 @@ public: vespalib::ThreadExecutor &summary() override { return *_summaryExecutor; } - vespalib::ThreadExecutor &shared() override { + vespalib::Executor &shared() override { return _sharedExecutor; } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 8b99c39dd65..16bd2537813 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -464,13 +464,14 @@ FeedHandler::close() void FeedHandler::replayTransactionLog(SerialNum flushedIndexMgrSerial, SerialNum flushedSummaryMgrSerial, SerialNum oldestFlushedSerial, SerialNum newestFlushedSerial, - ConfigStore &config_store) + ConfigStore &config_store, + const ReplayThrottlingPolicy& replay_throttling_policy) { (void) newestFlushedSerial; assert(_activeFeedView); assert(_bucketDBHandler); auto state = make_shared<ReplayTransactionLogState> - (getDocTypeName(), _activeFeedView, *_bucketDBHandler, _replayConfig, config_store, *this); + (getDocTypeName(), _activeFeedView, *_bucketDBHandler, _replayConfig, config_store, replay_throttling_policy, *this); changeFeedState(state); // Resurrected attribute vector might cause oldestFlushedSerial to // be lower than _prunedSerialNum, so don't warn for now. diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 417d9c21548..32a70f7c2b0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -36,6 +36,7 @@ class IReplayConfig; class JoinBucketsOperation; class PutOperation; class RemoveOperation; +class ReplayThrottlingPolicy; class SplitBucketOperation; class UpdateOperation; @@ -189,7 +190,8 @@ public: SerialNum flushedSummaryMgrSerial, SerialNum oldestFlushedSerial, SerialNum newestFlushedSerial, - ConfigStore &config_store); + ConfigStore &config_store, + const ReplayThrottlingPolicy& replay_throttling_policy); /** * Called when a flush is done and allows pruning of the transaction log. diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index d2626a0d9f4..d9fdee85e4a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -5,11 +5,14 @@ #include "ifeedview.h" #include "ireplayconfig.h" #include "replaypacketdispatcher.h" +#include "replay_throttling_policy.h" #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> +#include <vespa/searchcore/proton/common/replay_feedtoken_state.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/shared_operation_throttler.h> #include <cassert> #include <vespa/log/log.h> @@ -21,6 +24,7 @@ using search::SerialNum; using vespalib::Executor; using vespalib::makeLambdaTask; using vespalib::IDestructorCallback; +using vespalib::SharedOperationThrottler; using vespalib::make_string; using proton::bucketdb::IBucketDBHandler; @@ -52,31 +56,47 @@ class TransactionLogReplayPacketHandler : public IReplayPacketHandler { FeedConfigStore &_config_store; IIncSerialNum &_inc_serial_num; CommitTimeTracker _commitTimeTracker; + std::unique_ptr<SharedOperationThrottler> _throttler; + + static std::unique_ptr<SharedOperationThrottler> make_throttler(const ReplayThrottlingPolicy& replay_throttling_policy) { + auto& params = replay_throttling_policy.get_params(); + if (!params.has_value()) { + return SharedOperationThrottler::make_unlimited_throttler(); + } + return SharedOperationThrottler::make_dynamic_throttler(params.value()); + } public: TransactionLogReplayPacketHandler(IFeedView *& feed_view_ptr, IBucketDBHandler &bucketDBHandler, IReplayConfig &replay_config, FeedConfigStore &config_store, + const ReplayThrottlingPolicy& replay_throttling_policy, IIncSerialNum &inc_serial_num) : _feed_view_ptr(feed_view_ptr), _bucketDBHandler(bucketDBHandler), _replay_config(replay_config), _config_store(config_store), _inc_serial_num(inc_serial_num), - _commitTimeTracker(5ms) + _commitTimeTracker(5ms), + _throttler(make_throttler(replay_throttling_policy)) { } ~TransactionLogReplayPacketHandler() override = default; + FeedToken make_replay_feed_token() { + SharedOperationThrottler::Token throttler_token = _throttler->blocking_acquire_one(); + return std::make_shared<feedtoken::ReplayState>(std::move(throttler_token)); + } + void replay(const PutOperation &op) override { - _feed_view_ptr->handlePut(FeedToken(), op); + _feed_view_ptr->handlePut(make_replay_feed_token(), op); } void replay(const RemoveOperation &op) override { - _feed_view_ptr->handleRemove(FeedToken(), op); + _feed_view_ptr->handleRemove(make_replay_feed_token(), op); } void replay(const UpdateOperation &op) override { - _feed_view_ptr->handleUpdate(FeedToken(), op); + _feed_view_ptr->handleUpdate(make_replay_feed_token(), op); } void replay(const NoopOperation &) override {} // ignored void replay(const NewConfigOperation &op) override { @@ -84,7 +104,7 @@ public: } void replay(const DeleteBucketOperation &op) override { - _feed_view_ptr->handleDeleteBucket(op, IDestructorCallback::SP()); + _feed_view_ptr->handleDeleteBucket(op, make_replay_feed_token()); } void replay(const SplitBucketOperation &op) override { _bucketDBHandler.handleSplit(op.getSerialNum(), op.getSource(), @@ -95,15 +115,15 @@ public: op.getSource2(), op.getTarget()); } void replay(const PruneRemovedDocumentsOperation &op) override { - _feed_view_ptr->handlePruneRemovedDocuments(op, IDestructorCallback::SP()); + _feed_view_ptr->handlePruneRemovedDocuments(op, make_replay_feed_token()); } void replay(const MoveOperation &op) override { - _feed_view_ptr->handleMove(op, IDestructorCallback::SP()); + _feed_view_ptr->handleMove(op, make_replay_feed_token()); } void replay(const CreateBucketOperation &) override { } void replay(const CompactLidSpaceOperation &op) override { - _feed_view_ptr->handleCompactLidSpace(op, IDestructorCallback::SP()); + _feed_view_ptr->handleCompactLidSpace(op, make_replay_feed_token()); } NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() override { return _config_store; @@ -173,10 +193,11 @@ ReplayTransactionLogState::ReplayTransactionLogState( IBucketDBHandler &bucketDBHandler, IReplayConfig &replay_config, FeedConfigStore &config_store, + const ReplayThrottlingPolicy &replay_throttling_policy, IIncSerialNum& inc_serial_num) : FeedState(REPLAY_TRANSACTION_LOG), _doc_type_name(name), - _packet_handler(std::make_unique<TransactionLogReplayPacketHandler>(feed_view_ptr, bucketDBHandler, replay_config, config_store, inc_serial_num)) + _packet_handler(std::make_unique<TransactionLogReplayPacketHandler>(feed_view_ptr, bucketDBHandler, replay_config, config_store, replay_throttling_policy, inc_serial_num)) { } ReplayTransactionLogState::~ReplayTransactionLogState() = default; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h index 7533d6443a7..fcff28fe481 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h @@ -55,6 +55,7 @@ public: bucketdb::IBucketDBHandler &bucketDBHandler, IReplayConfig &replay_config, FeedConfigStore &config_store, + const ReplayThrottlingPolicy &replay_throttling_policy, IIncSerialNum &inc_serial_num); ~ReplayTransactionLogState() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 9a525731d0d..34369a0803e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -68,7 +68,6 @@ fsyncFile(const vespalib::string &fileName) if (!f.Sync()) { LOG(error, "Could not fsync file '%s'", fileName.c_str()); } - f.Close(); } template <class Config> @@ -131,7 +130,6 @@ ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &ful _content.resize(fileSize); file.ReadBuf(&_content[0], fileSize); _modTime = file.GetModificationTime(); - file.Close(); } nbostream & diff --git a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp index e317c86df05..c0d8168ab61 100644 --- a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp @@ -4,7 +4,7 @@ namespace proton { -JobTrackedMaintenanceJob::JobTrackedMaintenanceJob(IJobTracker::SP tracker, +JobTrackedMaintenanceJob::JobTrackedMaintenanceJob(std::shared_ptr<IJobTracker> tracker, IMaintenanceJob::SP job) : IMaintenanceJob(job->getName(), job->getDelay(), job->getInterval()), _tracker(std::move(tracker)), diff --git a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h index 1953393bad7..20ecfdf023d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h @@ -12,12 +12,14 @@ namespace proton { class JobTrackedMaintenanceJob final : public IMaintenanceJob { private: - IJobTracker::SP _tracker; - IMaintenanceJob::SP _job; - bool _running; + std::shared_ptr<IJobTracker> _tracker; + IMaintenanceJob::SP _job; + bool _running; public: - JobTrackedMaintenanceJob(IJobTracker::SP tracker, IMaintenanceJob::SP job); + JobTrackedMaintenanceJob(std::shared_ptr<IJobTracker> tracker, IMaintenanceJob::SP job); + JobTrackedMaintenanceJob(const JobTrackedMaintenanceJob &) = delete; + JobTrackedMaintenanceJob & operator = (const JobTrackedMaintenanceJob &) = delete; ~JobTrackedMaintenanceJob() override; bool isBlocked() const override { return _job->isBlocked(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index f4b92876891..49b301da26e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -17,7 +17,7 @@ namespace proton { namespace { IMaintenanceJob::UP -trackJob(IJobTracker::SP tracker, std::shared_ptr<IMaintenanceJob> job) +trackJob(std::shared_ptr<IJobTracker> tracker, std::shared_ptr<IMaintenanceJob> job) { return std::make_unique<JobTrackedMaintenanceJob>(std::move(tracker), std::move(job)); } @@ -28,7 +28,7 @@ injectLidSpaceCompactionJobs(MaintenanceController &controller, storage::spi::BucketExecutor & bucketExecutor, ILidSpaceCompactionHandler::Vector lscHandlers, IOperationStorer &opStorer, - IJobTracker::SP tracker, + std::shared_ptr<IJobTracker> tracker, IDiskMemUsageNotifier &diskMemUsageNotifier, IClusterStateChangedNotifier &clusterStateChangedNotifier, const std::shared_ptr<IBucketStateCalculator> &calc, @@ -89,7 +89,8 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); - controller.registerJobInDefaultPool(std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval())); + controller.registerJobInSharedExecutor( + std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval())); const auto & docTypeName = controller.getDocTypeName().getName(); const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index 0d75464a161..fa4bae8f01b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -39,12 +39,12 @@ isRunnable(const MaintenanceJobRunner & job, const Executor * master) { } -MaintenanceController::MaintenanceController(ISyncableThreadService &masterThread, - vespalib::Executor & defaultExecutor, - MonitoredRefCount & refCount, - const DocTypeName &docTypeName) +MaintenanceController::MaintenanceController(ISyncableThreadService& masterThread, + vespalib::Executor& shared_executor, + MonitoredRefCount& refCount, + const DocTypeName& docTypeName) : _masterThread(masterThread), - _defaultExecutor(defaultExecutor), + _shared_executor(shared_executor), _refCount(refCount), _readySubDB(), _remSubDB(), @@ -70,10 +70,10 @@ MaintenanceController::registerJobInMasterThread(IMaintenanceJob::UP job) } void -MaintenanceController::registerJobInDefaultPool(IMaintenanceJob::UP job) +MaintenanceController::registerJobInSharedExecutor(IMaintenanceJob::UP job) { // Called by master write thread - registerJob(_defaultExecutor, std::move(job)); + registerJob(_shared_executor, std::move(job)); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h index f2c425b2fd0..086f5a36404 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h @@ -42,11 +42,12 @@ public: using UP = std::unique_ptr<MaintenanceController>; enum class State {INITIALIZING, STARTED, PAUSED, STOPPING}; - MaintenanceController(ISyncableThreadService &masterThread, vespalib::Executor & defaultExecutor, vespalib::MonitoredRefCount & refCount, const DocTypeName &docTypeName); + MaintenanceController(ISyncableThreadService& masterThread, vespalib::Executor& shared_executor, + vespalib::MonitoredRefCount& refCount, const DocTypeName& docTypeName); ~MaintenanceController(); void registerJobInMasterThread(IMaintenanceJob::UP job); - void registerJobInDefaultPool(IMaintenanceJob::UP job); + void registerJobInSharedExecutor(IMaintenanceJob::UP job); void killJobs(); @@ -82,7 +83,7 @@ private: using Guard = std::lock_guard<Mutex>; ISyncableThreadService &_masterThread; - vespalib::Executor &_defaultExecutor; + vespalib::Executor &_shared_executor; vespalib::MonitoredRefCount &_refCount; MaintenanceDocumentSubDB _readySubDB; MaintenanceDocumentSubDB _remSubDB; diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp index 9eb0596ff1f..5e6cee94292 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "maintenancejobrunner.h" -#include <vespa/vespalib/util/lambdatask.h> #include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.maintenancejobrunner"); +using vespalib::CpuUsage; using vespalib::Executor; using vespalib::makeLambdaTask; @@ -34,7 +36,8 @@ MaintenanceJobRunner::addExecutorTask() Guard guard(_lock); if (!_stopped && !_job->isBlocked() && !_queued) { _queued = true; - _executor.execute(makeLambdaTask([this]() { runJobInExecutor(); })); + auto task = makeLambdaTask([this]() { runJobInExecutor(); }); + _executor.execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::COMPACT)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp index 51243b718ec..50a499c8a73 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp @@ -13,9 +13,10 @@ namespace { bool shouldUseConservativeMode(const ResourceUsageState &resourceState, bool currentlyUseConservativeMode, + double high_watermark_factor, double lowWatermarkFactor) { - return resourceState.aboveLimit() || + return resourceState.aboveLimit(high_watermark_factor) || (currentlyUseConservativeMode && resourceState.aboveLimit(lowWatermarkFactor)); } @@ -25,6 +26,7 @@ void MemoryFlushConfigUpdater::considerUseConservativeDiskMode(const LockGuard &guard, MemoryFlush::Config &newConfig) { if (shouldUseConservativeMode(_currState.diskState(), _useConservativeDiskMode, + _currConfig.conservative.highwatermarkfactor, _currConfig.conservative.lowwatermarkfactor)) { newConfig.maxGlobalTlsSize = _currConfig.maxtlssize * _currConfig.conservative.disklimitfactor; @@ -41,6 +43,7 @@ void MemoryFlushConfigUpdater::considerUseConservativeMemoryMode(const LockGuard &, MemoryFlush::Config &newConfig) { if (shouldUseConservativeMode(_currState.memoryState(), _useConservativeMemoryMode, + _currConfig.conservative.highwatermarkfactor, _currConfig.conservative.lowwatermarkfactor)) { newConfig.maxGlobalMemory = _currConfig.maxmemory * _currConfig.conservative.memorylimitfactor; diff --git a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp index 53b12972c44..9b1c6de15df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp @@ -1,14 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "operationdonecontext.h" +#include <vespa/searchcore/proton/common/feedtoken.h> namespace proton { -OperationDoneContext::OperationDoneContext(IDestructorCallback::SP token) - : _token(std::move(token)) +OperationDoneContext::OperationDoneContext(std::shared_ptr<feedtoken::IState> token, std::shared_ptr<vespalib::IDestructorCallback> done_callback) + : _token(std::move(token)), + _done_callback(std::move(done_callback)) { } OperationDoneContext::~OperationDoneContext() = default; +bool +OperationDoneContext::is_replay() const +{ + return (!_token || _token->is_replay()); +} + + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h index 113aae8c9bf..baf0e98d5a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h @@ -4,6 +4,8 @@ #include <vespa/vespalib/util/idestructorcallback.h> +namespace proton::feedtoken { class IState; } + namespace proton { /** @@ -16,13 +18,13 @@ namespace proton { class OperationDoneContext : public vespalib::IDestructorCallback { public: - using IDestructorCallback = vespalib::IDestructorCallback; - OperationDoneContext(IDestructorCallback::SP token); + OperationDoneContext(std::shared_ptr<feedtoken::IState> token, std::shared_ptr<IDestructorCallback> done_callback); ~OperationDoneContext() override; - bool hasToken() const { return static_cast<bool>(_token); } + bool is_replay() const; private: - IDestructorCallback::SP _token; + std::shared_ptr<feedtoken::IState> _token; + std::shared_ptr<vespalib::IDestructorCallback> _done_callback; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index b128fe16e5e..6001e6b88d4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -24,10 +24,10 @@ #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/flushengine/tls_stats_factory.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> +#include <vespa/searchcore/proton/metrics/content_proton_metrics.h> +#include <vespa/searchcore/proton/metrics/metrics_engine.h> #include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> -#include <vespa/searchcore/proton/metrics/metrics_engine.h> -#include <vespa/searchcore/proton/metrics/content_proton_metrics.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/transactionlog/trans_log_server_explorer.h> @@ -36,13 +36,13 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/state_server.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/random.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/invokeserviceimpl.h> #ifdef __linux__ #include <malloc.h> #endif @@ -54,20 +54,22 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.server.proton"); +using CpuCategory = vespalib::CpuUsage::Category; + using document::DocumentTypeRepo; +using search::engine::MonitorReply; +using search::transactionlog::DomainStats; +using vespa::config::search::core::ProtonConfig; +using vespa::config::search::core::internal::InternalProtonType; +using vespalib::CpuUsage; using vespalib::FileHeader; using vespalib::IllegalStateException; using vespalib::Slime; +using vespalib::compression::CompressionConfig; using vespalib::makeLambdaTask; using vespalib::slime::ArrayInserter; using vespalib::slime::Cursor; -using search::transactionlog::DomainStats; -using vespa::config::search::core::ProtonConfig; -using vespa::config::search::core::internal::InternalProtonType; -using vespalib::compression::CompressionConfig; -using search::engine::MonitorReply; - namespace proton { namespace { @@ -139,8 +141,8 @@ struct MetricsUpdateHook : metrics::UpdateHook const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; -VESPA_THREAD_STACK_TAG(initialize_executor) -VESPA_THREAD_STACK_TAG(close_executor) +VESPA_THREAD_STACK_TAG(proton_initialize_executor) +VESPA_THREAD_STACK_TAG(proton_close_executor) } @@ -207,6 +209,7 @@ Proton::Proton(const config::ConfigUri & configUri, StatusProducer(), IPersistenceEngineOwner(), ComponentConfigProducer(), + _cpu_util(), _configUri(configUri), _mutex(), _metricsHook(std::make_unique<MetricsUpdateHook>(*this)), @@ -310,7 +313,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _protonDiskLayout = std::make_unique<ProtonDiskLayout>(protonConfig.basedir, protonConfig.tlsspec); vespalib::chdir(protonConfig.basedir); vespalib::alloc::MmapFileAllocatorFactory::instance().setup(protonConfig.basedir + "/swapdirs"); - _tls->start(); + _tls->start(hwInfo.cpu().cores()); _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), strategy, flush.maxconcurrent, vespalib::from_s(flush.idleinterval)); _metricsEngine->addExternalMetrics(_summaryEngine->getMetrics()); @@ -329,7 +332,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_shared_service->shared_raw()); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { - initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128_Ki, initialize_executor); + initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128_Ki, + CpuUsage::wrap(proton_initialize_executor, CpuCategory::SETUP)); _initDocumentDbsInSequence = (protonConfig.initialize.threads == 1); } _protonConfigurer.applyInitialConfig(initializeThreads); @@ -463,7 +467,8 @@ Proton::~Proton() } } - vespalib::ThreadStackExecutor closePool(std::min(_documentDBMap.size(), numCores), 0x20000, close_executor); + vespalib::ThreadStackExecutor closePool(std::min(_documentDBMap.size(), numCores), 0x20000, + CpuUsage::wrap(proton_close_executor, CpuCategory::SETUP)); closeDocumentDBs(closePool); } _documentDBMap.clear(); @@ -753,12 +758,20 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter(); auto dm_metrics = usageFilter.get_metrics(); - metrics.resourceUsage.disk.set(dm_metrics.get_disk_usage()); - metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization()); - metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); - metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); - metrics.resourceUsage.transient_memory.set(dm_metrics.get_transient_memory_usage()); - metrics.resourceUsage.transient_disk.set(dm_metrics.get_transient_disk_usage()); + metrics.resourceUsage.disk.set(dm_metrics.non_transient_disk_usage()); + metrics.resourceUsage.diskUtilization.set(dm_metrics.total_disk_utilization()); + metrics.resourceUsage.transient_disk.set(dm_metrics.transient_disk_usage()); + metrics.resourceUsage.disk_usage.total.set(dm_metrics.total_disk_usage()); + metrics.resourceUsage.disk_usage.total_util.set(dm_metrics.total_disk_utilization()); + metrics.resourceUsage.disk_usage.transient.set(dm_metrics.transient_disk_usage()); + + metrics.resourceUsage.memory.set(dm_metrics.non_transient_memory_usage()); + metrics.resourceUsage.memoryUtilization.set(dm_metrics.total_memory_utilization()); + metrics.resourceUsage.transient_memory.set(dm_metrics.transient_memory_usage()); + metrics.resourceUsage.memory_usage.total.set(dm_metrics.total_memory_usage()); + metrics.resourceUsage.memory_usage.total_util.set(dm_metrics.total_memory_utilization()); + metrics.resourceUsage.memory_usage.transient.set(dm_metrics.transient_memory_usage()); + metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); @@ -775,6 +788,12 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) #else metrics.resourceUsage.mallocArena.set(UINT64_C(0)); #endif + auto cpu_util = _cpu_util.get_util(); + metrics.resourceUsage.cpu_util.setup.set(cpu_util[CpuCategory::SETUP]); + metrics.resourceUsage.cpu_util.read.set(cpu_util[CpuCategory::READ]); + metrics.resourceUsage.cpu_util.write.set(cpu_util[CpuCategory::WRITE]); + metrics.resourceUsage.cpu_util.compact.set(cpu_util[CpuCategory::COMPACT]); + metrics.resourceUsage.cpu_util.other.set(cpu_util[CpuCategory::OTHER]); } { ContentProtonMetrics::ProtonExecutorMetrics &metrics = _metricsEngine->root().executor; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 573f215c722..90a257a0aaa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -25,6 +25,7 @@ #include <vespa/vespalib/net/json_handler_repo.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <mutex> #include <shared_mutex> @@ -81,6 +82,7 @@ private: void setClusterName(const vespalib::string &clusterName, const vespalib::string &baseDir); }; + vespalib::CpuUtil _cpu_util; const config::ConfigUri _configUri; mutable std::shared_mutex _mutex; std::unique_ptr<metrics::UpdateHook> _metricsHook; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 56a4d1b8c9f..e6d036df7d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -2,14 +2,13 @@ #pragma once -#include <vespa/fastos/thread.h> -#include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/config/config.h> #include "bootstrapconfigmanager.h" #include "documentdbconfigmanager.h" #include "i_document_db_config_owner.h" -#include <chrono> +#include <vespa/fastos/thread.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/config/retriever/configretriever.h> +#include <vespa/config/subscription/configuri.h> namespace document { class DocumentTypeRepo; } diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index 7801f21c906..695f39c0097 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -9,9 +9,11 @@ using document::Document; namespace proton { -PutDoneContext::PutDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, +PutDoneContext::PutDoneContext(std::shared_ptr<feedtoken::IState> token, + std::shared_ptr<vespalib::IDestructorCallback> done_callback, + IPendingLidTracker::Token uncommitted, std::shared_ptr<const Document> doc, uint32_t lid) - : OperationDoneContext(std::move(token)), + : OperationDoneContext(std::move(token), std::move(done_callback)), _uncommitted(std::move(uncommitted)), _lid(lid), _docIdLimit(nullptr), diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h index 00e8ac46c93..e0f55816314 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h @@ -28,7 +28,9 @@ class PutDoneContext : public OperationDoneContext std::shared_ptr<const document::Document> _doc; public: - PutDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, + PutDoneContext(std::shared_ptr<feedtoken::IState> token, + std::shared_ptr<vespalib::IDestructorCallback> done_callback, + IPendingLidTracker::Token uncommitted, std::shared_ptr<const document::Document> doc, uint32_t lid); ~PutDoneContext() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index 558a57bb8c6..0c93992d427 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -4,8 +4,8 @@ namespace proton { -RemoveDoneContext::RemoveDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted) - : OperationDoneContext(std::move(token)), +RemoveDoneContext::RemoveDoneContext(std::shared_ptr<feedtoken::IState> token, std::shared_ptr<IDestructorCallback> done_callback, IPendingLidTracker::Token uncommitted) + : OperationDoneContext(std::move(token), std::move(done_callback)), _uncommitted(std::move(uncommitted)) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h index 28e15389bb2..62db0f20b84 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h @@ -19,7 +19,7 @@ class RemoveDoneContext : public OperationDoneContext IPendingLidTracker::Token _uncommitted; public: - RemoveDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted); + RemoveDoneContext(std::shared_ptr<feedtoken::IState>, std::shared_ptr<vespalib::IDestructorCallback> done_callback, IPendingLidTracker::Token uncommitted); ~RemoveDoneContext() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/replay_throttling_policy.h b/searchcore/src/vespa/searchcore/proton/server/replay_throttling_policy.h new file mode 100644 index 00000000000..3e1485d94dd --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/replay_throttling_policy.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/shared_operation_throttler.h> +#include <optional> + +namespace proton { + +/* + * Policy for transaction log replay throttling. If params are set then a dynamic throttler + * is used, otherwise an unlimited throttler is used. + */ +class ReplayThrottlingPolicy +{ + using DynamicThrottleParams = vespalib::SharedOperationThrottler::DynamicThrottleParams; + std::optional<DynamicThrottleParams> _params; + +public: + explicit ReplayThrottlingPolicy(std::optional<DynamicThrottleParams> params) + : _params(std::move(params)) + { + } + const std::optional<DynamicThrottleParams>& get_params() const noexcept { return _params; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 66b1ba1ae2e..207f1d813d8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -99,8 +99,8 @@ SearchableFeedView::updateIndexedFields(SerialNum serialNum, search::DocumentIdT { _writeService.index().execute( makeLambdaTask([serialNum, lid, futureDoc = std::move(futureDoc), - onWriteDone = std::move(onWriteDone), this]() mutable { - performIndexPut(serialNum, lid, std::move(futureDoc), std::move(onWriteDone)); + onWriteDone, this]() mutable { + performIndexPut(serialNum, lid, std::move(futureDoc), onWriteDone); })); } diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp index e32cd6f5f4e..7e799e506e3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -2,10 +2,12 @@ #include "shared_threading_service.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/invokeserviceimpl.h> + +using vespalib::CpuUsage; VESPA_THREAD_STACK_TAG(proton_field_writer_executor) VESPA_THREAD_STACK_TAG(proton_shared_executor) @@ -16,7 +18,7 @@ namespace proton { using SharedFieldWriterExecutor = ThreadingServiceConfig::ProtonConfig::Feeding::SharedFieldWriterExecutor; SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfig& cfg) - : _warmup(cfg.warmup_threads(), 128_Ki, proton_warmup_executor), + : _warmup(cfg.warmup_threads(), 128_Ki, CpuUsage::wrap(proton_warmup_executor, CpuUsage::Category::COMPACT)), _shared(std::make_shared<vespalib::BlockingThreadStackExecutor>(cfg.shared_threads(), 128_Ki, cfg.shared_task_limit(), proton_shared_executor)), _field_writer(), @@ -25,9 +27,10 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi { const auto& fw_cfg = cfg.field_writer_config(); if (fw_cfg.shared_field_writer() == SharedFieldWriterExecutor::DOCUMENT_DB) { - _field_writer = vespalib::SequencedTaskExecutor::create(proton_field_writer_executor, + _field_writer = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(proton_field_writer_executor, CpuUsage::Category::WRITE), fw_cfg.indexingThreads() * 3, fw_cfg.defaultTaskLimit(), + fw_cfg.is_task_limit_hard(), fw_cfg.optimize(), fw_cfg.kindOfwatermark()); if (fw_cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT) { diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp index 8a81c3f4388..76b7982fedd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -30,6 +30,11 @@ derive_shared_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) return std::max(scaled_cores, cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); } +size_t +derive_warmup_threads(const HwInfo::Cpu& cpu_info) { + return std::max(1u, std::min(4u, cpu_info.cores()/8)); +} + } SharedThreadingServiceConfig @@ -37,7 +42,7 @@ SharedThreadingServiceConfig::make(const proton::SharedThreadingServiceConfig::P const proton::HwInfo::Cpu& cpu_info) { size_t shared_threads = derive_shared_threads(cfg, cpu_info); - return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, 4, + return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, derive_warmup_threads(cpu_info), ThreadingServiceConfig::make(cfg, cfg.feeding.concurrency, cpu_info)); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 97bd940b403..170b6f99930 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -15,9 +15,10 @@ #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/destructor_callbacks.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.storeonlyfeedview"); @@ -25,28 +26,29 @@ LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; using document::Document; using document::DocumentId; -using document::GlobalId; using document::DocumentTypeRepo; using document::DocumentUpdate; -using vespalib::IDestructorCallback; +using document::GlobalId; +using proton::documentmetastore::LidReuseDelayer; using search::SerialNum; using search::index::Schema; using storage::spi::BucketInfoResult; using storage::spi::Timestamp; +using vespalib::CpuUsage; +using vespalib::IDestructorCallback; using vespalib::IllegalStateException; using vespalib::makeLambdaTask; using vespalib::make_string; -using proton::documentmetastore::LidReuseDelayer; namespace proton { namespace { std::shared_ptr<PutDoneContext> -createPutDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, +createPutDoneContext(FeedToken token, std::shared_ptr<IDestructorCallback> done_callback, IPendingLidTracker::Token uncommitted, std::shared_ptr<const Document> doc, uint32_t lid) { - return std::make_shared<PutDoneContext>(std::move(token), std::move(uncommitted), std::move(doc), lid); + return std::make_shared<PutDoneContext>(std::move(token), std::move(done_callback), std::move(uncommitted), std::move(doc), lid); } std::shared_ptr<UpdateDoneContext> @@ -66,9 +68,9 @@ void setPrev(DocumentOperation &op, const documentmetastore::IStore::Result &res } std::shared_ptr<RemoveDoneContext> -createRemoveDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted) +createRemoveDoneContext(FeedToken token, std::shared_ptr<IDestructorCallback> done_callback, IPendingLidTracker::Token uncommitted) { - return std::make_shared<RemoveDoneContext>(std::move(token), std::move(uncommitted)); + return std::make_shared<RemoveDoneContext>(std::move(token), std::move(done_callback), std::move(uncommitted)); } class SummaryPutDoneContext : public OperationDoneContext @@ -80,7 +82,7 @@ public: }; SummaryPutDoneContext::SummaryPutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted) - : OperationDoneContext(std::move(token)), + : OperationDoneContext(std::move(token), {}), _uncommitted(std::move(uncommitted)) {} @@ -247,9 +249,19 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) if (putOp.getValidDbdId(_params._subDbId)) { if (putOp.changedDbdId() && useDocumentMetaStore(serialNum)) { - _gidToLidChangeHandler.notifyPut(token, docId.getGlobalId(), putOp.getLid(), serialNum); + /* + * Don't pass replay feed token to GidToLidChangeHandler. + * + * The passed feed token is kept until the ForceCommitDoneTask scheduled by the next + * force commit has completed. If a replay feed token containing an active throttler + * token is passed to GidToLidChangeHandler then + * TransactionLogReplayFeedHandler::make_replay_feed_token() might deadlock, waiting for + * active throttler tokens to be destroyed. + */ + FeedToken token_copy = (token && !token->is_replay()) ? token : FeedToken(); + _gidToLidChangeHandler.notifyPut(std::move(token_copy), docId.getGlobalId(), putOp.getLid(), serialNum); } - auto onWriteDone = createPutDoneContext(std::move(token), get_pending_lid_token(putOp), doc, putOp.getLid()); + auto onWriteDone = createPutDoneContext(std::move(token), {}, get_pending_lid_token(putOp), doc, putOp.getLid()); putSummary(serialNum, putOp.getLid(), doc, onWriteDone); putAttributes(serialNum, putOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, putOp.getLid(), doc, onWriteDone); @@ -257,7 +269,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) if (docAlreadyExists && putOp.changedDbdId()) { //TODO, better to have an else than an assert ? assert(!putOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), _pendingLidsForCommit->produce(putOp.getPrevLid()), serialNum, putOp.getPrevLid()); + internalRemove(std::move(token), {}, _pendingLidsForCommit->produce(putOp.getPrevLid()), serialNum, putOp.getPrevLid()); } } @@ -357,7 +369,7 @@ StoreOnlyFeedView::removeSummaries(SerialNum serialNum, const LidVector & lids, trackerTokens.emplace_back(_pendingLidsForDocStore.produce(lid)); }); summaryExecutor().execute( - makeLambdaTask([serialNum, lids = std::move(lids), onDone, trackerTokens = std::move(trackerTokens), this] { + makeLambdaTask([serialNum, lids, onDone, trackerTokens = std::move(trackerTokens), this] { (void) onDone; (void) trackerTokens; std::for_each(lids.begin(), lids.end(), [this, serialNum](Lid lid) { @@ -439,13 +451,14 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) } else { putSummaryNoop(std::move(futureStream), onWriteDone); } - _writeService.shared().execute(makeLambdaTask( - [upd = updOp.getUpdate(), useDocStore, lid, onWriteDone, promisedDoc = std::move(promisedDoc), - promisedStream = std::move(promisedStream), this]() mutable - { - makeUpdatedDocument(useDocStore, lid, *upd, onWriteDone, - std::move(promisedDoc), std::move(promisedStream)); - })); + auto task = makeLambdaTask([upd = updOp.getUpdate(), useDocStore, lid, onWriteDone, + promisedDoc = std::move(promisedDoc), + promisedStream = std::move(promisedStream), this]() mutable + { + makeUpdatedDocument(useDocStore, lid, *upd, onWriteDone, + std::move(promisedDoc), std::move(promisedStream)); + }); + _writeService.shared().execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::WRITE)); updateAttributes(serialNum, lid, std::move(futureDoc), onWriteDone); } } @@ -458,13 +471,13 @@ StoreOnlyFeedView::makeUpdatedDocument(bool useDocStore, Lid lid, const Document Document::UP prevDoc = _summaryAdapter->get(lid, *_repo); Document::UP newDoc; vespalib::nbostream newStream(12345); - assert(!onWriteDone->hasToken() || useDocStore); + assert(onWriteDone->is_replay() || useDocStore); if (useDocStore) { assert(prevDoc); } if (!prevDoc) { // Replaying, document removed later before summary was flushed. - assert(!onWriteDone->hasToken()); + assert(onWriteDone->is_replay()); // If we've passed serial number for flushed index then we could // also check that this operation is marked for ignore by index // proxy. @@ -478,7 +491,7 @@ StoreOnlyFeedView::makeUpdatedDocument(bool useDocStore, Lid lid, const Document } else { // Replaying, document removed and lid reused before summary // was flushed. - assert(!onWriteDone->hasToken() && !useDocStore); + assert(onWriteDone->is_replay() && !useDocStore); } } promisedDoc.set_value(std::move(newDoc)); @@ -552,7 +565,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI if (rmOp.changedDbdId()) { //TODO Prefer else over assert ? assert(!rmOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid()); + internalRemove(std::move(token), {}, _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid()); } } } @@ -569,16 +582,16 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid if (rmOp.getValidPrevDbdId(_params._subDbId)) { if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid()); + internalRemove(std::move(token), {}, _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid()); } } } void -StoreOnlyFeedView::internalRemove(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, Lid lid) +StoreOnlyFeedView::internalRemove(FeedToken token, std::shared_ptr<IDestructorCallback> done_callback, IPendingLidTracker::Token uncommitted, SerialNum serialNum, Lid lid) { _lidReuseDelayer.delayReuse(lid); - auto onWriteDone = createRemoveDoneContext(std::move(token), std::move(uncommitted)); + auto onWriteDone = createRemoveDoneContext(std::move(token), std::move(done_callback), std::move(uncommitted)); removeSummary(serialNum, lid, onWriteDone); removeAttributes(serialNum, lid, onWriteDone); removeIndexedFields(serialNum, lid, onWriteDone); @@ -706,13 +719,13 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, DoneCallback doneCtx) if (moveOp.changedDbdId() && useDocumentMetaStore(serialNum)) { _gidToLidChangeHandler.notifyPut(FeedToken(), docId.getGlobalId(), moveOp.getLid(), serialNum); } - auto onWriteDone = createPutDoneContext(doneCtx, _pendingLidsForCommit->produce(moveOp.getLid()), doc, moveOp.getLid()); + auto onWriteDone = createPutDoneContext({}, doneCtx, _pendingLidsForCommit->produce(moveOp.getLid()), doc, moveOp.getLid()); putSummary(serialNum, moveOp.getLid(), doc, onWriteDone); putAttributes(serialNum, moveOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, moveOp.getLid(), doc, onWriteDone); } if (docAlreadyExists && moveOp.changedDbdId()) { - internalRemove(doneCtx, _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, moveOp.getPrevLid()); + internalRemove({}, doneCtx, _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, moveOp.getPrevLid()); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index c25accaf4a4..bd8509fa796 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -180,7 +180,7 @@ private: // returns the number of documents removed. size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields, DoneCallback onDone); - void internalRemove(IDestructorCallbackSP token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, Lid lid); + void internalRemove(FeedToken token, std::shared_ptr<vespalib::IDestructorCallback> done_callback,IPendingLidTracker::Token uncommitted, SerialNum serialNum, Lid lid); IPendingLidTracker::Token get_pending_lid_token(const DocumentOperation &op); diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 7982e8a8414..335d5bab8d0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -12,14 +12,15 @@ using OptimizeFor = vespalib::Executor::OptimizeFor; ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t master_task_limit_, - uint32_t defaultTaskLimit_, + int32_t defaultTaskLimit_, OptimizeFor optimize_, uint32_t kindOfWatermark_, vespalib::duration reactionTime_, SharedFieldWriterExecutor shared_field_writer_) : _indexingThreads(indexingThreads_), _master_task_limit(master_task_limit_), - _defaultTaskLimit(defaultTaskLimit_), + _defaultTaskLimit(std::abs(defaultTaskLimit_)), + _is_task_limit_hard(defaultTaskLimit_ >= 0), _optimize(optimize_), _kindOfWatermark(kindOfWatermark_), _reactionTime(reactionTime_), @@ -81,6 +82,7 @@ ThreadingServiceConfig::operator==(const ThreadingServiceConfig &rhs) const return _indexingThreads == rhs._indexingThreads && _master_task_limit == rhs._master_task_limit && _defaultTaskLimit == rhs._defaultTaskLimit && + _is_task_limit_hard == rhs._is_task_limit_hard && _optimize == rhs._optimize && _kindOfWatermark == rhs._kindOfWatermark && _reactionTime == rhs._reactionTime && diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h index f1a4f0525d1..a54c0674263 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h @@ -23,15 +23,16 @@ private: uint32_t _indexingThreads; uint32_t _master_task_limit; uint32_t _defaultTaskLimit; + bool _is_task_limit_hard; OptimizeFor _optimize; uint32_t _kindOfWatermark; vespalib::duration _reactionTime; // Maximum reaction time to new tasks SharedFieldWriterExecutor _shared_field_writer; private: - ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t master_task_limit_, uint32_t defaultTaskLimit_, - OptimizeFor optimize_, uint32_t kindOfWatermark_, vespalib::duration reactionTime_, - SharedFieldWriterExecutor shared_field_writer_); + ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t master_task_limit_, int32_t defaultTaskLimit_, + OptimizeFor optimize_, uint32_t kindOfWatermark_, + vespalib::duration reactionTime_, SharedFieldWriterExecutor shared_field_writer_); public: static ThreadingServiceConfig make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo); @@ -40,6 +41,7 @@ public: uint32_t indexingThreads() const { return _indexingThreads; } uint32_t master_task_limit() const { return _master_task_limit; } uint32_t defaultTaskLimit() const { return _defaultTaskLimit; } + bool is_task_limit_hard() const { return _is_task_limit_hard; } OptimizeFor optimize() const { return _optimize; } uint32_t kindOfwatermark() const { return _kindOfWatermark; } vespalib::duration reactionTime() const { return _reactionTime; } diff --git a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp index 5842965a18a..16099700980 100644 --- a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp @@ -7,8 +7,8 @@ using document::Document; namespace proton { -UpdateDoneContext::UpdateDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, const document::DocumentUpdate::SP &upd) - : OperationDoneContext(std::move(token)), +UpdateDoneContext::UpdateDoneContext(std::shared_ptr<feedtoken::IState> token, IPendingLidTracker::Token uncommitted, const document::DocumentUpdate::SP &upd) + : OperationDoneContext(std::move(token), {}), _uncommitted(std::move(uncommitted)), _upd(upd), _doc() diff --git a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h index 62448102f30..7dd6c4a61f2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h @@ -24,7 +24,7 @@ class UpdateDoneContext : public OperationDoneContext document::DocumentUpdate::SP _upd; std::shared_future<std::unique_ptr<const document::Document>> _doc; public: - UpdateDoneContext(IDestructorCallback::SP token, IPendingLidTracker::Token uncommitted, const document::DocumentUpdate::SP &upd); + UpdateDoneContext(std::shared_ptr<feedtoken::IState> token, IPendingLidTracker::Token uncommitted, const document::DocumentUpdate::SP &upd); ~UpdateDoneContext() override; const document::DocumentUpdate &getUpdate() { return *_upd; } diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index 95643c7d1a8..0bf5b351fd0 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -2,6 +2,7 @@ #include "summaryengine.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/log/log.h> LOG_SETUP(".proton.summaryengine.summaryengine"); @@ -10,6 +11,7 @@ using namespace search::engine; using namespace proton; using vespalib::Memory; using vespalib::slime::Inspector; +using vespalib::CpuUsage; namespace { @@ -61,7 +63,7 @@ SummaryEngine::SummaryEngine(size_t numThreads, bool async) _closed(false), _forward_issues(true), _handlers(), - _executor(numThreads, 128_Ki, summary_engine_executor), + _executor(numThreads, 128_Ki, CpuUsage::wrap(summary_engine_executor, CpuUsage::Category::READ)), _metrics(std::make_unique<DocsumMetrics>()) { } @@ -141,6 +143,9 @@ SummaryEngine::getDocsums(DocsumRequest::UP req) } } updateDocsumMetrics(vespalib::to_s(req->getTimeUsed()), getNumDocs(*reply)); + if (req->expired()) { + vespalib::Issue::report("docsum request timed out; results may be incomplete"); + } } if (! reply) { reply = std::make_unique<DocsumReply>(); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 743cd9af8fc..0e2491e62fa 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -14,7 +14,7 @@ private: std::vector<search::AttributeVector*> _writables; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; vespalib::ISequencedTaskExecutor* _writer; - vespalib::ThreadExecutor* _shared; + vespalib::Executor* _shared; public: MockAttributeManager() @@ -33,7 +33,7 @@ public: void set_writer(vespalib::ISequencedTaskExecutor& writer) { _writer = &writer; } - void set_shared_executor(vespalib::ThreadExecutor& shared) { + void set_shared_executor(vespalib::Executor& shared) { _shared = &shared; } search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override { @@ -72,7 +72,7 @@ public: assert(_writer != nullptr); return *_writer; } - vespalib::ThreadExecutor& get_shared_executor() const override { + vespalib::Executor& get_shared_executor() const override { assert(_shared != nullptr); return *_shared; } diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h index e93b1632b3f..78a740ec2c3 100644 --- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h @@ -15,7 +15,7 @@ private: SyncableThreadServiceObserver _master; ThreadServiceObserver _index; ThreadExecutorObserver _summary; - vespalib::ThreadExecutor & _shared; + vespalib::Executor & _shared; vespalib::SequencedTaskExecutorObserver _indexFieldInverter; vespalib::SequencedTaskExecutorObserver _indexFieldWriter; vespalib::SequencedTaskExecutorObserver _attributeFieldWriter; @@ -46,7 +46,7 @@ public: vespalib::ThreadExecutor &summary() override { return _summary; } - vespalib::ThreadExecutor &shared() override { + vespalib::Executor &shared() override { return _shared; } vespalib::ISequencedTaskExecutor &indexFieldInverter() override { diff --git a/searchcorespi/CMakeLists.txt b/searchcorespi/CMakeLists.txt index 1029619150a..0bbaa813752 100644 --- a/searchcorespi/CMakeLists.txt +++ b/searchcorespi/CMakeLists.txt @@ -20,6 +20,6 @@ vespa_define_module( src/vespa/searchcorespi/index TESTS - src/tests/index/active_disk_indexes + src/tests/index/disk_indexes src/tests/index/index_disk_layout ) diff --git a/searchcorespi/src/tests/index/active_disk_indexes/CMakeLists.txt b/searchcorespi/src/tests/index/active_disk_indexes/CMakeLists.txt deleted file mode 100644 index e10ada381bf..00000000000 --- a/searchcorespi/src/tests/index/active_disk_indexes/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcorespi_active_disk_indexes_test_app - SOURCES - active_disk_indexes_test.cpp - DEPENDS - searchcorespi - GTest::GTest -) -vespa_add_test(NAME searchcorespi_active_disk_indexes_test_app COMMAND searchcorespi_active_disk_indexes_test_app) diff --git a/searchcorespi/src/tests/index/active_disk_indexes/active_disk_indexes_test.cpp b/searchcorespi/src/tests/index/active_disk_indexes/active_disk_indexes_test.cpp deleted file mode 100644 index a6d412817d9..00000000000 --- a/searchcorespi/src/tests/index/active_disk_indexes/active_disk_indexes_test.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchcorespi/index/activediskindexes.h> -#include <vespa/searchcorespi/index/index_disk_dir.h> -#include <vespa/vespalib/gtest/gtest.h> - -namespace searchcorespi::index { - -class ActiveDiskIndexesTest : public ::testing::Test, - public ActiveDiskIndexes -{ -protected: - ActiveDiskIndexesTest(); - ~ActiveDiskIndexesTest(); -}; - -ActiveDiskIndexesTest::ActiveDiskIndexesTest() - : ::testing::Test(), - ActiveDiskIndexes() -{ -} - -ActiveDiskIndexesTest::~ActiveDiskIndexesTest() = default; - -TEST_F(ActiveDiskIndexesTest, simple_set_active_works) -{ - EXPECT_FALSE(isActive("index.flush.1")); - setActive("index.flush.1"); - EXPECT_TRUE(isActive("index.flush.1")); - notActive("index.flush.1"); - EXPECT_FALSE(isActive("index.flush.1")); -} - -TEST_F(ActiveDiskIndexesTest, nested_set_active_works) -{ - setActive("index.flush.1"); - setActive("index.flush.1"); - EXPECT_TRUE(isActive("index.flush.1")); - notActive("index.flush.1"); - EXPECT_TRUE(isActive("index.flush.1")); - notActive("index.flush.1"); - EXPECT_FALSE(isActive("index.flush.1")); -} - -TEST_F(ActiveDiskIndexesTest, is_active_returns_false_for_bad_name) -{ - EXPECT_FALSE(isActive("foo/bar/baz")); - EXPECT_FALSE(isActive("index.flush.0")); -} - -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt b/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt new file mode 100644 index 00000000000..81b6bb0e8a9 --- /dev/null +++ b/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcorespi_disk_indexes_test_app + SOURCES + disk_indexes_test.cpp + DEPENDS + searchcorespi + GTest::GTest +) +vespa_add_test(NAME searchcorespi_disk_indexes_test_app COMMAND searchcorespi_disk_indexes_test_app) diff --git a/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp b/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp new file mode 100644 index 00000000000..d22ad499316 --- /dev/null +++ b/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp @@ -0,0 +1,196 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcorespi/index/disk_indexes.h> +#include <vespa/searchcorespi/index/index_disk_dir.h> +#include <vespa/searchcorespi/index/indexdisklayout.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <fstream> + +namespace { + +vespalib::string base_dir("base"); + +constexpr uint32_t block_size = 4_Ki; + +} + +namespace searchcorespi::index { + +class DiskIndexesTest : public ::testing::Test, + public DiskIndexes +{ + IndexDiskLayout _layout; +protected: + DiskIndexesTest(); + ~DiskIndexesTest(); + + static IndexDiskDir get_index_disk_dir(const vespalib::string& dir) { + return IndexDiskLayout::get_index_disk_dir(dir); + } + + void assert_transient_size(uint64_t exp, IndexDiskDir index_disk_dir) { + EXPECT_EQ(exp, get_transient_size(_layout, index_disk_dir)); + } +}; + +DiskIndexesTest::DiskIndexesTest() + : ::testing::Test(), + DiskIndexes(), + _layout(base_dir) +{ +} + +DiskIndexesTest::~DiskIndexesTest() = default; + +TEST_F(DiskIndexesTest, simple_set_active_works) +{ + EXPECT_FALSE(isActive("index.flush.1")); + setActive("index.flush.1", 0); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_FALSE(isActive("index.flush.1")); +} + +TEST_F(DiskIndexesTest, nested_set_active_works) +{ + setActive("index.flush.1", 0); + setActive("index.flush.1", 0); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_FALSE(isActive("index.flush.1")); +} + +TEST_F(DiskIndexesTest, is_active_returns_false_for_bad_name) +{ + EXPECT_FALSE(isActive("foo/bar/baz")); + EXPECT_FALSE(isActive("index.flush.0")); +} + +TEST_F(DiskIndexesTest, remove_works) +{ + EXPECT_TRUE(remove(IndexDiskDir())); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + EXPECT_TRUE(remove(fusion1)); + add_not_active(fusion1); + EXPECT_TRUE(remove(fusion1)); + setActive("index.fusion.1", 0); + EXPECT_FALSE(remove(fusion1)); + notActive("index.fusion.1"); + EXPECT_TRUE(remove(fusion1)); +} + +TEST_F(DiskIndexesTest, basic_get_transient_size_works) +{ + /* + * When starting to use a new fusion index, we have a transient + * period with two ISearchableIndexCollection instances: + * - old, containing index.fusion.1 and index.flush.2 + * - new, containing index.fusion.2 + */ + setActive("index.fusion.1", 1000000); + setActive("index.flush.2", 500000); + setActive("index.fusion.2", 1200000); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + auto flush2 = get_index_disk_dir("index.flush.2"); + auto fusion2 = get_index_disk_dir("index.fusion.2"); + { + /* + * When using the old index collection, disk space used by + * index.fusion.2 is considered transient. + */ + SCOPED_TRACE("index.fusion.1"); + assert_transient_size(1200000, fusion1); + } + { + SCOPED_TRACE("index.flush.2"); + assert_transient_size(0, flush2); + } + { + /* + * When using the new index collection, disk space used by + * index.fusion.1 and index.flush.2 is considered transient. + */ + SCOPED_TRACE("index.fusion.2"); + assert_transient_size(1500000, fusion2); + } + notActive("index.fusion.1"); + notActive("index.flush.2"); + { + /* + * old index collection removed. + */ + SCOPED_TRACE("index.fusion.2 after remove of index.fusion.1 and index.flush.1"); + assert_transient_size(0, fusion2); + } +} + +TEST_F(DiskIndexesTest, get_transient_size_during_ongoing_fusion) +{ + /* + * During ongoing fusion, we have one ISearchableIndexCollection instance: + * - old, containing index.fusion.1 and index.flush.2 + * + * Fusion output directory is index.fusion.2 + */ + setActive("index.fusion.1", 1000000); + setActive("index.flush.2", 500000); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + auto fusion2 = get_index_disk_dir("index.fusion.2"); + add_not_active(fusion2); // start tracking disk space for fusion output + { + /* + * Fusion not yet started. + */ + SCOPED_TRACE("dir missing"); + assert_transient_size(0, fusion1); + } + auto dir = base_dir + "/index.fusion.2"; + vespalib::mkdir(dir, true); + { + /* + * Fusion started, but no files written yet. + */ + SCOPED_TRACE("empty dir"); + assert_transient_size(0, fusion1); + } + constexpr uint32_t seek_pos = 999999; + { + std::string name = dir + "/foo"; + std::ofstream ostr(name, std::ios::binary); + ostr.seekp(seek_pos); + ostr.write(" ", 1); + ostr.flush(); + ostr.close(); + } + { + /* + * Fusion started, one file written. + */ + SCOPED_TRACE("single file"); + assert_transient_size((seek_pos + block_size) / block_size * block_size, fusion1); + } + EXPECT_TRUE(remove(fusion2)); // stop tracking disk space for fusion output + { + /* + * Fusion aborted. + */ + SCOPED_TRACE("removed"); + assert_transient_size(0, fusion1); + } +} + +} + +int +main(int argc, char* argv[]) +{ + vespalib::rmdir(base_dir, true); + ::testing::InitGoogleTest(&argc, argv); + auto result = RUN_ALL_TESTS(); + vespalib::rmdir(base_dir, true); + return result; +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt index 1987304dc7e..3995eb836fd 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt +++ b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt @@ -1,14 +1,14 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcorespi_index OBJECT SOURCES - activediskindexes.cpp diskindexcleaner.cpp + disk_indexes.cpp disk_index_stats.cpp eventlogger.cpp fusionrunner.cpp iindexmanager.cpp iindexcollection.cpp - index_disk_dir_active_state.cpp + index_disk_dir_state.cpp index_manager_explorer.cpp index_manager_stats.cpp indexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp deleted file mode 100644 index fb9585fb58e..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "activediskindexes.h" -#include "indexdisklayout.h" -#include "index_disk_dir.h" -#include "index_disk_dir_active_state.h" -#include <cassert> - -using vespalib::string; - -namespace searchcorespi::index { - -ActiveDiskIndexes::ActiveDiskIndexes() = default; -ActiveDiskIndexes::~ActiveDiskIndexes() = default; - -void ActiveDiskIndexes::setActive(const string &index) { - auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); - assert(index_disk_dir.valid()); - std::lock_guard lock(_lock); - auto insres = _active.insert(std::make_pair(index_disk_dir, IndexDiskDirActiveState())); - insres.first->second.activate(); -} - -void ActiveDiskIndexes::notActive(const string & index) { - auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); - assert(index_disk_dir.valid()); - std::lock_guard lock(_lock); - auto it = _active.find(index_disk_dir); - assert(it != _active.end()); - assert(it->second.is_active()); - it->second.deactivate(); - if (!it->second.is_active()) { - _active.erase(it); - } -} - -bool ActiveDiskIndexes::isActive(const string &index) const { - auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); - if (!index_disk_dir.valid()) { - return false; - } - std::lock_guard lock(_lock); - auto it = _active.find(index_disk_dir); - return (it != _active.end()) && it->second.is_active(); -} - -} diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h deleted file mode 100644 index 365025e7450..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <map> -#include <mutex> -#include <memory> - -namespace searchcorespi::index { - -class IndexDiskDir; -class IndexDiskDirActiveState; - -/** - * Class used to keep track of the set of active disk indexes in an index maintainer. - * The index directories are used as identifiers. - */ -class ActiveDiskIndexes { - std::map<IndexDiskDir, IndexDiskDirActiveState> _active; - mutable std::mutex _lock; - -public: - using SP = std::shared_ptr<ActiveDiskIndexes>; - ActiveDiskIndexes(); - ~ActiveDiskIndexes(); - ActiveDiskIndexes(const ActiveDiskIndexes &) = delete; - ActiveDiskIndexes & operator = (const ActiveDiskIndexes &) = delete; - void setActive(const vespalib::string & index); - void notActive(const vespalib::string & index); - bool isActive(const vespalib::string & index) const; -}; - -} diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp new file mode 100644 index 00000000000..28f6a886d06 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp @@ -0,0 +1,131 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "disk_indexes.h" +#include "indexdisklayout.h" +#include "index_disk_dir.h" +#include "index_disk_dir_state.h" +#include <vespa/searchlib/util/dirtraverse.h> +#include <cassert> +#include <vector> + +using vespalib::string; + +namespace searchcorespi::index { + +DiskIndexes::DiskIndexes() = default; +DiskIndexes::~DiskIndexes() = default; + +void +DiskIndexes::setActive(const string &index, uint64_t size_on_disk) +{ + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + assert(index_disk_dir.valid()); + std::lock_guard lock(_lock); + auto insres = _active.insert(std::make_pair(index_disk_dir, IndexDiskDirState())); + insres.first->second.activate(); + if (!insres.first->second.get_size_on_disk().has_value()) { + insres.first->second.set_size_on_disk(size_on_disk); + } +} + +void DiskIndexes::notActive(const string & index) { + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + assert(index_disk_dir.valid()); + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + assert(it != _active.end()); + assert(it->second.is_active()); + it->second.deactivate(); + if (!it->second.is_active()) { + _active.erase(it); + } +} + +bool DiskIndexes::isActive(const string &index) const { + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + if (!index_disk_dir.valid()) { + return false; + } + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + return (it != _active.end()) && it->second.is_active(); +} + + +void +DiskIndexes::add_not_active(IndexDiskDir index_disk_dir) +{ + std::lock_guard lock(_lock); + _active.insert(std::make_pair(index_disk_dir, IndexDiskDirState())); +} + +bool +DiskIndexes::remove(IndexDiskDir index_disk_dir) +{ + if (!index_disk_dir.valid()) { + return true; + } + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + if (it == _active.end()) { + return true; + } + if (it->second.is_active()) { + return false; + } + _active.erase(it); + return true; +} + +uint64_t +DiskIndexes::get_transient_size(IndexDiskLayout& layout, IndexDiskDir index_disk_dir) const +{ + /* + * Only report transient size related to a valid fusion index. This ensures + * that transient size is reported once per index collection. + */ + if (!index_disk_dir.valid() || !index_disk_dir.is_fusion_index()) { + return 0u; + } + uint64_t transient_size = 0u; + std::vector<IndexDiskDir> deferred; + { + std::lock_guard lock(_lock); + for (auto &entry : _active) { + if (entry.first < index_disk_dir) { + /* + * Indexes before current fusion index are on the way out and + * will be removed when all older index collections + * referencing them are destroyed. Disk space used by these + * indexes is considered transient. + */ + if (entry.second.get_size_on_disk().has_value()) { + transient_size += entry.second.get_size_on_disk().value(); + } + } + if (index_disk_dir < entry.first && entry.first.is_fusion_index()) { + /* + * Fusion indexes after current fusion index can be partially + * complete and might be removed if fusion is aborted. Disk + * space used by these indexes is consider transient. + */ + if (entry.second.get_size_on_disk().has_value()) { + transient_size += entry.second.get_size_on_disk().value(); + } else { + deferred.emplace_back(entry.first); + } + } + } + } + for (auto& entry : deferred) { + auto index_dir = layout.getFusionDir(entry.get_id()); + try { + search::DirectoryTraverse dirt(index_dir.c_str()); + transient_size += dirt.GetTreeSize(); + } catch (std::exception &) { + } + } + return transient_size; +} + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h new file mode 100644 index 00000000000..842c1814faf --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include <mutex> +#include <memory> + +namespace searchcorespi::index { + +class IndexDiskDir; +class IndexDiskDirState; +class IndexDiskLayout; + +/** + * Class used to keep track of the set of disk indexes in an index maintainer. + * The index directories are used as identifiers. + * + * DiskIndexCleaner will remove old disk indexes not marked active, + * i.e. old disk indexes used by old index collections are not removed. + * + * At start of fusion, an entry for fusion output index is added, to allow for + * tracking of transient disk use while fusion is ongoing. If fusion fails then + * the entry is removed, otherwise the entry is marked active as a side effect + * of setting up a new index collection. + */ +class DiskIndexes { + std::map<IndexDiskDir, IndexDiskDirState> _active; + mutable std::mutex _lock; + +public: + using SP = std::shared_ptr<DiskIndexes>; + DiskIndexes(); + ~DiskIndexes(); + DiskIndexes(const DiskIndexes &) = delete; + DiskIndexes & operator = (const DiskIndexes &) = delete; + void setActive(const vespalib::string & index, uint64_t size_on_disk); + void notActive(const vespalib::string & index); + bool isActive(const vespalib::string & index) const; + void add_not_active(IndexDiskDir index_disk_dir); + bool remove(IndexDiskDir index_disk_dir); + uint64_t get_transient_size(IndexDiskLayout& layout, IndexDiskDir index_disk_dir) const; +}; + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index cce880eed3f..3bed7ea8ea7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "diskindexcleaner.h" -#include "activediskindexes.h" +#include "disk_indexes.h" +#include "indexdisklayout.h" +#include "index_disk_dir.h" #include <vespa/fastos/file.h> #include <vespa/vespalib/io/fileutil.h> #include <sstream> @@ -80,13 +82,13 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) { } void removeOld(const string &base_dir, const vector<string> &indexes, - const ActiveDiskIndexes &active_indexes, bool remove) { + DiskIndexes &disk_indexes, bool remove) { uint32_t last_fusion_id = findLastFusionId(base_dir, indexes); for (size_t i = 0; i < indexes.size(); ++i) { const string index_dir = base_dir + "/" + indexes[i]; + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(indexes[i]); if (isOldIndex(indexes[i], last_fusion_id) && - !active_indexes.isActive(index_dir)) - { + disk_indexes.remove(index_disk_dir)) { if (remove) { removeDir(index_dir); } else { @@ -108,16 +110,16 @@ void removeInvalid(const string &base_dir, const vector<string> &indexes) { } // namespace void DiskIndexCleaner::clean(const string &base_dir, - const ActiveDiskIndexes &active_indexes) { + DiskIndexes &disk_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes, false); + removeOld(base_dir, indexes, disk_indexes, false); removeInvalid(base_dir, indexes); } void DiskIndexCleaner::removeOldIndexes( - const string &base_dir, const ActiveDiskIndexes &active_indexes) { + const string &base_dir, DiskIndexes &disk_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes, true); + removeOld(base_dir, indexes, disk_indexes, true); } } diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h index 798193ab00b..cbd3a5aa94f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h @@ -6,7 +6,7 @@ namespace searchcorespi { namespace index { -class ActiveDiskIndexes; +class DiskIndexes; /** * Utility class used to clean and remove index directories. @@ -16,9 +16,9 @@ struct DiskIndexCleaner { * Deletes all indexes with id lower than the most recent fusion id. */ static void clean(const vespalib::string &index_dir, - const ActiveDiskIndexes& active_indexes); + DiskIndexes& disk_indexes); static void removeOldIndexes(const vespalib::string &index_dir, - const ActiveDiskIndexes& active_indexes); + DiskIndexes& disk_indexes); }; } // namespace index diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h index 278fd73e555..335838ddf2e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h @@ -29,6 +29,8 @@ public: return (_id == rhs._id) && (_fusion == rhs._fusion); } bool valid() const noexcept { return _id != 0u; } + bool is_fusion_index() const noexcept { return _fusion; } + uint64_t get_id() const noexcept { return _id; } }; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_active_state.h b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_active_state.h deleted file mode 100644 index ac84de5adee..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_active_state.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstdint> - -namespace searchcorespi::index { - -/* - * Class describing active state for a disk index directory. - */ -class IndexDiskDirActiveState { - uint32_t _active_count; -public: - IndexDiskDirActiveState() - : _active_count(0) - { - } - - void activate() noexcept { ++_active_count; } - void deactivate() noexcept; - bool is_active() const noexcept { return _active_count != 0; } -}; - -} diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_active_state.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp index 603971c866e..ffe33d704c8 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_active_state.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "index_disk_dir_active_state.h" +#include "index_disk_dir_state.h" #include <cassert> namespace searchcorespi::index { void -IndexDiskDirActiveState::deactivate() noexcept +IndexDiskDirState::deactivate() noexcept { assert(_active_count > 0u); --_active_count; diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h new file mode 100644 index 00000000000..d8b790b3960 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <optional> + +namespace searchcorespi::index { + +/* + * Class describing state for a disk index directory. + */ +class IndexDiskDirState { + uint32_t _active_count; + std::optional<uint64_t> _size_on_disk; +public: + IndexDiskDirState() + : _active_count(0), + _size_on_disk() + { + } + + void activate() noexcept { ++_active_count; } + void deactivate() noexcept; + bool is_active() const noexcept { return _active_count != 0; } + const std::optional<uint64_t>& get_size_on_disk() const noexcept { return _size_on_disk; } + void set_size_on_disk(uint64_t size_on_disk) noexcept { _size_on_disk = size_on_disk; } +}; + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 6489cb156ce..afe5b573f21 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -8,6 +8,7 @@ #include "indexmaintainer.h" #include "indexreadutilities.h" #include "indexwriteutilities.h" +#include "index_disk_dir.h" #include <vespa/searchcorespi/flush/lambdaflushtask.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> @@ -88,13 +89,22 @@ class DiskIndexWithDestructorCallback : public IDiskIndex { private: std::shared_ptr<IDestructorCallback> _callback; IDiskIndex::SP _index; + IndexDiskDir _index_disk_dir; + IndexDiskLayout& _layout; + DiskIndexes& _disk_indexes; public: DiskIndexWithDestructorCallback(IDiskIndex::SP index, - std::shared_ptr<IDestructorCallback> callback) noexcept + std::shared_ptr<IDestructorCallback> callback, + IndexDiskLayout& layout, + DiskIndexes& disk_indexes) noexcept : _callback(std::move(callback)), - _index(std::move(index)) - { } + _index(std::move(index)), + _index_disk_dir(IndexDiskLayout::get_index_disk_dir(_index->getIndexDir())), + _layout(layout), + _disk_indexes(disk_indexes) + { + } ~DiskIndexWithDestructorCallback() override; const IDiskIndex &getWrapped() const { return *_index; } @@ -117,8 +127,7 @@ public: { return _index->createBlueprint(requestContext, fields, term); } - // TODO: Calculate the total disk size of current fusion indexes and set fusion_size_on_disk(). - search::SearchableStats getSearchableStats() const override { return _index->getSearchableStats(); } + search::SearchableStats getSearchableStats() const override; search::SerialNum getSerialNum() const override { return _index->getSerialNum(); } @@ -143,6 +152,16 @@ public: DiskIndexWithDestructorCallback::~DiskIndexWithDestructorCallback() = default; +search::SearchableStats +DiskIndexWithDestructorCallback::getSearchableStats() const +{ + auto stats = _index->getSearchableStats(); + uint64_t transient_size = _disk_indexes.get_transient_size(_layout, _index_disk_dir); + stats.fusion_size_on_disk(transient_size); + return stats; +} + + } // namespace IndexMaintainer::FusionArgs::FusionArgs() @@ -272,7 +291,7 @@ IndexMaintainer::updateActiveFusionPrunedSchema(const Schema &schema) void IndexMaintainer::deactivateDiskIndexes(vespalib::string indexDir) { - _active_indexes->notActive(indexDir); + _disk_indexes->notActive(indexDir); removeOldDiskIndexes(); } @@ -284,10 +303,13 @@ IndexMaintainer::loadDiskIndex(const string &indexDir) EventLogger::diskIndexLoadStart(indexDir); } vespalib::Timer timer; - _active_indexes->setActive(indexDir); + auto index = _operations.loadDiskIndex(indexDir); + auto stats = index->getSearchableStats(); + _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( - _operations.loadDiskIndex(indexDir), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); })); + std::move(index), + makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); } @@ -303,11 +325,14 @@ IndexMaintainer::reloadDiskIndex(const IDiskIndex &oldIndex) EventLogger::diskIndexLoadStart(indexDir); } vespalib::Timer timer; - _active_indexes->setActive(indexDir); const IDiskIndex &wrappedDiskIndex = (dynamic_cast<const DiskIndexWithDestructorCallback &>(oldIndex)).getWrapped(); + auto index = _operations.reloadDiskIndex(wrappedDiskIndex); + auto stats = index->getSearchableStats(); + _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( - _operations.reloadDiskIndex(wrappedDiskIndex), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); })); + std::move(index), + makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); } @@ -844,7 +869,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, IIndexMaintainerOperations &operations) : _base_dir(config.getBaseDir()), _warmupConfig(config.getWarmup()), - _active_indexes(std::make_shared<ActiveDiskIndexes>()), + _disk_indexes(std::make_shared<DiskIndexes>()), _layout(config.getBaseDir()), _schema(config.getSchema()), _activeFusionSchema(), @@ -877,7 +902,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, { // Called by document db init executor thread _changeGens.bumpPruneGen(); - DiskIndexCleaner::clean(_base_dir, *_active_indexes); + DiskIndexCleaner::clean(_base_dir, *_disk_indexes); FusionSpec spec = IndexReadUtilities::readFusionSpec(_base_dir); _next_id = 1 + (spec.flush_ids.empty() ? spec.last_fusion_id : spec.flush_ids.back()); _last_fusion_id = spec.last_fusion_id; @@ -1013,6 +1038,27 @@ IndexMaintainer::doFusion(SerialNum serialNum, std::shared_ptr<search::IFlushTok return getFusionDir(new_fusion_id); } +namespace { + +class RemoveFusionIndexGuard { + DiskIndexes* _disk_indexes; + IndexDiskDir _index_disk_dir; +public: + RemoveFusionIndexGuard(DiskIndexes& disk_indexes, IndexDiskDir index_disk_dir) + : _disk_indexes(&disk_indexes), + _index_disk_dir(index_disk_dir) + { + _disk_indexes->add_not_active(index_disk_dir); + } + ~RemoveFusionIndexGuard() { + if (_disk_indexes != nullptr) { + (void) _disk_indexes->remove(_index_disk_dir); + } + } + void reset() { _disk_indexes = nullptr; } +}; + +} uint32_t IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search::IFlushToken> flush_token) @@ -1034,6 +1080,8 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search if (FastOS_File::Stat(lastSerialFile.c_str(), &statInfo)) { serialNum = IndexReadUtilities::readSerialNum(lastFlushDir); } + IndexDiskDir fusion_index_disk_dir(fusion_spec.flush_ids.back(), true); + RemoveFusionIndexGuard remove_fusion_index_guard(*_disk_indexes, fusion_index_disk_dir); FusionRunner fusion_runner(_base_dir, args._schema, tuneFileAttributes, _ctx.getFileHeaderContext()); uint32_t new_fusion_id = fusion_runner.fuse(fusion_spec, serialNum, _operations, flush_token); bool ok = (new_fusion_id != 0); @@ -1066,6 +1114,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search } ChangeGens changeGens = getChangeGens(); IDiskIndex::SP new_index(loadDiskIndex(new_fusion_dir)); + remove_fusion_index_guard.reset(); // Post processing after fusion operation has completed and new disk // index has been opened. @@ -1101,7 +1150,7 @@ void IndexMaintainer::removeOldDiskIndexes() { LockGuard slock(_remove_lock); - DiskIndexCleaner::removeOldIndexes(_base_dir, *_active_indexes); + DiskIndexCleaner::removeOldIndexes(_base_dir, *_disk_indexes); } IndexMaintainer::FlushStats diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 8213c02b90c..fa7a9145a3c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -2,7 +2,7 @@ #pragma once #include "iindexmanager.h" -#include "activediskindexes.h" +#include "disk_indexes.h" #include "fusionspec.h" #include "idiskindex.h" #include "iindexmaintaineroperations.h" @@ -76,7 +76,7 @@ class IndexMaintainer : public IIndexManager, const vespalib::string _base_dir; const WarmupConfig _warmupConfig; - ActiveDiskIndexes::SP _active_indexes; + DiskIndexes::SP _disk_indexes; IndexDiskLayout _layout; Schema _schema; // Protected by SL + IUL Schema::SP _activeFusionSchema; // Protected by SL + IUL diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp index 058a21e15c7..cc2575f74d2 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -24,8 +24,7 @@ using search::SerialNum; using vespalib::IllegalStateException; using vespalib::FileHeader; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { namespace { @@ -42,22 +41,24 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum, IndexDiskLayout::getSerialNumFileName(dir); const vespalib::string tmpFileName = fileName + ".tmp"; - SerialNumFileHeaderContext snFileHeaderContext(fileHeaderContext, - serialNum); + SerialNumFileHeaderContext snFileHeaderContext(fileHeaderContext, serialNum); Fast_BufferedFile file; file.WriteOpen(tmpFileName.c_str()); FileHeader fileHeader; snFileHeaderContext.addTags(fileHeader, fileName); - fileHeader.putTag(FileHeader::Tag(IndexDiskLayout::SerialNumTag, - serialNum)); + fileHeader.putTag(FileHeader::Tag(IndexDiskLayout::SerialNumTag, serialNum)); bool ok = (fileHeader.writeFile(file) >= fileHeader.getSize()); - if (!file.Sync()) { + if ( ! ok) { + LOG(error, "Unable to write file header '%s'", tmpFileName.c_str()); + } + if ( ! file.Sync()) { + ok = false; + LOG(error, "Unable to fsync '%s'", tmpFileName.c_str()); + } + if ( ! file.Close()) { ok = false; - LOG(error, - "Unable to fsync '%s'", - tmpFileName.c_str()); + LOG(error, "Unable to close '%s'", tmpFileName.c_str()); } - file.Close(); vespalib::File::sync(dir); if (ok) { @@ -85,23 +86,20 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir, } FastOS_File file(tmpDest.c_str()); if (!file.OpenReadWrite()) { - LOG(error, - "Unable to open '%s' for fsync", - tmpDest.c_str()); + LOG(error, "Unable to open '%s' for fsync", tmpDest.c_str()); return false; } if (!file.Sync()) { - LOG(error, - "Unable to fsync '%s'", - tmpDest.c_str()); + LOG(error, "Unable to fsync '%s'", tmpDest.c_str()); + return false; + } + if (!file.Close()) { + LOG(error, "Unable to close '%s'", tmpDest.c_str()); return false; } - file.Close(); vespalib::File::sync(destDir); if (!file.Rename(dest.c_str())) { - LOG(error, - "Unable to rename file '%s' to '%s'", - tmpDest.c_str(), dest.c_str()); + LOG(error, "Unable to rename file '%s' to '%s'", tmpDest.c_str(), dest.c_str()); return false; } vespalib::File::sync(destDir); @@ -193,6 +191,4 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, vespalib::File::sync(indexDir); } -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h index 37193cce4d5..313ab3cc1c7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h @@ -8,8 +8,7 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/stllike/string.h> -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { /** * Utility class with functions to write aspects of an index to disk. @@ -40,7 +39,6 @@ struct IndexWriteUtilities search::SerialNum serialNum); }; -} // namespace index -} // namespace searchcorespi +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h index c95a42f601b..59fe73e0dec 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h @@ -72,7 +72,7 @@ struct IThreadingService virtual ISyncableThreadService &master() = 0; virtual IThreadService &index() = 0; virtual vespalib::ThreadExecutor &summary() = 0; - virtual vespalib::ThreadExecutor &shared() = 0; + virtual vespalib::Executor &shared() = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldInverter() = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldWriter() = 0; virtual vespalib::ISequencedTaskExecutor &attributeFieldWriter() = 0; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index c9c44af3aa3..de46dfa6441 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -104,6 +104,7 @@ vespa_define_module( src/tests/bitvector src/tests/btree src/tests/common/bitvector + src/tests/common/geogcd src/tests/common/location src/tests/common/location_iterator src/tests/common/matching_elements diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 2d7daf2300e..5d7e281df87 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1611,8 +1611,10 @@ "public void <init>(java.util.Collection)", "public void <init>(java.util.Map)", "public void <init>(java.util.Collection, java.util.Map)", - "public void <init>(java.util.Collection, java.util.Map, java.util.Map)", + "public void <init>(java.util.Collection, java.util.Map, com.yahoo.tensor.evaluation.TypeContext)", + "public java.util.Optional typeContext()", "public void <init>(java.util.Map, java.util.Map, java.util.Map)", + "public void <init>(java.util.Map, java.util.Map, java.util.Optional, java.util.Map)", "public void <init>(com.google.common.collect.ImmutableMap, java.util.Map, java.util.Map)", "public void addFunctionSerialization(java.lang.String, java.lang.String)", "public void addArgumentTypeSerialization(java.lang.String, java.lang.String, com.yahoo.tensor.TensorType)", diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 477282fdbf4..30af02ddb92 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -79,7 +79,8 @@ generate(uint64_t serialNum, size_t chunks, FastOS_FileInterface & idxFile, size fprintf(stdout, "Failed with lengthError %ld due to '%s'\n", lengthError, e.what()); } } - idxFile.Write2(os.data(), os.size()); + ssize_t written = idxFile.Write2(os.data(), os.size()); + assert(written == ssize_t(os.size())); return serialNum; } diff --git a/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp index 29c0a533fec..bdc841ab235 100644 --- a/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp +++ b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp @@ -110,7 +110,6 @@ Application::Main() std::cerr << e.getMessage() << std::endl; return EXIT_FAILURE; } - file.Close(); if (_quiet) { printQuiet(header); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java index 2bab10d1830..e26900ac77e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java @@ -16,9 +16,7 @@ public class ConstantNode extends ExpressionNode { public static final int classId = registerClass(0x4000 + 49, ConstantNode.class); private ResultNode value = null; - public ConstantNode() { - - } + public ConstantNode() {} public ConstantNode(ResultNode value) { this.value = value; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Arguments.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Arguments.java index fe22a5b1267..e770e6ac038 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Arguments.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Arguments.java @@ -15,7 +15,9 @@ import java.util.List; * @author bratseth */ public final class Arguments implements Serializable { + public static final Arguments EMPTY = new Arguments(); + private final ImmutableList<ExpressionNode> expressions; public Arguments() { @@ -47,9 +49,9 @@ public final class Arguments implements Serializable { /** Evaluate all arguments in this */ public Value[] evaluate(Context context) { - Value[] values=new Value[expressions.size()]; - for (int i=0; i<expressions.size(); i++) - values[i]=expressions.get(i).evaluate(context); + Value[] values = new Value[expressions.size()]; + for (int i = 0; i < expressions.size(); i++) + values[i] = expressions.get(i).evaluate(context); return values; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 93b8b8aca5e..8ac1829b16b 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -42,9 +42,11 @@ public final class ReferenceNode extends CompositeNode { return reference.name(); } + @Override public int hashCode() { return reference.hashCode(); } + /** Returns the arguments, never null */ public Arguments getArguments() { return reference.arguments(); } @@ -118,7 +120,7 @@ public final class ReferenceNode extends CompositeNode { throw new IllegalArgumentException(reference + " is invalid", e); } if (type == null) - throw new IllegalArgumentException("Unknown feature '" + toString() + "'"); + throw new IllegalArgumentException("Unknown feature '" + this + "'"); return type; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 535ad013caf..1f3203f2e35 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -4,13 +4,16 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; /** * Context needed to serialize an expression to a string. This has the lifetime of a single serialization @@ -22,6 +25,8 @@ public class SerializationContext extends FunctionReferenceContext { /** Serialized form of functions indexed by name */ private final Map<String, String> serializedFunctions; + private final Optional<TypeContext<Reference>> typeContext; + /** Create a context for a single serialization task */ public SerializationContext() { this(Collections.emptyList()); @@ -29,7 +34,7 @@ public class SerializationContext extends FunctionReferenceContext { /** Create a context for a single serialization task */ public SerializationContext(Collection<ExpressionFunction> functions) { - this(functions, Collections.emptyMap(), new LinkedHashMap<>()); + this(functions, Collections.emptyMap(), Optional.empty(), new LinkedHashMap<>()); } /** Create a context for a single serialization task */ @@ -39,7 +44,13 @@ public class SerializationContext extends FunctionReferenceContext { /** Create a context for a single serialization task */ public SerializationContext(Collection<ExpressionFunction> functions, Map<String, String> bindings) { - this(functions, bindings, new LinkedHashMap<>()); + this(functions, bindings, Optional.empty(), new LinkedHashMap<>()); + } + + /** Create a context for a single serialization task */ + public SerializationContext(Collection<ExpressionFunction> functions, Map<String, String> bindings, + TypeContext<Reference> typeContext) { + this(functions, bindings, Optional.of(typeContext), new LinkedHashMap<>()); } /** @@ -47,14 +58,19 @@ public class SerializationContext extends FunctionReferenceContext { * * @param functions the functions of this * @param bindings the arguments of this + * @param typeContext the type context of this: Serialization may depend on type resolution * @param serializedFunctions a cache of serializedFunctions - the ownership of this map * is <b>transferred</b> to this and will be modified in it */ - public SerializationContext(Collection<ExpressionFunction> functions, Map<String, String> bindings, + private SerializationContext(Collection<ExpressionFunction> functions, Map<String, String> bindings, + Optional<TypeContext<Reference>> typeContext, Map<String, String> serializedFunctions) { - this(toMap(functions), bindings, serializedFunctions); + this(toMap(functions), bindings, typeContext, serializedFunctions); } + /** Returns the type context of this, if it is able to resolve types. */ + public Optional<TypeContext<Reference>> typeContext() { return typeContext; } + private static Map<String, ExpressionFunction> toMap(Collection<ExpressionFunction> list) { Map<String,ExpressionFunction> mapBuilder = new HashMap<>(); for (ExpressionFunction function : list) @@ -70,9 +86,16 @@ public class SerializationContext extends FunctionReferenceContext { * @param serializedFunctions a cache of serializedFunctions - the ownership of this map * is <b>transferred</b> to this and will be modified in it */ + public SerializationContext(Map<String, ExpressionFunction> functions, Map<String, String> bindings, + Map<String, String> serializedFunctions) { + this(functions, bindings, Optional.empty(), serializedFunctions); + } + public SerializationContext(Map<String,ExpressionFunction> functions, Map<String, String> bindings, + Optional<TypeContext<Reference>> typeContext, Map<String, String> serializedFunctions) { super(functions, bindings); + this.typeContext = typeContext; this.serializedFunctions = serializedFunctions; } @@ -88,7 +111,7 @@ public class SerializationContext extends FunctionReferenceContext { serializedFunctions.put(name, expressionString); } - /** Adds the serialization of the an argument type to a function */ + /** Adds the serialization of the argument type to a function */ public void addArgumentTypeSerialization(String functionName, String argumentName, TensorType type) { serializedFunctions.put("rankingExpression(" + functionName + ")." + argumentName + ".type", type.toString()); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index d873963bb6e..52d54c9163e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -168,8 +168,8 @@ public class TensorFunctionNode extends CompositeNode { } @Override - public String toString(ToStringContext c) { - ToStringContext outermost = c; + public String toString(ToStringContext<Reference> c) { + ToStringContext<Reference> outermost = c; while (outermost.parent() != null) outermost = outermost.parent(); @@ -251,15 +251,17 @@ public class TensorFunctionNode extends CompositeNode { } @Override - public String toString(ToStringContext c) { - ToStringContext outermost = c; + public String toString(ToStringContext<Reference> c) { + ToStringContext<Reference> outermost = c; while (outermost.parent() != null) outermost = outermost.parent(); if (outermost instanceof ExpressionToStringContext) { ExpressionToStringContext context = (ExpressionToStringContext)outermost; return expression.toString(new StringBuilder(), - new ExpressionToStringContext(context.wrappedSerializationContext, c, context.path, context.parent), + new ExpressionToStringContext(context.wrappedSerializationContext, c, + context.path, + context.parent), context.path, context.parent) .toString(); @@ -281,9 +283,9 @@ public class TensorFunctionNode extends CompositeNode { * to add more context, we need to keep track of both these contexts here separately and map between * contexts as seen in the toString methods of the function classes above. */ - private static class ExpressionToStringContext extends SerializationContext implements ToStringContext { + private static class ExpressionToStringContext extends SerializationContext implements ToStringContext<Reference> { - private final ToStringContext wrappedToStringContext; + private final ToStringContext<Reference> wrappedToStringContext; private final SerializationContext wrappedSerializationContext; private final Deque<String> path; private final CompositeNode parent; @@ -297,7 +299,7 @@ public class TensorFunctionNode extends CompositeNode { } ExpressionToStringContext(SerializationContext wrappedSerializationContext, - ToStringContext wrappedToStringContext, + ToStringContext<Reference> wrappedToStringContext, Deque<String> path, CompositeNode parent) { this.wrappedSerializationContext = wrappedSerializationContext; @@ -328,6 +330,12 @@ public class TensorFunctionNode extends CompositeNode { /** Returns a function or null if it isn't defined in this context */ public ExpressionFunction getFunction(String name) { return wrappedSerializationContext.getFunction(name); } + /** Returns the type context of this, or empty if none. */ + @Override + public Optional<TypeContext<Reference>> typeContext() { + return wrappedSerializationContext.typeContext(); + } + /** @deprecated Use {@link #getFunctions()} instead */ @SuppressWarnings("removal") @Deprecated(forRemoval = true, since = "7") @@ -335,9 +343,10 @@ public class TensorFunctionNode extends CompositeNode { return ImmutableMap.copyOf(wrappedSerializationContext.getFunctions()); } - @Override protected Map<String, ExpressionFunction> getFunctions() { return wrappedSerializationContext.getFunctions(); } + @Override + protected Map<String, ExpressionFunction> getFunctions() { return wrappedSerializationContext.getFunctions(); } - public ToStringContext parent() { return wrappedToStringContext; } + public ToStringContext<Reference> parent() { return wrappedToStringContext; } /** Returns the resolution of an identifier, or null if it isn't defined in this context */ @Override @@ -361,6 +370,7 @@ public class TensorFunctionNode extends CompositeNode { SerializationContext serializationContext = new SerializationContext(getFunctions(), null, serializedFunctions()); return new ExpressionToStringContext(serializationContext, null, path, parent); } + } /** Turns an EvaluationContext into a Context */ diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 0d46ab4ddb6..865820320d8 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -1032,11 +1032,13 @@ Slice.DimensionValue dimensionValue(Optional dimensionName) : { value = expression() { - if (value instanceof ReferenceNode && ((ReferenceNode)value).reference().isIdentifier()) - return new Slice.DimensionValue(dimensionName, ((ReferenceNode)value).reference().name()); - else - return new Slice.DimensionValue(dimensionName, TensorFunctionNode.wrapScalar(value)); -} + if (value instanceof ReferenceNode && ((ReferenceNode)value).reference().isIdentifier()) // A label + return new Slice.DimensionValue(dimensionName, ((ReferenceNode)value).reference().name()); + else if (value instanceof ConstantNode && ((ConstantNode)value).getValue() instanceof StringValue) // A quoted label + return new Slice.DimensionValue(dimensionName, ((StringValue)((ConstantNode)value).getValue()).asString()); + else + return new Slice.DimensionValue(dimensionName, TensorFunctionNode.wrapScalar(value)); + } } String label() : diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index e0071e28428..291114ffa90 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -1605,7 +1605,7 @@ AttributeGuard createInt8Attribute() { } AttributeGuard createBoolAttribute() { - SingleBoolAttribute *selectAttr1(new SingleBoolAttribute("selectAttr1", search::GrowStrategy())); + SingleBoolAttribute *selectAttr1(new SingleBoolAttribute("selectAttr1", search::GrowStrategy(), false)); DocId docId(0); selectAttr1->addDoc(docId); selectAttr1->setBit(docId, true); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index bd84a6ca419..73dd7d2f776 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -5,16 +5,13 @@ #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/attrvector.h> #include <vespa/searchlib/attribute/multienumattribute.hpp> -#include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multistringattribute.h> #include <vespa/searchlib/attribute/multivalueattribute.hpp> #include <vespa/searchlib/attribute/predicate_attribute.h> @@ -25,6 +22,10 @@ #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> +#include <vespa/vespalib/util/round_up_to_page_size.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/fastos/file.h> #include <cmath> #include <iostream> @@ -41,6 +42,8 @@ using search::attribute::IAttributeVector; using vespalib::stringref; using vespalib::string; +namespace search { + namespace { string empty; @@ -157,8 +160,6 @@ bool contains_value(const Container& c, size_t elems, const V& value) { } -namespace search { - using attribute::CollectionType; using attribute::Config; @@ -277,6 +278,10 @@ private: void testPendingCompaction(); void testConditionalCommit(); + static int64_t stat_size(const vespalib::string& swapfile); + int test_paged_attribute(const vespalib::string& name, const vespalib::string& swapfile, const search::attribute::Config& cfg); + void test_paged_attributes(); + public: AttributeTest(); int Main() override; @@ -2268,6 +2273,98 @@ AttributeTest::testConditionalCommit() { EXPECT_EQUAL(0u, iv.getChangeVectorMemoryUsage().usedBytes()); } +int64_t +AttributeTest::stat_size(const vespalib::string& swapfile) +{ + auto stat = vespalib::stat(swapfile); + ASSERT_TRUE(stat); + return stat->_size; +} + +int +AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib::string& swapfile, const search::attribute::Config& cfg) +{ + int result = 1; + size_t rounded_size = vespalib::round_up_to_page_size(1); + size_t lid_mapping_size = 1200; + size_t sv_maxlid = 1200; + if (rounded_size == 64_Ki) { + lid_mapping_size = 17000; + sv_maxlid = 1500; + } + if (cfg.basicType() == search::attribute::BasicType::Type::BOOL) { + lid_mapping_size = rounded_size * 8 + 100; + } + LOG(info, "test_paged_attribute '%s'", name.c_str()); + auto av = createAttribute(name, cfg); + auto v = std::dynamic_pointer_cast<IntegerAttribute>(av); + ASSERT_TRUE(v || (!cfg.collectionType().isMultiValue() && !cfg.fastSearch())); + auto size1 = stat_size(swapfile); + // Grow mapping from lid to value or multivalue index + addClearedDocs(av, lid_mapping_size); + auto size2 = stat_size(swapfile); + auto size3 = size2; + EXPECT_LESS(size1, size2); + if (cfg.collectionType().isMultiValue()) { + // Grow multi value mapping + for (uint32_t lid = 1; lid < 100; ++lid) { + av->clearDoc(lid); + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_TRUE(v->append(lid, 0, 1)); + } + av->commit(); + } + size3 = stat_size(swapfile); + EXPECT_LESS(size2, size3); + result += 2; + } + if (cfg.fastSearch()) { + // Grow enum store + uint32_t maxlid = cfg.collectionType().isMultiValue() ? 100 : sv_maxlid; + for (uint32_t lid = 1; lid < maxlid; ++lid) { + av->clearDoc(lid); + if (cfg.collectionType().isMultiValue()) { + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_TRUE(v->append(lid, lid * 100 + i, 1)); + } + } else { + EXPECT_TRUE(v->update(lid, lid * 100)); + } + av->commit(); + } + auto size4 = stat_size(swapfile); + EXPECT_LESS(size3, size4); + result += 4; + } + return result; +} + +void +AttributeTest::test_paged_attributes() +{ + vespalib::string basedir("mmap-file-allocator-factory-dir"); + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); + search::attribute::Config cfg1(BasicType::INT32, CollectionType::SINGLE); + cfg1.setPaged(true); + EXPECT_EQUAL(1, test_paged_attribute("std-int-sv-paged", basedir + "/0.std-int-sv-paged/swapfile", cfg1)); + search::attribute::Config cfg2(BasicType::INT32, CollectionType::ARRAY); + cfg2.setPaged(true); + EXPECT_EQUAL(3, test_paged_attribute("std-int-mv-paged", basedir + "/1.std-int-mv-paged/swapfile", cfg2)); + search::attribute::Config cfg3(BasicType::INT32, CollectionType::SINGLE); + cfg3.setPaged(true); + cfg3.setFastSearch(true); + EXPECT_EQUAL(5, test_paged_attribute("fs-int-sv-paged", basedir + "/2.fs-int-sv-paged/swapfile", cfg3)); + search::attribute::Config cfg4(BasicType::INT32, CollectionType::ARRAY); + cfg4.setPaged(true); + cfg4.setFastSearch(true); + EXPECT_EQUAL(7, test_paged_attribute("fs-int-mv-paged", basedir + "/3.fs-int-mv-paged/swapfile", cfg4)); + search::attribute::Config cfg5(BasicType::BOOL, CollectionType::SINGLE); + cfg5.setPaged(true); + EXPECT_EQUAL(1, test_paged_attribute("std-bool-sv-paged", basedir + "/4.std-bool-sv-paged/swapfile", cfg5)); + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); + vespalib::rmdir(basedir, true); +} + void testNamePrefix() { Config cfg(BasicType::INT32, CollectionType::SINGLE); AttributeVector::SP vFlat = createAttribute("sfsint32_pc", cfg); @@ -2350,6 +2447,7 @@ int AttributeTest::Main() TEST_DO(testConditionalCommit()); TEST_DO(testNamePrefix()); test_multi_value_mapping_has_free_lists_enabled(); + TEST_DO(test_paged_attributes()); deleteDataDirs(); TEST_DONE(); diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 5346cc7f764..99fdd9f4b0a 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> @@ -11,6 +12,8 @@ using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; using vespalib::datastore::EntryRefFilter; using RefT = vespalib::datastore::EntryRefT<22>; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; namespace vespalib::datastore { @@ -374,6 +377,13 @@ TEST(EnumStoreTest, address_space_usage_is_reported) EXPECT_EQ(AddressSpace(3, 3, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); } +TEST(EnumStoreTest, provided_memory_allocator_is_used) +{ + AllocStats stats; + NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats)); + EXPECT_EQ(AllocStats(1, 0), stats); +} + class BatchUpdaterTest : public ::testing::Test { public: NumericEnumStore store; diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index bddaa4f4e31..29af989d484 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/size_literals.h> @@ -16,6 +17,8 @@ LOG_SETUP("multivaluemapping_test"); using vespalib::datastore::ArrayStoreConfig; using vespalib::datastore::CompactionSpec; using vespalib::datastore::CompactionStrategy; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; template <typename EntryT> void @@ -69,6 +72,7 @@ class MappingTestBase : public ::testing::Test { protected: using MvMapping = search::attribute::MultiValueMapping<EntryT>; using AttributeType = MyAttribute<MvMapping>; + AllocStats _stats; std::unique_ptr<MvMapping> _mvMapping; std::unique_ptr<AttributeType> _attr; uint32_t _maxSmallArraySize; @@ -78,7 +82,8 @@ protected: public: using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; MappingTestBase() - : _mvMapping(), + : _stats(), + _mvMapping(), _attr(), _maxSmallArraySize() { @@ -87,7 +92,7 @@ public: ArrayStoreConfig config(maxSmallArraySize, ArrayStoreConfig::AllocSpec(0, RefType::offsetSize(), 8_Ki, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); - _mvMapping = std::make_unique<MvMapping>(config); + _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } @@ -95,7 +100,7 @@ public: ArrayStoreConfig config(maxSmallArraySize, ArrayStoreConfig::AllocSpec(minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); - _mvMapping = std::make_unique<MvMapping>(config); + _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } @@ -129,6 +134,7 @@ public: _mvMapping->clearDocs(lidLow, lidLimit, [this](uint32_t docId) { _attr->clearDoc(docId); }); } size_t getTotalValueCnt() const { return _mvMapping->getTotalValueCnt(); } + const AllocStats &get_stats() const noexcept { return _stats; } uint32_t countBuffers() { using RefVector = typename MvMapping::RefCopyVector; @@ -326,6 +332,12 @@ TEST_F(IntMappingTest, test_that_free_lists_can_be_disabled) EXPECT_FALSE(_mvMapping->has_free_lists_enabled()); } +TEST_F(IntMappingTest, provided_memory_allocator_is_used) +{ + setup(3, 64, 512, 129, true); + EXPECT_EQ(AllocStats(5, 0), get_stats()); +} + TEST_F(CompactionIntMappingTest, test_that_compaction_works) { setup(3, 64, 512, 129); diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index de54386e4af..29f7b5c0276 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -1830,7 +1830,7 @@ private: public: BoolAttributeFixture(const SimpleResult& true_docs, uint32_t num_docs) - : _attr("bool_attr", search::GrowStrategy()) + : _attr("bool_attr", search::GrowStrategy(), false) { _attr.addDocs(num_docs); for (uint32_t i = 0; i < true_docs.getHitCount(); ++i) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 8a6f1e08fa6..c77dfb2c2d2 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -681,7 +681,6 @@ Fixture::get_file_header() vespalib::string file_name = attr_name + ".dat"; EXPECT_TRUE(file.OpenReadOnly(file_name.c_str())); (void) header.readFile(file); - file.Close(); return header; } diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index faeebf9984a..a9850756ac6 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -8,11 +8,16 @@ #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/rand48.h> #include <algorithm> using namespace search; +using vespalib::alloc::Alloc; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; + namespace { std::string @@ -653,5 +658,26 @@ TEST("requireThatGrowWorks") g.trimHoldLists(2); } +TEST("require that growable bit vectors keeps memory allocator") +{ + AllocStats stats; + auto memory_allocator = std::make_unique<MemoryAllocatorObserver>(stats); + Alloc init_alloc = Alloc::alloc_with_allocator(memory_allocator.get()); + vespalib::GenerationHolder g; + GrowableBitVector v(200, 200, g, &init_alloc); + EXPECT_EQUAL(AllocStats(1, 0), stats); + v.resize(1); + EXPECT_EQUAL(AllocStats(2, 1), stats); + v.reserve(2000); + EXPECT_EQUAL(AllocStats(3, 1), stats); + v.extend(4000); + EXPECT_EQUAL(AllocStats(4, 1), stats); + v.shrink(200); + EXPECT_EQUAL(AllocStats(4, 1), stats); + v.resize(1); + EXPECT_EQUAL(AllocStats(5, 2), stats); + g.transferHoldLists(1); + g.trimHoldLists(2); +} TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/common/geogcd/CMakeLists.txt b/searchlib/src/tests/common/geogcd/CMakeLists.txt new file mode 100644 index 00000000000..3721b609668 --- /dev/null +++ b/searchlib/src/tests/common/geogcd/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_geo_gcd_test_app TEST + SOURCES + geo_gcd_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_geo_gcd_test_app COMMAND searchlib_geo_gcd_test_app) diff --git a/searchlib/src/tests/common/geogcd/geo_gcd_test.cpp b/searchlib/src/tests/common/geogcd/geo_gcd_test.cpp new file mode 100644 index 00000000000..dd218c4ff04 --- /dev/null +++ b/searchlib/src/tests/common/geogcd/geo_gcd_test.cpp @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <stdio.h> +#include <vespa/searchlib/common/geo_gcd.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace search::common; + +struct Point { + const char *name; + double lat; + double lng; +}; + +constexpr size_t NUM = 9; + +const Point airports[NUM] = { + { "SFO", 37.61, -122.38}, + { "LHR", 51.47, -0.46}, + { "OSL", 60.20, 11.08}, + { "GIG", -22.8, -43.25}, + { "HKG", 22.31, 113.91}, + { "TRD", 63.45, 10.92}, + { "SYD", -33.95, 151.17}, + { "LAX", 33.94, -118.41}, + { "JFK", 40.64, -73.78}, +}; + +const double exact_distances[NUM][NUM] = { + { 0, 5367, 5196, 6604, 6927, 5012, 7417, 337, 2586 }, + { 5367, 0, 750, 5734, 5994, 928, 10573, 5456, 3451 }, + { 5196, 750, 0, 6479, 5319, 226, 9888, 5345, 3687 }, + { 6604, 5734, 6479, 0, 10989, 6623, 8414, 6294, 4786 }, + { 6927, 5994, 5319, 10989, 0, 5240, 4581, 7260, 8072 }, + { 5012, 928, 226, 6623, 5240, 0, 9782, 5171, 3611 }, + { 7417, 10573, 9888, 8414, 4581, 9782, 0, 7488, 9950 }, + { 337, 5456, 5345, 6294, 7260, 5171, 7488, 0, 2475 }, + { 2586, 3451, 3687, 4786, 8072, 3611, 9950, 2475, 0 } +}; + +TEST(GeoGcdTest, computed_distances_seem_legit) { + for (size_t i = 0; i < NUM; ++i) { + const Point &from = airports[i]; + printf("\n"); + GeoGcd geo_from{from.lat, from.lng}; + for (size_t j = 0; j < NUM; ++j) { + const Point &to = airports[j]; + double km = geo_from.km_great_circle_distance(to.lat, to.lng); + double miles = km / 1.609344; + EXPECT_GE(miles, 0); + if (from.name == to.name) { + EXPECT_EQ(miles, 0.0); + } else { + double exact = exact_distances[i][j]; + printf("Distance from %s to %s (in miles): %.1f [more exact would be %.1f]\n", from.name, to.name, miles, exact); + EXPECT_LT(miles*0.99, exact); + EXPECT_GT(miles*1.01, exact); + } + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index c5ea12a7568..570f1a6ea03 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -1,13 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("bitvector_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/index/field_length_info.h> #include <vespa/searchlib/diskindex/bitvectordictionary.h> #include <vespa/searchlib/diskindex/fieldwriter.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/log/log.h> +LOG_SETUP("bitvector_test"); + using namespace search::index; using search::index::schema::DataType; diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index 6c78d185cfb..35b42223cfc 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -19,6 +19,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/time.h> #include <openssl/evp.h> +#include <vespa/fastos/file.h> #include <vespa/fastos/app.h> #include <vespa/log/log.h> LOG_SETUP("fieldwriter_test"); @@ -330,7 +331,6 @@ FileChecksum::FileChecksum(const vespalib::string &file_name) EVP_DigestUpdate(md_ctx.get(), buf.get(), thistime); remainder -= thistime; } - f.Close(); EVP_DigestFinal_ex(md_ctx.get(), &_digest[0], &_digest_len); assert(_digest_len > 0u && _digest_len <= EVP_MAX_MD_SIZE); } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 72867edf474..37fac07a8d4 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -63,12 +63,14 @@ class FusionTest : public ::testing::Test { protected: Schema _schema; + bool _force_small_merge_chunk; const Schema & getSchema() const { return _schema; } void requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap, bool force_short_merge_chunk); void make_simple_index(const vespalib::string &dump_dir, const IFieldLengthInspector &field_length_inspector); bool try_merge_simple_indexes(const vespalib::string &dump_dir, const std::vector<vespalib::string> &sources, std::shared_ptr<IFlushToken> flush_token); void merge_simple_indexes(const vespalib::string &dump_dir, const std::vector<vespalib::string> &sources); + void reconstruct_interleaved_features(); public: FusionTest(); }; @@ -494,6 +496,7 @@ FusionTest::try_merge_simple_indexes(const vespalib::string &dump_dir, const std SelectorArray selector(20, 0); Fusion fusion(_schema, dump_dir, sources, selector, tuneFileIndexing, fileHeaderContext); + fusion.set_force_small_merge_chunk(_force_small_merge_chunk); return fusion.merge(executor, flush_token); } @@ -505,7 +508,8 @@ FusionTest::merge_simple_indexes(const vespalib::string &dump_dir, const std::ve FusionTest::FusionTest() : ::testing::Test(), - _schema(make_schema(false)) + _schema(make_schema(false)), + _force_small_merge_chunk(false) { } @@ -557,7 +561,8 @@ TEST_F(FusionTest, require_that_average_field_length_is_preserved) clean_field_length_testdirs(); } -TEST_F(FusionTest, require_that_interleaved_features_can_be_reconstructed) +void +FusionTest::reconstruct_interleaved_features() { clean_field_length_testdirs(); make_simple_index("fldump2", MockFieldLengthInspector()); @@ -573,6 +578,17 @@ TEST_F(FusionTest, require_that_interleaved_features_can_be_reconstructed) clean_field_length_testdirs(); } +TEST_F(FusionTest, require_that_interleaved_features_can_be_reconstructed) +{ + reconstruct_interleaved_features(); +} + +TEST_F(FusionTest, require_that_interleaved_features_can_be_reconstructed_with_small_merge_chunk) +{ + _force_small_merge_chunk = true; + reconstruct_interleaved_features(); +} + namespace { void clean_stopped_fusion_testdirs() diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 3b9f36d9f1f..b295291d7c4 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -1,23 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/docstore/filechunk.h> #include <vespa/searchlib/docstore/writeablefilechunk.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <iomanip> #include <iostream> #include <vespa/log/log.h> -#include <vespa/vespalib/util/compressionconfig.h> LOG_SETUP("file_chunk_test"); using namespace search; using common::FileHeaderContext; +using vespalib::CpuUsage; using vespalib::ThreadStackExecutor; struct MyFileHeaderContext : public FileHeaderContext { @@ -136,12 +138,12 @@ struct WriteFixture : public FixtureBase { dir.cleanup(dirCleanup); } void flush() { - chunk.flush(true, serialNum); + chunk.flush(true, serialNum, CpuUsage::Category::WRITE); chunk.flushPendingChunks(serialNum); } WriteFixture &append(uint32_t lid) { vespalib::string data = getData(lid); - chunk.append(nextSerialNum(), lid, data.c_str(), data.size()); + chunk.append(nextSerialNum(), lid, data.c_str(), data.size(), CpuUsage::Category::WRITE); return *this; } void updateLidMap(uint32_t docIdLimit) { diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 1ba069818ba..7ebc3759813 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/features/attributefeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> +#include <vespa/searchlib/features/great_circle_distance_feature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> @@ -93,6 +94,7 @@ Test::Main() TEST_DO(testAttributeMatch()); TEST_FLUSH(); TEST_DO(testCloseness()); TEST_FLUSH(); TEST_DO(testMatchCount()); TEST_FLUSH(); + TEST_DO(testGreatCircleDistance()); TEST_FLUSH(); TEST_DO(testDistance()); TEST_FLUSH(); TEST_DO(testDistanceToPath()); TEST_FLUSH(); TEST_DO(testDotProduct()); TEST_FLUSH(); @@ -819,6 +821,67 @@ Test::assertFreshness(feature_t expFreshness, const vespalib::string & attr, uin ASSERT_TRUE(ft.execute(RankResult().addScore(feature, expFreshness).setEpsilon(EPS))); } +namespace { + +struct AirPort { + const char *tla; + double lat; + double lng; +}; + +std::pair<int32_t, int32_t> toXY(const AirPort &p) { + return std::make_pair((int)(p.lng * 1.0e6), + (int)(p.lat * 1.0e6)); +} + +GeoLocation toGL(const AirPort &p) { + int32_t x = (int)(p.lng * 1.0e6); + int32_t y = (int)(p.lat * 1.0e6); + GeoLocation::Point gp{x, y}; + return GeoLocation{gp}; +} + +} + +void +Test::testGreatCircleDistance() +{ + { // Test blueprint. + GreatCircleDistanceBlueprint pt; + EXPECT_TRUE(assertCreateInstance(pt, "great_circle_distance")); + StringList params, in, out; + FT_SETUP_FAIL(pt, params); + FtIndexEnvironment idx_env; + idx_env + .getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos_zcurve"); + FT_SETUP_OK(pt, idx_env, params.add("pos"), in, + out.add("km").add("latitude").add("longitude")); + FT_DUMP_EMPTY(_factory, "great_circle_distance"); + } + { // Test executor. + FtFeatureTest ft(_factory, "great_circle_distance(pos)"); + const AirPort SFO = { "SFO", 37.618806, -122.375416 }; + const AirPort TRD = { "TRD", 63.457556, 10.924250 }; + std::vector<std::pair<int32_t,int32_t>> pos = { toXY(SFO), toXY(TRD) }; + setupForDistanceTest(ft, "pos_zcurve", pos, true); + const AirPort LHR = { "LHR", 51.477500, -0.461388 }; + const AirPort JFK = { "JFK", 40.639928, -73.778692 }; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", toGL(LHR)}); + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", toGL(JFK)}); + ASSERT_TRUE(ft.setup()); + double exp = 1494; // according to gcmap.com + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(10.0). + addScore("great_circle_distance(pos)", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(10.0). + addScore("great_circle_distance(pos).km", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). + addScore("great_circle_distance(pos).latitude", TRD.lat))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). + addScore("great_circle_distance(pos).longitude", TRD.lng))); + } +} + void Test::testDistance() { @@ -830,7 +893,7 @@ Test::testDistance() StringList params, in, out; FT_SETUP_FAIL(pt, params); FT_SETUP_OK(pt, params.add("pos"), in, - out.add("out").add("index").add("latitude").add("longitude")); + out.add("out").add("index").add("latitude").add("longitude").add("km")); FT_DUMP_EMPTY(_factory, "distance"); } @@ -963,6 +1026,8 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-4). addScore("distance(pos)", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-4). + addScore("distance(pos).km", exp * 0.00011119508023))); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-30). addScore("distance(pos).index", hit_index))); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 58e6b4953cc..ad21d7d7ccc 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -19,6 +19,7 @@ public: void testAttributeMatch(); void testCloseness(); void testMatchCount(); + void testGreatCircleDistance(); void testDistance(); void testDistanceToPath(); void testDotProduct(); diff --git a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp index b65f7d08e58..65e067ae7b1 100644 --- a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp +++ b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp @@ -128,7 +128,11 @@ TEST("require that expression with only number inputs produce number output (com } TEST("require that expression with object input produces object output (interpreted)") { - TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::object(ValueType::double_type()))); + TEST_DO(verify_output_type({{"b", "tensor(x{})"}}, "a*b", FeatureType::object(ValueType::from_spec("tensor(x{})")))); +} + +TEST("require that scalar expressions are auto-unboxed (interpreted)") { + TEST_DO(verify_output_type({{"b", "tensor(x{})"}}, "reduce(a*b,sum)", FeatureType::number())); } TEST("require that ranking expression can resolve to concrete complex type") { @@ -138,7 +142,7 @@ TEST("require that ranking expression can resolve to concrete complex type") { TEST("require that ranking expression can be external") { TEST_DO(verify_output_type({}, "a*b", FeatureType::number(), "my_expr")); - TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::object(ValueType::double_type()), "my_expr")); + TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::number(), "my_expr")); TEST_DO(verify_output_type({{"a", "tensor(x{},y{})"}, {"b", "tensor(y{},z{})"}}, "a*b", FeatureType::object(ValueType::from_spec("tensor(x{},y{},z{})")), "my_expr")); } diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp index 46aaf7369e3..3639da05b9e 100644 --- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp +++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp @@ -79,7 +79,8 @@ struct Fixture { explicit Fixture() { factory.addPrototype(std::make_shared<ValueBlueprint>()); factory.addPrototype(std::make_shared<UnboxBlueprint>()); - factory.addPrototype(std::make_shared<ProxyBlueprint>("box", Blueprint::AcceptInput::NUMBER, true)); + factory.addPrototype(std::make_shared<ProxyBlueprint>("do_box", Blueprint::AcceptInput::NUMBER, true)); + factory.addPrototype(std::make_shared<ProxyBlueprint>("do_unbox", Blueprint::AcceptInput::OBJECT, false)); factory.addPrototype(std::make_shared<ProxyBlueprint>("maybe_box", Blueprint::AcceptInput::ANY, true)); factory.addPrototype(std::make_shared<ProxyBlueprint>("maybe_unbox", Blueprint::AcceptInput::ANY, false)); } @@ -106,26 +107,33 @@ struct Fixture { }; TEST_F("require that values can be boxed and unboxed", Fixture()) { - EXPECT_EQUAL(3.0, f1.eval("box(value(3))")); - EXPECT_EQUAL(0.0, f1.eval("box(value(3)).was_object")); - EXPECT_EQUAL(3.0, f1.eval("unbox(box(value(3)))")); - EXPECT_EQUAL(1.0, f1.eval("maybe_unbox(box(value(3))).was_object")); - EXPECT_EQUAL(3.0, f1.eval("box(unbox(box(value(3))))")); - EXPECT_EQUAL(0.0, f1.eval("box(unbox(box(value(3)))).was_object")); + EXPECT_EQUAL(3.0, f1.eval("do_box(value(3))")); + EXPECT_EQUAL(0.0, f1.eval("do_box(value(3)).was_object")); + EXPECT_EQUAL(3.0, f1.eval("do_unbox(do_box(value(3)))")); + EXPECT_EQUAL(1.0, f1.eval("maybe_unbox(do_box(value(3))).was_object")); + EXPECT_EQUAL(3.0, f1.eval("do_box(do_unbox(do_box(value(3))))")); + EXPECT_EQUAL(0.0, f1.eval("do_box(do_unbox(do_box(value(3)))).was_object")); } TEST_F("require that output features may be either objects or numbers", Fixture()) { EXPECT_TRUE(f1.verify("value(3)")); - EXPECT_TRUE(f1.verify("box(value(3))")); + EXPECT_TRUE(f1.verify("do_box(value(3))")); } TEST_F("require that feature input/output types must be compatible", Fixture()) { - EXPECT_TRUE(!f1.verify("unbox(value(3))")); + EXPECT_TRUE(!f1.verify("do_unbox(value(3))")); EXPECT_TRUE(f1.verify("maybe_unbox(value(3))")); - EXPECT_TRUE(f1.verify("unbox(box(value(3)))")); - EXPECT_TRUE(!f1.verify("unbox(box(box(value(3))))")); - EXPECT_TRUE(f1.verify("unbox(maybe_box(box(value(3))))")); - EXPECT_TRUE(f1.verify("unbox(box(unbox(box(value(3)))))")); + EXPECT_TRUE(f1.verify("do_unbox(do_box(value(3)))")); + EXPECT_TRUE(!f1.verify("do_unbox(do_box(do_box(value(3))))")); + EXPECT_TRUE(f1.verify("do_unbox(maybe_box(do_box(value(3))))")); + EXPECT_TRUE(f1.verify("do_unbox(do_box(do_unbox(do_box(value(3)))))")); +} + +TEST_F("require that 'unbox' feature works for both numbers and objects", Fixture()) { + EXPECT_EQUAL(3.0, f1.eval("unbox(value(3))")); + EXPECT_EQUAL(3.0, f1.eval("unbox(do_box(value(3)))")); + EXPECT_EQUAL(0.0, f1.eval("maybe_unbox(unbox(do_box(value(3)))).was_object")); + EXPECT_EQUAL(0.0, f1.eval("maybe_unbox(unbox(value(3))).was_object")); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp index 11f4a01888b..40e1cee3629 100644 --- a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp +++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp @@ -339,7 +339,7 @@ TEST_F("require that interpreted ranking expressions are always lazy", Fixture() f1.add_expr("rank", "if(docid<10,box(track(ivalue(1))),track(ivalue(2)))"); f1.compile(); EXPECT_EQUAL(7u, f1.program.num_executors()); - EXPECT_EQUAL(8u, count_features(f1.program)); + EXPECT_EQUAL(7u, count_features(f1.program)); EXPECT_EQUAL(0u, count_const_features(f1.program)); EXPECT_EQUAL(f1.track_cnt, 0u); EXPECT_EQUAL(f1.get(expr_feature("rank"), 5), 1.0); @@ -364,8 +364,8 @@ TEST_F("require that lazy compiled ranking expressions are pure", Fixture()) { TEST_F("require that interpreted ranking expressions are pure", Fixture()) { f1.add_expr("rank", "box(value(7))").compile(); - EXPECT_EQUAL(4u, count_features(f1.program)); - EXPECT_EQUAL(4u, count_const_features(f1.program)); + EXPECT_EQUAL(3u, count_features(f1.program)); + EXPECT_EQUAL(3u, count_const_features(f1.program)); EXPECT_EQUAL(f1.get(), 7.0); } diff --git a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp index 3662e371839..1b1550354d7 100644 --- a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp +++ b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp @@ -9,24 +9,7 @@ LOG_SETUP("fileheadertk_test"); using namespace search; -class Test : public vespalib::TestApp { -private: - void testVersionTags(); - -public: - int Main() override { - TEST_INIT("fileheadertk_test"); - - testVersionTags(); TEST_FLUSH(); - - TEST_DONE(); - } -}; - -TEST_APPHOOK(Test); - -void -Test::testVersionTags() +TEST("testVersionTags") { vespalib::FileHeader header; FileHeaderTk::addVersionTags(header); @@ -34,7 +17,6 @@ Test::testVersionTags() FastOS_File file; ASSERT_TRUE(file.OpenWriteOnlyTruncate("versiontags.dat")); EXPECT_EQUAL(header.getSize(), header.writeFile(file)); - file.Close(); EXPECT_EQUAL(8u, header.getNumTags()); EXPECT_TRUE(header.hasTag("version-arch")); @@ -46,3 +28,5 @@ Test::testVersionTags() EXPECT_TRUE(header.hasTag("version-tag")); EXPECT_TRUE(header.hasTag("version-pkg")); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index 9a326af4c3b..34f9f7d27a9 100644 --- a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/memoryindex/feature_store.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp index cdfe1cfca88..cc9e90bb761 100644 --- a/searchlib/src/tests/queryeval/queryeval.cpp +++ b/searchlib/src/tests/queryeval/queryeval.cpp @@ -339,7 +339,7 @@ class DummySingleValueBitNumericAttributeBlueprint : public SimpleLeafBlueprint public: DummySingleValueBitNumericAttributeBlueprint(const SimpleResult & result) : SimpleLeafBlueprint(FieldSpecBaseList()), - _a("a", search::GrowStrategy()), + _a("a", search::GrowStrategy(), false), _sc(), _tfmd() { diff --git a/searchlib/src/tests/stringenum/stringenum_test.cpp b/searchlib/src/tests/stringenum/stringenum_test.cpp index 15ec1862338..f5915db6df0 100644 --- a/searchlib/src/tests/stringenum/stringenum_test.cpp +++ b/searchlib/src/tests/stringenum/stringenum_test.cpp @@ -1,43 +1,28 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("stringenum"); -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/stringenum.h> - +LOG_SETUP("stringenum"); #include <vespa/vespalib/testkit/testapp.h> using namespace vespalib; -class MyApp : public vespalib::TestApp -{ -public: - void CheckLookup( search::util::StringEnum *strEnum, const char *str, int value); - int Main() override; - MyApp() {} -}; - - void -MyApp::CheckLookup( search::util::StringEnum *strEnum, const char *str, int value) +CheckLookup( search::util::StringEnum *strEnum, const char *str, int value) { EXPECT_EQUAL(0, strcmp(str, strEnum->Lookup(value))); EXPECT_EQUAL(value, strEnum->Lookup(str)); } -int -MyApp::Main() +TEST("test StringEnum Add and Lookup") { - TEST_INIT("stringenum_test"); search::util::StringEnum enum1; - search::util::StringEnum enum2; // check number of entries EXPECT_EQUAL(enum1.GetNumEntries(), 0u); - EXPECT_EQUAL(enum2.GetNumEntries(), 0u); // check add non-duplicates EXPECT_EQUAL(enum1.Add("zero"), 0); @@ -80,63 +65,11 @@ MyApp::Main() TEST_DO(CheckLookup(&enum1, "nine", 9)); TEST_DO(CheckLookup(&enum1, "ten", 10)); - TEST_FLUSH(); - - // save/load - EXPECT_TRUE(enum1.Save("tmp.enum")); - EXPECT_TRUE(enum2.Load("tmp.enum")); - - // check mapping and reverse mapping - EXPECT_EQUAL(enum2.GetNumEntries(), 11u); - TEST_DO(CheckLookup(&enum2, "zero", 0)); - TEST_DO(CheckLookup(&enum2, "one", 1)); - TEST_DO(CheckLookup(&enum2, "two", 2)); - TEST_DO(CheckLookup(&enum2, "three", 3)); - TEST_DO(CheckLookup(&enum2, "four", 4)); - TEST_DO(CheckLookup(&enum2, "five", 5)); - TEST_DO(CheckLookup(&enum2, "six", 6)); - TEST_DO(CheckLookup(&enum2, "seven", 7)); - TEST_DO(CheckLookup(&enum2, "eight", 8)); - TEST_DO(CheckLookup(&enum2, "nine", 9)); - TEST_DO(CheckLookup(&enum2, "ten", 10)); - - // add garbage - enum2.Add("sfsdffgdfh"); - enum2.Add("sf24dfsgg3"); - enum2.Add("sfwertfgdh"); - enum2.Add("sfewrgtsfh"); - enum2.Add("sfgdsdgdfh"); - - TEST_FLUSH(); - - // reload - EXPECT_TRUE(enum2.Load("tmp.enum")); - - // check garbage lost - EXPECT_EQUAL(enum2.GetNumEntries(), 11u); - EXPECT_EQUAL(-1, enum2.Lookup("sfewrgtsfh")); - // check mapping and reverse mapping - TEST_DO(CheckLookup(&enum2, "zero", 0)); - TEST_DO(CheckLookup(&enum2, "one", 1)); - TEST_DO(CheckLookup(&enum2, "two", 2)); - TEST_DO(CheckLookup(&enum2, "three", 3)); - TEST_DO(CheckLookup(&enum2, "four", 4)); - TEST_DO(CheckLookup(&enum2, "five", 5)); - TEST_DO(CheckLookup(&enum2, "six", 6)); - TEST_DO(CheckLookup(&enum2, "seven", 7)); - TEST_DO(CheckLookup(&enum2, "eight", 8)); - TEST_DO(CheckLookup(&enum2, "nine", 9)); - TEST_DO(CheckLookup(&enum2, "ten", 10)); - // clear enum1.Clear(); - enum2.Clear(); // check number of entries EXPECT_EQUAL(enum1.GetNumEntries(), 0u); - EXPECT_EQUAL(enum2.GetNumEntries(), 0u); - - TEST_DONE(); } -TEST_APPHOOK(MyApp); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index f0e156a96ed..7abc83b0047 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/typed_cells.h> +#include <vespa/searchlib/common/geo_gcd.h> #include <vespa/searchlib/tensor/distance_functions.h> #include <vespa/searchlib/tensor/distance_function_factory.h> #include <vespa/vespalib/gtest/gtest.h> @@ -33,6 +34,10 @@ void verify_geo_miles(const DistanceFunction *dist_fun, EXPECT_LE(d_miles, exp_miles*1.01); double threshold = dist_fun->convert_threshold(km); EXPECT_DOUBLE_EQ(threshold, abstract_distance); + // compare with common Great Circle Distance implementation: + search::common::GeoGcd gp1{p1[0], p1[1]}; + double km_gcd = gp1.km_great_circle_distance(p2[0], p2[1]); + EXPECT_DOUBLE_EQ(km, km_gcd); } else { EXPECT_LE(d_miles, 7e-13); EXPECT_LE(abstract_distance, 6e-33); diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index d3c3af3a9ca..fa0753373bf 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -967,7 +967,6 @@ TEST("test truncation after short read") { FastOS_File trfile(filename.c_str()); EXPECT_TRUE(trfile.OpenReadWrite(nullptr)); trfile.SetSize(trfile.getSize() - 1); - trfile.Close(); } { TransLogServer tlss(topdir.getDir(), 18377, ".", fileHeaderContext, domainConfig); diff --git a/searchlib/src/tests/util/rawbuf_test.cpp b/searchlib/src/tests/util/rawbuf_test.cpp index e24314336af..fd77b5b4ddb 100644 --- a/searchlib/src/tests/util/rawbuf_test.cpp +++ b/searchlib/src/tests/util/rawbuf_test.cpp @@ -118,8 +118,8 @@ TEST("require that prealloc makes enough room") { TEST("require that rawbuf can read from file") { FastOS_File file("mytemporaryfile"); - file.OpenReadWrite(); - file.Write2("barbaz", 6); + ASSERT_TRUE(file.OpenReadWrite()); + ASSERT_EQUAL(6, file.Write2("barbaz", 6)); file.SetPosition(0); RawBuf buf(10); @@ -129,7 +129,7 @@ TEST("require that rawbuf can read from file") { buf.readFile(file, 100); EXPECT_EQUAL("foobarbaz", getString(buf)); - file.Close(); + ASSERT_TRUE(file.Close()); file.Delete(); } diff --git a/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp index 15c7e114761..7ea480bf542 100644 --- a/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp +++ b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp @@ -17,7 +17,6 @@ bool writeHeader(const FileHeader &header, const vespalib::string &fileName) { if (!EXPECT_EQUAL(header.getSize(), header.writeFile(file))) { return false; } - file.Close(); return true; } @@ -30,7 +29,6 @@ vespalib::string readFile(const vespalib::string &fileName) { EXPECT_LESS(len, sizeof(buf)); // make sure we got everything vespalib::string str(buf, len); - file.Close(); return str; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index 30f417c7532..8a3885393b3 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -48,8 +48,8 @@ updateHeader(const vespalib::string &name, uint64_t fileBitSize) h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", fileBitSize)); h.rewriteFile(f); - f.Sync(); - f.Close(); + bool sync_ok = f.Sync(); + assert(sync_ok); } /* @@ -155,8 +155,10 @@ void AttributeFileWriter::close() { if (_file->IsOpened()) { - _file->Sync(); - _file->Close(); + bool synk_ok = _file->Sync(); + assert(synk_ok); + bool close_ok = _file->Close(); + assert(close_ok); updateHeader(_file->GetFileName(), _fileBitSize); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index a2ac482ebf3..4c3134d3235 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -23,6 +23,7 @@ #include <vespa/searchlib/util/file_settings.h> #include <vespa/searchlib/util/logutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> @@ -107,6 +108,34 @@ AttributeVector::ValueModifier::~ValueModifier() { } } +namespace { + +bool +allow_paged(const search::attribute::Config& config) +{ + if (!config.paged()) { + return false; + } + using Type = search::attribute::BasicType::Type; + if (config.basicType() == Type::REFERENCE || config.basicType() == Type::PREDICATE) { + return false; + } + if (config.basicType() == Type::TENSOR) { + return (!config.tensorType().is_error() && config.tensorType().is_dense()); + } + return true; +} + +std::unique_ptr<vespalib::alloc::MemoryAllocator> +make_memory_allocator(const vespalib::string& name, const search::attribute::Config& config) +{ + if (allow_paged(config)) { + return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); + } + return {}; +} + +} AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config &c) : _baseFileName(baseFileName), @@ -124,7 +153,9 @@ AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config _compactLidSpaceGeneration(0u), _hasEnum(false), _loaded(false), - _isUpdateableInMemoryOnly(attribute::isUpdateableInMemoryOnly(getName(), getConfig())) + _isUpdateableInMemoryOnly(attribute::isUpdateableInMemoryOnly(getName(), getConfig())), + _nextStatUpdateTime(), + _memory_allocator(make_memory_allocator(_baseFileName.getAttributeName(), c)) { } @@ -805,6 +836,12 @@ AttributeVector::update_config(const Config& cfg) drain_hold(1_Mi); // Wait until 1MiB or less on hold } +vespalib::alloc::Alloc +AttributeVector::get_initial_alloc() +{ + return (_memory_allocator ? vespalib::alloc::Alloc::alloc_with_allocator(_memory_allocator.get()) : vespalib::alloc::Alloc::alloc()); +} + template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool); template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &); template bool AttributeVector::remove<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 90d08fa681c..8b42b19cc60 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -26,9 +26,6 @@ #include <mutex> #include <shared_mutex> -class Fast_BufferedFile; -class FastOS_FileInterface; - namespace document { class ArithmeticValueUpdate; class AssignValueUpdate; @@ -380,6 +377,8 @@ protected: virtual vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const; virtual void populate_address_space_usage(AddressSpaceUsage& usage) const; + const std::shared_ptr<vespalib::alloc::MemoryAllocator>& get_memory_allocator() const noexcept { return _memory_allocator; } + vespalib::alloc::Alloc get_initial_alloc(); public: DECLARE_IDENTIFIABLE_ABSTRACT(AttributeVector); bool isLoaded() const { return _loaded; } @@ -587,6 +586,7 @@ private: bool _loaded; bool _isUpdateableInMemoryOnly; vespalib::steady_time _nextStatUpdateTime; + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; ////// Locking strategy interface. only available from the Guards. /** diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index 87f1985d7c0..f398fe0b46b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -4,7 +4,6 @@ #include "attrvector.h" #include "load_utils.h" #include <vespa/vespalib/util/hdr_abort.h> -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filekit.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp index 71eb88d4e52..66e6e7ccd4d 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp @@ -24,7 +24,7 @@ AttributeFactory::createSingleFastSearch(stringref name, const Config & info) assert(info.fastSearch()); switch(info.basicType().type()) { case BasicType::BOOL: - return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy()); + return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy(), info.paged()); case BasicType::UINT2: case BasicType::UINT4: break; diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index 3294ecd8dd2..fe2b0c9f989 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -21,7 +21,7 @@ AttributeFactory::createSingleStd(stringref name, const Config & info) assert(info.collectionType().type() == attribute::CollectionType::SINGLE); switch(info.basicType().type()) { case BasicType::BOOL: - return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy()); + return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy(), info.paged()); case BasicType::UINT2: return std::make_shared<SingleValueSemiNibbleNumericAttribute>(name, info.getGrowStrategy()); case BasicType::UINT4: diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index c0680fd9238..24dfe742120 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -14,7 +14,7 @@ EnumAttribute<B>:: EnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - _enumStore(cfg.fastSearch(), cfg.get_dictionary_config()) + _enumStore(cfg.fastSearch(), cfg.get_dictionary_config(), this->get_memory_allocator()) { this->setEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 7fe586b8ccc..73378311bec 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -77,6 +77,7 @@ private: std::unique_ptr<EntryComparator> allocate_optionally_folded_comparator(bool folded) const; ComparatorType make_optionally_folded_comparator(bool folded) const; public: + EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); EnumStoreT(bool has_postings, const search::DictionaryConfig & dict_cfg); ~EnumStoreT() override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index e1adca2b89a..fa0b8977c8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -72,8 +72,8 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig & dict_cfg) - : _store(), +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : _store(std::move(memory_allocator)), _dict(), _is_folded(dict_cfg.getMatch() == DictionaryConfig::Match::UNCASED), _comparator(_store.get_data_store()), @@ -87,6 +87,12 @@ EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig & dict_ } template <typename EntryT> +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg) + : EnumStoreT<EntryT>(has_postings, dict_cfg, {}) +{ +} + +template <typename EntryT> EnumStoreT<EntryT>::~EnumStoreT() = default; template <typename EntryT> diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index 1491a92166a..161e2883cbc 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -5,8 +5,8 @@ #include "multi_value_mapping.h" #include "multi_value_mapping.hpp" #include "multivalue.h" -#include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/util/array.hpp> using search::multivalue::Value; using search::multivalue::WeightedValue; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 81abaa05a45..f5f2950a59c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -27,7 +27,8 @@ public: MultiValueMapping(const MultiValueMapping &) = delete; MultiValueMapping & operator = (const MultiValueMapping &) = delete; MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, - const vespalib::GrowStrategy &gs = vespalib::GrowStrategy()); + const vespalib::GrowStrategy &gs, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~MultiValueMapping() override; ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); } ConstArrayRef getDataForIdx(EntryRef idx) const { return _store.get(idx); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index fb81a60cb13..16b29bf33cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -10,16 +10,17 @@ namespace search::attribute { template <typename EntryT, typename RefT> MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, - const vespalib::GrowStrategy &gs) + const vespalib::GrowStrategy &gs, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) #ifdef __clang__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wuninitialized" #endif - : MultiValueMappingBase(gs, _store.getGenerationHolder()), + : MultiValueMappingBase(gs, _store.getGenerationHolder(), memory_allocator), #ifdef __clang__ #pragma clang diagnostic pop #endif - _store(storeCfg) + _store(storeCfg, std::move(memory_allocator)) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index b0d50c129c6..7ad61ccedc5 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -10,8 +10,10 @@ namespace search::attribute { using vespalib::datastore::CompactionStrategy; MultiValueMappingBase::MultiValueMappingBase(const vespalib::GrowStrategy &gs, - vespalib::GenerationHolder &genHolder) - : _indices(gs, genHolder), + vespalib::GenerationHolder &genHolder, + std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : _memory_allocator(std::move(memory_allocator)), + _indices(gs, genHolder, _memory_allocator ? vespalib::alloc::Alloc::alloc_with_allocator(_memory_allocator.get()) : vespalib::alloc::Alloc::alloc()), _totalValues(0u), _compaction_spec() { diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h index f27a9f1667c..2b2b4d5f8a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h @@ -27,11 +27,12 @@ public: using RefVector = vespalib::RcuVectorBase<EntryRef>; protected: + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; RefVector _indices; size_t _totalValues; CompactionSpec _compaction_spec; - MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder); + MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); virtual ~MultiValueMappingBase(); void updateValueCount(size_t oldValues, size_t newValues) { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 10f837ec1ab..73d248a21fa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -8,7 +8,6 @@ #include "multinumericattributesaver.h" #include "load_utils.h" #include "primitivereader.h" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 5449f85bf68..4322faefe67 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -6,7 +6,6 @@ #include "load_utils.h" #include "loadednumericvalue.h" #include "multinumericenumattribute.h" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.hpp> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index 89cd8ca310a..c8330225fb9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -6,7 +6,6 @@ #include "multistringattribute.h" #include "enumattribute.hpp" #include "multienumattribute.hpp" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/util/bufferwriter.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 2bb4d2ada60..a1f8b42df2f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -5,7 +5,6 @@ #include "stringattribute.h" #include "multistringpostattribute.h" #include "multistringattribute.hpp" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/query/query_term_simple.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index b9ef16c6adf..326dbd4a380 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -28,7 +28,7 @@ MultiValueAttribute(const vespalib::string &baseFileName, 8 * 1024, cfg.getGrowStrategy().getMultiValueAllocGrowFactor(), multivalueattribute::enable_free_lists), - cfg.getGrowStrategy().to_generic_strategy()) + cfg.getGrowStrategy().to_generic_strategy(), this->get_memory_allocator()) { } diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp index f08130b9fbe..d023d9b56b1 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp @@ -5,7 +5,6 @@ #include "readerbase.h" #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filesizecalculator.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 4212a4ad247..f7317f2d8c6 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -41,7 +41,7 @@ extractUniqueValueCount(const vespalib::GenericHeader &header) ReferenceAttribute::ReferenceAttribute(const vespalib::stringref baseFileName, const Config & cfg) : NotImplementedAttribute(baseFileName, cfg), - _store(), + _store({}), _indices(getGenerationHolder()), _compaction_spec(), _gidToLidMapperFactory(), diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 3fe7d147c1d..474265c914b 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -17,9 +17,10 @@ namespace search { using attribute::Config; SingleBoolAttribute:: -SingleBoolAttribute(const vespalib::string &baseFileName, const GrowStrategy & grow) - : IntegerAttributeTemplate<int8_t>(baseFileName, Config(BasicType::BOOL, CollectionType::SINGLE).setGrowStrategy(grow), BasicType::BOOL), - _bv(0, 0, getGenerationHolder()) +SingleBoolAttribute(const vespalib::string &baseFileName, const GrowStrategy & grow, bool paged) + : IntegerAttributeTemplate<int8_t>(baseFileName, Config(BasicType::BOOL, CollectionType::SINGLE).setGrowStrategy(grow).setPaged(paged), BasicType::BOOL), + _init_alloc(get_initial_alloc()), + _bv(0, 0, getGenerationHolder(), get_memory_allocator() ? &_init_alloc : nullptr) { } diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 78c07719271..469bd54fa55 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -15,7 +15,7 @@ namespace search { class SingleBoolAttribute final : public IntegerAttributeTemplate<int8_t> { public: - SingleBoolAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow); + SingleBoolAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow, bool paged); ~SingleBoolAttribute() override; void onCommit() override; @@ -108,6 +108,7 @@ private: int8_t getFast(DocId doc) const { return _bv.testBit(doc) ? 1 : 0; } + vespalib::alloc::Alloc _init_alloc; GrowableBitVector _bv; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index 18805a7b20f..cffd52f3dc4 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -14,11 +14,11 @@ namespace search { using attribute::Config; SingleValueEnumAttributeBase:: -SingleValueEnumAttributeBase(const Config & c, GenerationHolder &genHolder) +SingleValueEnumAttributeBase(const Config & c, GenerationHolder &genHolder, const vespalib::alloc::Alloc& initial_alloc) : _enumIndices(c.getGrowStrategy().getDocsInitialCapacity(), c.getGrowStrategy().getDocsGrowPercent(), c.getGrowStrategy().getDocsGrowDelta(), - genHolder) + genHolder, initial_alloc) { } @@ -49,7 +49,7 @@ SingleValueEnumAttributeBase::remap_enum_store_refs(const EnumIndexRemapper& rem { // update _enumIndices with new EnumIndex values after enum store has been compacted. v.logEnumStoreEvent("reenumerate", "reserved"); - vespalib::Array<EnumIndex> new_indexes; + auto new_indexes = _enumIndices.create_replacement_vector(); new_indexes.reserve(_enumIndices.capacity()); v.logEnumStoreEvent("reenumerate", "start"); auto& filter = remapper.get_entry_ref_filter(); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h index a6bcfd92449..8fcef670fb0 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h @@ -30,7 +30,7 @@ public: IEnumStore::Index getEnumIndex(DocId docId) const { return _enumIndices[docId]; } EnumHandle getE(DocId doc) const { return _enumIndices[doc].ref(); } protected: - SingleValueEnumAttributeBase(const attribute::Config & c, GenerationHolder &genHolder); + SingleValueEnumAttributeBase(const attribute::Config & c, GenerationHolder &genHolder, const vespalib::alloc::Alloc& initial_alloc); ~SingleValueEnumAttributeBase(); AttributeVector::DocId addDoc(bool & incGeneration); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index a51c9804cf2..56aa6672696 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -16,7 +16,7 @@ SingleValueEnumAttribute<B>:: SingleValueEnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - SingleValueEnumAttributeBase(cfg, getGenerationHolder()) + SingleValueEnumAttributeBase(cfg, getGenerationHolder(), this->get_initial_alloc()) { } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 89c2a21231b..ef618262bcd 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -19,7 +19,8 @@ SingleValueNumericAttribute(const vespalib::string & baseFileName, const Attribu _data(c.getGrowStrategy().getDocsInitialCapacity(), c.getGrowStrategy().getDocsGrowPercent(), c.getGrowStrategy().getDocsGrowDelta(), - getGenerationHolder()) + getGenerationHolder(), + this->get_initial_alloc()) { } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 606c99161be..730ad1107a7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -6,7 +6,6 @@ #include "stringattribute.h" #include "singleenumattribute.hpp" #include "attributevector.hpp" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/util/bufferwriter.h> diff --git a/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp index b46866e61e2..3bc85783c2f 100644 --- a/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp @@ -95,7 +95,6 @@ SourceSelector::LoadInfo::load() if (fileHeader.hasTag(docIdLimitTag)) { _header._docIdLimit = fileHeader.getTag(docIdLimitTag).asInteger(); } - file.Close(); } SourceSelector::SourceSelector(Source defaultSource, AttributeVector::SP realSource) : @@ -106,14 +105,14 @@ SourceSelector::SourceSelector(Source defaultSource, AttributeVector::SP realSou SourceSelector::SaveInfo::UP SourceSelector::extractSaveInfo(const vespalib::string & baseFileName) { - return SaveInfo::UP(new SaveInfo(baseFileName, getDefaultSource(), getBaseId(), - getDocIdLimit(), *_realSource)); + return std::make_unique<SaveInfo>(baseFileName, getDefaultSource(), getBaseId(), + getDocIdLimit(), *_realSource); } SourceSelector::LoadInfo::UP SourceSelector::extractLoadInfo(const vespalib::string & baseFileName) { - return LoadInfo::UP(new LoadInfo(baseFileName)); + return std::make_unique<LoadInfo>(baseFileName); } SourceSelector::Histogram SourceSelector::getDistribution() const diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 0d5835b4fa9..5ac506e4fc2 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -3,6 +3,7 @@ #include "compression.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/arrayref.h> diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 973d622461a..45005d499fb 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/util/comprfile.h> -#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/stllike/string.h> #include <cassert> @@ -14,7 +13,10 @@ template <typename T> class ConstArrayRef; } -namespace search::index { class DocIdAndFeatures; } +namespace search::index { + class DocIdAndFeatures; + class PostingListParams; +} namespace search::fef { class TermFieldMatchDataArray; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h index 664a1245c2f..6eb37e1d1ad 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h @@ -19,15 +19,13 @@ public: uint32_t _minChunkDocs; // Minimum number of documents for chunking uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) uint64_t _numWordIds; // Number of words in dictionary - uint64_t _minWordNum; // Minimum word number PostingListCountFileDecodeContext() : ParentClass(), _avgBitsPerDoc(10), _minChunkDocs(262144), _docIdLimit(10000000), - _numWordIds(0), - _minWordNum(0u) + _numWordIds(0) { } @@ -50,15 +48,13 @@ public: uint32_t _minChunkDocs; // Minimum number of documents for chunking uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) uint64_t _numWordIds; // Number of words in dictionary - uint64_t _minWordNum; // Mininum word number PostingListCountFileEncodeContext() : ParentClass(), _avgBitsPerDoc(10), _minChunkDocs(262144), _docIdLimit(10000000), - _numWordIds(0), - _minWordNum(0u) + _numWordIds(0) { } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index 76a65a7244a..fd6c723e901 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compression.h" #include "posocccompression.h" #include "posocc_fields_params.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> @@ -36,8 +36,7 @@ EG2PosOccDecodeContext<bigEndian>:: readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix) { - const_cast<PosOccFieldsParams *>(_fieldsParams)->readHeader(header, - prefix); + const_cast<PosOccFieldsParams *>(_fieldsParams)->readHeader(header, prefix); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h index 184e2414638..aadd58f9152 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "compression.h" #include <vespa/searchlib/index/docidandfeatures.h> -#include <vespa/searchcommon/common/schema.h> #define K_VALUE_POSOCC_FIRST_WORDPOS 8 @@ -16,35 +16,6 @@ #define K_VALUE_POSOCC_ELEMENTID 0 #define K_VALUE_POSOCC_ELEMENTWEIGHT 9 -namespace search::index { - -class DocIdAndPosOccFeatures : public DocIdAndFeatures -{ -public: - - void - addNextOcc(uint32_t elementId, - uint32_t wordPos, - int32_t elementWeight, - uint32_t elementLen) - { - assert(wordPos < elementLen); - if (_elements.empty() || elementId > _elements.back().getElementId()) { - _elements.emplace_back(elementId, elementWeight, elementLen); - } else { - assert(elementId == _elements.back().getElementId()); - assert(elementWeight == _elements.back().getWeight()); - assert(elementLen == _elements.back().getElementLen()); - } - assert(_elements.back().getNumOccs() == 0 || - wordPos > _word_positions.back().getWordPos()); - _elements.back().incNumOccs(); - _word_positions.emplace_back(wordPos); - } -}; - -} - namespace search::bitcompression { class PosOccFieldsParams; diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 73c8999520b..a7c8d56f11d 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(searchlib_common OBJECT featureset.cpp fileheadercontext.cpp flush_token.cpp + geo_gcd.cpp geo_location.cpp geo_location_parser.cpp geo_location_spec.cpp diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index d428ce52953..ab56317db6f 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -59,10 +59,10 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Alloc buffer, siz { } -AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBits, const void * rhsBuf, size_t rhsSize) : +AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBits, const void * rhsBuf, size_t rhsSize, const Alloc* init_alloc) : BitVector(), _capacityBits(capacityBits), - _alloc(allocatePaddedAndAligned(0, numberOfElements, capacityBits)) + _alloc(allocatePaddedAndAligned(0, numberOfElements, capacityBits, init_alloc)) { _capacityBits = computeCapacity(_capacityBits, _alloc.size()); init(_alloc.get(), 0, numberOfElements); @@ -104,17 +104,9 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, std::pair<Index, I AllocatedBitVector::~AllocatedBitVector() = default; void -AllocatedBitVector::cleanup() -{ - init(nullptr, 0, 0); - Alloc().swap(_alloc); - _capacityBits = 0; -} - -void AllocatedBitVector::resize(Index newLength) { - _alloc = allocatePaddedAndAligned(newLength); + _alloc = allocatePaddedAndAligned(0, newLength, newLength, &_alloc); _capacityBits = computeCapacity(newLength, _alloc.size()); init(_alloc.get(), 0, newLength); clear(); @@ -145,7 +137,7 @@ AllocatedBitVector::grow(Index newSize, Index newCapacity) assert(newCapacity >= newSize); GenerationHeldBase::UP ret; if (newCapacity != capacity()) { - AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size()); + AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); if (newSize > size()) { tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index c57ca93ac02..2223c7b4701 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -35,7 +35,7 @@ public: * Creates a new bitvector with size of numberOfElements bits and at least a capacity of capacity. * Copies what it can from the original vector. This is used for extending vector. */ - AllocatedBitVector(Index numberOfElements, Index capacity, const void * rhsBuf, size_t rhsSize); + AllocatedBitVector(Index numberOfElements, Index capacity, const void * rhsBuf, size_t rhsSize, const Alloc* init_alloc); AllocatedBitVector(const BitVector &other); AllocatedBitVector(const AllocatedBitVector &other); @@ -74,12 +74,6 @@ private: } AllocatedBitVector(const BitVector &other, std::pair<Index, Index> size_capacity); - - /** - * Prepare for potential reuse where new value might be filled in by - * Read method. - */ - void cleanup(); }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index a3a1b9bed66..d2e6d9027ea 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -47,13 +47,13 @@ using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; Alloc -BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity) +BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity, const Alloc* init_alloc) { assert(capacity >= end); uint32_t words = numActiveWords(start, capacity); words += (-words & 15); // Pad to 64 byte alignment const size_t sz(words * sizeof(Word)); - Alloc alloc = Alloc::alloc(sz, MMAP_LIMIT); + Alloc alloc = (init_alloc != nullptr) ? init_alloc->create(sz) : Alloc::alloc(sz, MMAP_LIMIT); assert(alloc.size()/sizeof(Word) >= words); // Clear padding size_t usedBytes = numBytes(end - start); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 184669829cb..c1d447047d1 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -281,7 +281,7 @@ protected: static Alloc allocatePaddedAndAligned(Index start, Index end) { return allocatePaddedAndAligned(start, end, end); } - static Alloc allocatePaddedAndAligned(Index start, Index end, Index capacity); + static Alloc allocatePaddedAndAligned(Index start, Index end, Index capacity, const Alloc* init_alloc = nullptr); private: friend PartialBitVector; diff --git a/searchlib/src/vespa/searchlib/common/documentsummary.cpp b/searchlib/src/vespa/searchlib/common/documentsummary.cpp index 8e70ffa7844..55ecc558598 100644 --- a/searchlib/src/vespa/searchlib/common/documentsummary.cpp +++ b/searchlib/src/vespa/searchlib/common/documentsummary.cpp @@ -33,7 +33,6 @@ DocumentSummary::readDocIdLimit(const vespalib::string &dir, uint32_t &count) p = qcntfile.ReadLine(numbuf, sizeof(numbuf)); while (*p >= '0' && *p <= '9') qcnt = qcnt * 10 + *p++ - '0'; - qcntfile.Close(); count = qcnt; return true; } @@ -52,8 +51,14 @@ DocumentSummary::writeDocIdLimit(const vespalib::string &dir, uint32_t count) } qcntfile.addNum(count, 0, ' '); qcntfile.WriteByte('\n'); - qcntfile.Sync(); - qcntfile.Close(); + if ( ! qcntfile.Sync() ) { + LOG(error, "Could not sync %s: %s", qcntname.c_str(), getLastErrorString().c_str()); + return false; + } + if ( ! qcntfile.Close() ) { + LOG(error, "Could not sync %s: %s", qcntname.c_str(), getLastErrorString().c_str()); + return false; + } return true; } diff --git a/searchlib/src/vespa/searchlib/common/geo_gcd.cpp b/searchlib/src/vespa/searchlib/common/geo_gcd.cpp new file mode 100644 index 00000000000..a7fc870fc31 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_gcd.cpp @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "geo_gcd.h" + +namespace search::common { + +namespace { + +// in km, as defined by IUGG, see: +// https://en.wikipedia.org/wiki/Earth_radius#Mean_radius +static constexpr double earth_mean_radius = 6371.0088; + +static constexpr double degrees_to_radians = M_PI / 180.0; + +// with input in radians +double greatCircleDistance(double theta_A, double phi_A, + double theta_B, double phi_B) +{ + // convert to radians: + double theta_diff = theta_A - theta_B; + double phi_diff = phi_A - phi_B; + // haversines of differences: + double hav_theta = GeoGcd::haversine(theta_diff); + double hav_phi = GeoGcd::haversine(phi_diff); + // haversine of central angle between the two points: + double hav_central_angle = hav_theta + cos(theta_A)*cos(theta_B)*hav_phi; + // sine of half the central angle: + double half_sine_diff = sqrt(hav_central_angle); + // distance in kilometers: + double d = 2 * asin(half_sine_diff) * earth_mean_radius; + return d; +} + +} + +GeoGcd::GeoGcd(double lat, double lng) + : _latitude_radians(lat * degrees_to_radians), + _longitude_radians(lng * degrees_to_radians) +{} + + +double GeoGcd::km_great_circle_distance(double lat, double lng) const { + double theta_A = _latitude_radians; + double phi_A = _longitude_radians; + double theta_B = lat * degrees_to_radians; + double phi_B = lng * degrees_to_radians; + return greatCircleDistance(theta_A, phi_A, theta_B, phi_B); +} + +} // namespace search::common diff --git a/searchlib/src/vespa/searchlib/common/geo_gcd.h b/searchlib/src/vespa/searchlib/common/geo_gcd.h new file mode 100644 index 00000000000..acd10207057 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_gcd.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <cmath> + +namespace search::common { + +/** + * An immutable struct for a (geo) location point, + * with methods for computing Great Circle Distance + * using the haversine formula + **/ +struct GeoGcd +{ + GeoGcd(double lat, double lng); + + // haversine function: + static constexpr double haversine(double angle) { + double s = sin(0.5*angle); + return s*s; + } + + double km_great_circle_distance(double lat, double lng) const; +private: + double _latitude_radians; + double _longitude_radians; +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 177266c5618..7c29945292e 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -10,8 +10,9 @@ using vespalib::GenerationHeldBase; using vespalib::GenerationHolder; GrowableBitVector::GrowableBitVector(Index newSize, Index newCapacity, - GenerationHolder &generationHolder) - : AllocatedBitVector(newSize, newCapacity, nullptr, 0), + GenerationHolder &generationHolder, + const Alloc* init_alloc) + : AllocatedBitVector(newSize, newCapacity, nullptr, 0, init_alloc), _generationHolder(generationHolder) { assert(newSize <= newCapacity); diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 08c9fede8f2..71261e130b6 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -9,7 +9,7 @@ namespace search { class GrowableBitVector : public AllocatedBitVector { public: - GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder); + GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder, const Alloc* init_alloc = nullptr); /** Will return true if a a buffer is held */ bool reserve(Index newCapacity); diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 6a8a1201d36..c5e271a047b 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -18,7 +18,7 @@ basedir string default="tmp" restart usefsync bool default=true ##Number of threads available for visiting/subscription. -maxthreads int default=4 restart +maxthreads int default=0 restart ##Default crc method used crcmethod enum {ccitt_crc32, xxh64} default=xxh64 diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 74a873a4e29..0f7c77f8451 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -11,6 +11,8 @@ vespa_add_library(searchlib_diskindex OBJECT docidmapper.cpp extposocc.cpp field_merger.cpp + field_mergers_state.cpp + field_merger_task.cpp fieldreader.cpp fieldwriter.cpp field_length_scanner.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp index 12866e74137..f6dd5a318ae 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp @@ -19,12 +19,7 @@ BitVectorDictionary::BitVectorDictionary() { } -BitVectorDictionary::~BitVectorDictionary() -{ - if (_datFile) { - _datFile->Close(); - } -} +BitVectorDictionary::~BitVectorDictionary() = default; bool @@ -35,31 +30,33 @@ BitVectorDictionary::open(const vespalib::string &pathPrefix, vespalib::string booloccIdxName = pathPrefix + "boolocc" + getBitVectorKeyScopeSuffix(scope); vespalib::string booloccDatName = pathPrefix + "boolocc.bdat"; - FastOS_File idxFile; - idxFile.OpenReadOnly(booloccIdxName.c_str()); - if (!idxFile.IsOpened()) { - LOG(warning, "Could not open bitvector idx file '%s'", - booloccIdxName.c_str()); - return false; - } + { + FastOS_File idxFile; + idxFile.OpenReadOnly(booloccIdxName.c_str()); + if (!idxFile.IsOpened()) { + LOG(warning, "Could not open bitvector idx file '%s'", + booloccIdxName.c_str()); + return false; + } - vespalib::FileHeader idxHeader; - uint32_t idxHeaderLen = idxHeader.readFile(idxFile); - idxFile.SetPosition(idxHeaderLen); - assert(idxHeader.hasTag("frozen")); - assert(idxHeader.hasTag("docIdLimit")); - assert(idxHeader.hasTag("numKeys")); - assert(idxHeader.getTag("frozen").asInteger() != 0); - _docIdLimit = idxHeader.getTag("docIdLimit").asInteger(); - uint32_t numEntries = idxHeader.getTag("numKeys").asInteger(); + vespalib::FileHeader idxHeader; + uint32_t idxHeaderLen = idxHeader.readFile(idxFile); + idxFile.SetPosition(idxHeaderLen); + assert(idxHeader.hasTag("frozen")); + assert(idxHeader.hasTag("docIdLimit")); + assert(idxHeader.hasTag("numKeys")); + assert(idxHeader.getTag("frozen").asInteger() != 0); + _docIdLimit = idxHeader.getTag("docIdLimit").asInteger(); + uint32_t numEntries = idxHeader.getTag("numKeys").asInteger(); - _entries.resize(numEntries); - size_t bufSize = sizeof(WordSingleKey) * numEntries; - assert(idxFile.GetSize() >= static_cast<int64_t>(idxHeaderLen + bufSize)); - if (bufSize > 0) { - idxFile.Read(&_entries[0], bufSize); + _entries.resize(numEntries); + size_t bufSize = sizeof(WordSingleKey) * numEntries; + assert(idxFile.GetSize() >= static_cast<int64_t>(idxHeaderLen + bufSize)); + if (bufSize > 0) { + ssize_t has_read = idxFile.Read(&_entries[0], bufSize); + assert(has_read == ssize_t(bufSize)); + } } - idxFile.Close(); _vectorSize = BitVector::getFileBytes(_docIdLimit); _datFile = std::make_unique<FastOS_File>(); @@ -79,7 +76,7 @@ BitVectorDictionary::open(const vespalib::string &pathPrefix, vespalib::FileHeader datHeader(64); _datHeaderLen = datHeader.readFile(*_datFile); assert(_datFile->GetSize() >= - static_cast<int64_t>(_vectorSize) * numEntries + _datHeaderLen); + static_cast<int64_t>(_vectorSize * _entries.size() + _datHeaderLen)); return true; } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index 65ce5decc50..70309645ee2 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/util/file_settings.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/fastlib/io/bufferedfile.h> #include <cassert> namespace search::diskindex { @@ -23,24 +24,19 @@ readHeader(vespalib::FileHeader &h, Fast_BufferedFile file(32_Ki); file.OpenReadOnly(name.c_str()); h.readFile(file); - file.Close(); } } BitVectorFileWrite::BitVectorFileWrite(BitVectorKeyScope scope) : BitVectorIdxFileWrite(scope), - _datFile(nullptr), + _datFile(), _datHeaderLen(0) { } -BitVectorFileWrite::~BitVectorFileWrite() -{ - // No implicit close() call, but cleanup memory allocations. - delete _datFile; -} +BitVectorFileWrite::~BitVectorFileWrite() = default; void @@ -51,12 +47,11 @@ BitVectorFileWrite::open(const vespalib::string &name, { vespalib::string datname = name + ".bdat"; - assert(_datFile == nullptr); + assert( ! _datFile); Parent::open(name, docIdLimit, tuneFileWrite, fileHeaderContext); - FastOS_FileInterface *datfile = new FastOS_File; - _datFile = new Fast_BufferedFile(datfile); + _datFile = std::make_unique<Fast_BufferedFile>(new FastOS_File); if (tuneFileWrite.getWantSyncWrites()) { _datFile->EnableSyncWrites(); } @@ -116,13 +111,13 @@ BitVectorFileWrite::updateDatHeader(uint64_t fileBitSize) h.putTag(Tag("numKeys", _numKeys)); h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", fileBitSize)); - _datFile->Flush(); - _datFile->Sync(); + bool sync_ok = _datFile->Sync(); + assert(sync_ok); assert(h.getSize() == _datHeaderLen); _datFile->SetPosition(0); h.writeFile(*_datFile); - _datFile->Flush(); - _datFile->Sync(); + sync_ok = _datFile->Sync(); + assert(sync_ok); } @@ -151,7 +146,8 @@ BitVectorFileWrite::sync() { flush(); Parent::syncCommon(); - _datFile->Sync(); + bool sync_ok = _datFile->Sync(); + assert(sync_ok); } @@ -168,10 +164,10 @@ BitVectorFileWrite::close() (void) bitmapbytes; _datFile->alignEndForDirectIO(); updateDatHeader(pos * 8); - _datFile->Close(); + bool close_ok = _datFile->Close(); + assert(close_ok); } - delete _datFile; - _datFile = nullptr; + _datFile.reset(); } Parent::close(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 37ca72d3ec0..007368babdd 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -2,12 +2,13 @@ #pragma once -#include <vespa/fastlib/io/bufferedfile.h> +#include "bitvectoridxfile.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/allocator.h> -#include "bitvectoridxfile.h" + +class Fast_BufferedFile; namespace search::diskindex { @@ -16,7 +17,7 @@ class BitVectorFileWrite : public BitVectorIdxFileWrite private: using Parent = BitVectorIdxFileWrite; - Fast_BufferedFile *_datFile; + std::unique_ptr<Fast_BufferedFile> _datFile; public: private: @@ -32,13 +33,13 @@ public: void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, - const common::FileHeaderContext &fileHeaderContext); + const common::FileHeaderContext &fileHeaderContext) override; void addWordSingle(uint64_t wordNum, const BitVector &bitVector); - void flush(); - void sync(); - void close(); + void flush() override; + void sync() override; + void close() override; void makeDatHeader(const common::FileHeaderContext &fileHeaderContext); void updateDatHeader(uint64_t fileBitSize); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index f85c6c20624..e87238bef2d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bitvectoridxfile.h" -#include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/util/file_settings.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/fastlib/io/bufferedfile.h> #include <cassert> namespace search::diskindex { @@ -22,7 +22,6 @@ readHeader(vespalib::FileHeader &h, const vespalib::string &name) Fast_BufferedFile file(32_Ki); file.OpenReadOnly(name.c_str()); h.readFile(file); - file.Close(); } } @@ -116,13 +115,13 @@ BitVectorIdxFileWrite::updateIdxHeader(uint64_t fileBitSize) if (_scope != BitVectorKeyScope::SHARED_WORDS) { h.putTag(Tag("fileBitSize", fileBitSize)); } - _idxFile->Flush(); - _idxFile->Sync(); + bool sync_ok = _idxFile->Sync(); + assert(sync_ok); assert(h.getSize() == _idxHeaderLen); _idxFile->SetPosition(0); h.writeFile(*_idxFile); - _idxFile->Flush(); - _idxFile->Sync(); + sync_ok = _idxFile->Sync(); + assert(sync_ok); } void @@ -148,7 +147,8 @@ BitVectorIdxFileWrite::flush() void BitVectorIdxFileWrite::syncCommon() { - _idxFile->Sync(); + bool sync_ok = _idxFile->Sync(); + assert(sync_ok); } void @@ -167,7 +167,8 @@ BitVectorIdxFileWrite::close() assert(pos == idxSize()); _idxFile->alignEndForDirectIO(); updateIdxHeader(pos * 8); - _idxFile->Close(); + bool close_ok = _idxFile->Close(); + assert(close_ok); } _idxFile.reset(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index a5b6226fd43..f814cd20f5a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -2,12 +2,13 @@ #pragma once -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> #include "bitvectorkeyscope.h" +class Fast_BufferedFile; + namespace search::common { class FileHeaderContext; } namespace search::diskindex { @@ -35,18 +36,18 @@ public: BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &&) = delete; BitVectorIdxFileWrite(BitVectorKeyScope scope); - ~BitVectorIdxFileWrite(); + virtual ~BitVectorIdxFileWrite(); - void open(const vespalib::string &name, uint32_t docIdLimit, - const TuneFileSeqWrite &tuneFileWrite, - const common::FileHeaderContext &fileHeaderContext); + virtual void open(const vespalib::string &name, uint32_t docIdLimit, + const TuneFileSeqWrite &tuneFileWrite, + const common::FileHeaderContext &fileHeaderContext); void addWordSingle(uint64_t wordNum, uint32_t numDocs); - void flush(); - void sync(); - void close(); + virtual void flush(); + virtual void sync(); + virtual void close(); static uint32_t getBitVectorLimit(uint32_t docIdLimit) { // Must match FastS_BinSizeParams::CalcMaxBinSize() diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp index a3b9259a278..7bc1bdbfb9b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp @@ -3,6 +3,7 @@ #include "dictionarywordreader.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/vespalib/util/error.h> +#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.dictionarywordreader"); @@ -43,21 +44,27 @@ DictionaryWordReader::open(const vespalib::string & dictionaryName, _old2newwordfile->EnableDirectIO(); } // no checking possible - _old2newwordfile->WriteOpen(wordMapName.c_str()); + _old2newwordfile->OpenWriteOnly(wordMapName.c_str()); _old2newwordfile->SetSize(0); return true; } void +DictionaryWordReader::writeNewWordNum(uint64_t newWordNum) { + _old2newwordfile->WriteBuf(&newWordNum, sizeof(newWordNum)); +} + +void DictionaryWordReader::close() { if (!_dictFile->close()) { LOG(error, "Error closing input dictionary"); } - _old2newwordfile->Flush(); - _old2newwordfile->Sync(); - _old2newwordfile->Close(); + bool sync_ok = _old2newwordfile->Sync(); + assert(sync_ok); + bool close_ok = _old2newwordfile->Close(); + assert(close_ok); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h index 18e886ca22e..5c5dc60f4e7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h @@ -2,7 +2,7 @@ #pragma once #include "pagedict4file.h" -#include <vespa/fastlib/io/bufferedfile.h> + namespace search::diskindex { @@ -47,7 +47,7 @@ public: private: // "owners" of file handles. - std::unique_ptr<Fast_BufferedFile> _old2newwordfile; + std::unique_ptr<FastOS_FileInterface> _old2newwordfile; using DictionaryFileSeqRead = index::DictionaryFileSeqRead; std::unique_ptr<DictionaryFileSeqRead> _dictFile; @@ -82,9 +82,7 @@ public: void close(); - void writeNewWordNum(uint64_t newWordNum) { - _old2newwordfile->WriteBuf(&newWordNum, sizeof(newWordNum)); - } + void writeNewWordNum(uint64_t newWordNum); void write(WordAggregator &writer) { writer.tryWriteWord(_word); diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.h b/searchlib/src/vespa/searchlib/diskindex/diskindex.h index 05492a59ee3..12be8979cc3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.h +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.h @@ -7,6 +7,7 @@ #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/field_length_info.h> #include <vespa/searchlib/queryeval/searchable.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/cache.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp index a4e9e4d06f7..dcf897df955 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp @@ -5,8 +5,8 @@ #include "fileheader.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/docidandfeatures.h> -#include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.extposocc"); diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp index 68672a0a930..dfd4d891818 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp @@ -43,6 +43,7 @@ constexpr uint32_t renumber_word_ids_heap_limit = 4; constexpr uint32_t renumber_word_ids_merge_chunk = 1000000; constexpr uint32_t merge_postings_heap_limit = 4; constexpr uint32_t merge_postings_merge_chunk = 50000; +constexpr uint32_t scan_chunk = 80000; vespalib::string createTmpPath(const vespalib::string & base, uint32_t index) { @@ -69,6 +70,8 @@ FieldMerger::FieldMerger(uint32_t id, const FusionOutputIndex& fusion_out_index, _readers(), _heap(), _writer(), + _field_length_scanner(), + _open_reader_idx(std::numeric_limits<uint32_t>::max()), _state(State::MERGE_START), _failed(false) { @@ -228,7 +231,7 @@ FieldMerger::renumber_word_ids_failed() LOG(error, "Could not renumber field word ids for field %s dir %s", _field_name.c_str(), _field_dir.c_str()); } -std::shared_ptr<FieldLengthScanner> +void FieldMerger::allocate_field_length_scanner() { SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); @@ -242,35 +245,72 @@ FieldMerger::allocate_field_length_scanner() const Schema &old_schema = old_index.getSchema(); if (index.hasOldFields(old_schema) && !index.has_matching_use_interleaved_features(old_schema)) { - return std::make_shared<FieldLengthScanner>(_fusion_out_index.get_doc_id_limit()); + _field_length_scanner = std::make_shared<FieldLengthScanner>(_fusion_out_index.get_doc_id_limit()); + return; } } } } - return std::shared_ptr<FieldLengthScanner>(); } bool +FieldMerger::open_input_field_reader() +{ + auto& oi = _fusion_out_index.get_old_indexes()[_open_reader_idx]; + if (!_readers.back()->open(oi.getPath() + "/" + _field_name + "/", _fusion_out_index.get_tune_file_indexing()._read)) { + _readers.pop_back(); + return false; + } + return true; +} + +void FieldMerger::open_input_field_readers() { - _readers.reserve(_fusion_out_index.get_old_indexes().size()); SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); - auto field_length_scanner = allocate_field_length_scanner(); - for (const auto &oi : _fusion_out_index.get_old_indexes()) { + for (; _open_reader_idx < _fusion_out_index.get_old_indexes().size(); ++_open_reader_idx) { + auto& oi = _fusion_out_index.get_old_indexes()[_open_reader_idx]; const Schema &oldSchema = oi.getSchema(); if (!index.hasOldFields(oldSchema)) { continue; // drop data } - auto reader = FieldReader::allocFieldReader(index, oldSchema, field_length_scanner); - reader->setup(_word_num_mappings[oi.getIndex()], oi.getDocIdMapping()); - if (!reader->open(oi.getPath() + "/" + _field_name + "/", _fusion_out_index.get_tune_file_indexing()._read)) { - return false; + _readers.push_back(FieldReader::allocFieldReader(index, oldSchema, _field_length_scanner)); + auto& reader = *_readers.back(); + reader.setup(_word_num_mappings[oi.getIndex()], oi.getDocIdMapping()); + if (!open_input_field_reader()) { + merge_postings_failed(); + return; + } + if (reader.need_regenerate_interleaved_features_scan()) { + _state = State::SCAN_ELEMENT_LENGTHS; + return; } - _readers.push_back(std::move(reader)); } - return true; + _field_length_scanner.reset(); + _open_reader_idx = std::numeric_limits<uint32_t>::max(); + _state = State::OPEN_POSTINGS_FIELD_READERS_FINISH; +} + +void +FieldMerger::scan_element_lengths() +{ + auto& reader = *_readers.back(); + if (reader.isValid()) { + reader.scan_element_lengths(_fusion_out_index.get_force_small_merge_chunk() ? 1u : scan_chunk); + if (reader.isValid()) { + return; + } + } + reader.close(); + if (!open_input_field_reader()) { + merge_postings_failed(); + } else { + ++_open_reader_idx; + _state = State::OPEN_POSTINGS_FIELD_READERS; + } } + bool FieldMerger::open_field_writer() { @@ -368,19 +408,25 @@ FieldMerger::setup_merge_heap() return true; } -bool +void FieldMerger::merge_postings_start() { /* OUTPUT */ _writer = std::make_unique<FieldWriter>(_fusion_out_index.get_doc_id_limit(), _num_word_ids); + _readers.reserve(_fusion_out_index.get_old_indexes().size()); + allocate_field_length_scanner(); + _open_reader_idx = 0; + _state = State::OPEN_POSTINGS_FIELD_READERS; +} - if (!open_input_field_readers()) { - return false; - } - if (!open_field_writer()) { - return false; +void +FieldMerger::merge_postings_open_field_readers_done() +{ + if (!open_field_writer() || !setup_merge_heap()) { + merge_postings_failed(); + } else { + _state = State::MERGE_POSTINGS; } - return setup_merge_heap(); } void @@ -457,7 +503,6 @@ FieldMerger::merge_field_finish() bool res = merge_postings_finish(); if (!res) { merge_postings_failed(); - _failed = true; return; } if (!FileKit::createStamp(_field_dir + "/.mergeocc_done")) { @@ -489,11 +534,19 @@ FieldMerger::process_merge_field() case State::RENUMBER_WORD_IDS_FINISH: if (!renumber_word_ids_finish()) { renumber_word_ids_failed(); - } else if (!merge_postings_start()) { - merge_postings_failed(); + break; } else { - _state = State::MERGE_POSTINGS; + merge_postings_start(); } + [[fallthrough]]; + case State::OPEN_POSTINGS_FIELD_READERS: + open_input_field_readers(); + break; + case State::SCAN_ELEMENT_LENGTHS: + scan_element_lengths(); + break; + case State::OPEN_POSTINGS_FIELD_READERS_FINISH: + merge_postings_open_field_readers_done(); break; case State::MERGE_POSTINGS: merge_postings_main(); @@ -507,13 +560,4 @@ FieldMerger::process_merge_field() } } -bool -FieldMerger::merge_field() -{ - while (!_failed && _state != State::MERGE_DONE) { - process_merge_field(); - } - return !_failed; -} - } diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.h b/searchlib/src/vespa/searchlib/diskindex/field_merger.h index 5017a7d5192..347367b3aa5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.h +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.h @@ -33,6 +33,9 @@ class FieldMerger MERGE_START, RENUMBER_WORD_IDS, RENUMBER_WORD_IDS_FINISH, + OPEN_POSTINGS_FIELD_READERS, + SCAN_ELEMENT_LENGTHS, + OPEN_POSTINGS_FIELD_READERS_FINISH, MERGE_POSTINGS, MERGE_POSTINGS_FINISH, MERGE_DONE @@ -51,9 +54,10 @@ class FieldMerger std::vector<std::unique_ptr<FieldReader>> _readers; std::unique_ptr<PostingPriorityQueueMerger<FieldReader, FieldWriter>> _heap; std::unique_ptr<FieldWriter> _writer; + std::shared_ptr<FieldLengthScanner> _field_length_scanner; + uint32_t _open_reader_idx; State _state; bool _failed; - bool _force_small_merge_chunk; void make_tmp_dirs(); bool clean_tmp_dirs(); @@ -63,12 +67,15 @@ class FieldMerger void renumber_word_ids_main(); bool renumber_word_ids_finish(); void renumber_word_ids_failed(); - std::shared_ptr<FieldLengthScanner> allocate_field_length_scanner(); - bool open_input_field_readers(); + void allocate_field_length_scanner(); + bool open_input_field_reader(); + void open_input_field_readers(); + void scan_element_lengths(); bool open_field_writer(); bool select_cooked_or_raw_features(FieldReader& reader); bool setup_merge_heap(); - bool merge_postings_start(); + void merge_postings_start(); + void merge_postings_open_field_readers_done(); void merge_postings_main(); bool merge_postings_finish(); void merge_postings_failed(); @@ -78,7 +85,9 @@ public: void merge_field_start(); void merge_field_finish(); void process_merge_field(); // Called multiple times - bool merge_field(); + uint32_t get_id() const noexcept { return _id; } + bool done() const noexcept { return _state == State::MERGE_DONE; } + bool failed() const noexcept { return _failed; } }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger_task.cpp b/searchlib/src/vespa/searchlib/diskindex/field_merger_task.cpp new file mode 100644 index 00000000000..bd73685a0a9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger_task.cpp @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "field_merger_task.h" +#include "field_merger.h" +#include "field_mergers_state.h" + +namespace search::diskindex { + +void +FieldMergerTask::run() +{ + _field_merger.process_merge_field(); + if (_field_merger.failed()) { + _field_mergers_state.field_merger_done(_field_merger, true); + } else if (_field_merger.done()) { + _field_mergers_state.field_merger_done(_field_merger, false); + } else { + _field_mergers_state.schedule_task(_field_merger); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger_task.h b/searchlib/src/vespa/searchlib/diskindex/field_merger_task.h new file mode 100644 index 00000000000..179feabe652 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger_task.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/threadexecutor.h> + +namespace search::diskindex { + +class FieldMerger; +class FieldMergersState; + +/* + * Task for processing a portion of a field merge. + */ +class FieldMergerTask : public vespalib::Executor::Task +{ + FieldMerger& _field_merger; + FieldMergersState& _field_mergers_state; + + void run() override; +public: + FieldMergerTask(FieldMerger& field_merger, FieldMergersState& field_mergers_state) + : vespalib::Executor::Task(), + _field_merger(field_merger), + _field_mergers_state(field_mergers_state) + { + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.cpp b/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.cpp new file mode 100644 index 00000000000..c924937d343 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.cpp @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "field_mergers_state.h" +#include "field_merger.h" +#include "field_merger_task.h" +#include "fusion_output_index.h" +#include <vespa/searchcommon/common/schema.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/executor.h> +#include <cassert> + +using vespalib::CpuUsage; + +namespace search::diskindex { + +FieldMergersState::FieldMergersState(const FusionOutputIndex& fusion_out_index, vespalib::Executor& executor, std::shared_ptr<IFlushToken> flush_token) + : _fusion_out_index(fusion_out_index), + _executor(executor), + _flush_token(std::move(flush_token)), + _done(_fusion_out_index.get_schema().getNumIndexFields()), + _failed(0u), + _field_mergers(_fusion_out_index.get_schema().getNumIndexFields()) +{ +} + +FieldMergersState::~FieldMergersState() +{ + wait_field_mergers_done(); +} + +FieldMerger& +FieldMergersState::alloc_field_merger(uint32_t id) +{ + assert(id < _field_mergers.size()); + auto field_merger = std::make_unique<FieldMerger>(id, _fusion_out_index, _flush_token); + auto& result = *field_merger; + assert(!_field_mergers[id]); + _field_mergers[id] = std::move(field_merger); + return result; +} + +void +FieldMergersState::destroy_field_merger(FieldMerger& field_merger) +{ + uint32_t id = field_merger.get_id(); + assert(id < _field_mergers.size()); + std::unique_ptr<FieldMerger> old_merger; + old_merger = std::move(_field_mergers[id]); + assert(old_merger.get() == &field_merger); + old_merger.reset(); + _done.countDown(); +} + +void +FieldMergersState::field_merger_done(FieldMerger& field_merger, bool failed) +{ + if (failed) { + ++_failed; + } + destroy_field_merger(field_merger); +} + +void +FieldMergersState::wait_field_mergers_done() +{ + _done.await(); +} + +void +FieldMergersState::schedule_task(FieldMerger& field_merger) +{ + auto task = std::make_unique<FieldMergerTask>(field_merger, *this); + auto rejected = _executor.execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::COMPACT)); + assert(!rejected); +} + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.h b/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.h new file mode 100644 index 00000000000..f4bad9a2b8c --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/field_mergers_state.h @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/count_down_latch.h> +#include <atomic> +#include <vector> + +namespace search { class IFlushToken; } +namespace vespalib { class Executor; } + +namespace search::diskindex { + +class FieldMerger; +class FusionOutputIndex; + +/* + * This class has ownership of active field mergers until they are + * done or failed. + */ +class FieldMergersState { + const FusionOutputIndex& _fusion_out_index; + vespalib::Executor& _executor; + std::shared_ptr<IFlushToken> _flush_token; + vespalib::CountDownLatch _done; + std::atomic<uint32_t> _failed; + std::vector<std::unique_ptr<FieldMerger>> _field_mergers; + + void destroy_field_merger(FieldMerger& field_merger); +public: + FieldMergersState(const FusionOutputIndex& fusion_out_index, vespalib::Executor& executor, std::shared_ptr<IFlushToken> flush_token); + ~FieldMergersState(); + FieldMerger& alloc_field_merger(uint32_t id); + void field_merger_done(FieldMerger& field_merger, bool failed); + void wait_field_mergers_done(); + void schedule_task(FieldMerger& field_merger); + uint32_t get_failed() const noexcept { return _failed; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index 202ac2b08ff..1794db8bba3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -99,6 +99,16 @@ FieldReader::allowRawFeatures() return true; } +bool +FieldReader::need_regenerate_interleaved_features_scan() +{ + return false; +} + +void +FieldReader::scan_element_lengths(uint32_t) +{ +} void FieldReader::setup(const WordNumMapping &wordNumMapping, @@ -277,27 +287,30 @@ FieldReaderStripInfo::open(const vespalib::string &prefix, const TuneFileSeqRead _regenerate_interleaved_features = true; } } - if (_regenerate_interleaved_features && _hasElements && _field_length_scanner) { - scan_element_lengths(); - close(); - if (!FieldReader::open(prefix, tuneFileRead)) { - return false; - } - } return true; } +bool +FieldReaderStripInfo::need_regenerate_interleaved_features_scan() +{ + return (_regenerate_interleaved_features && _hasElements && _field_length_scanner); +} + void -FieldReaderStripInfo::scan_element_lengths() +FieldReaderStripInfo::scan_element_lengths(uint32_t scan_chunk) { - for (;;) { + if (!isValid()) { + return; + } + while (scan_chunk != 0u) { FieldReader::read(); - if (_wordNum == noWordNumHigh()) { + if (!isValid()) { break; } DocIdAndFeatures &features = _docIdAndFeatures; assert(!features.has_raw_data()); _field_length_scanner->scan_features(features); + --scan_chunk; } } diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index ef319971fa7..75c8d35740e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -72,6 +72,8 @@ public: virtual void read(); virtual bool allowRawFeatures(); + virtual bool need_regenerate_interleaved_features_scan(); + virtual void scan_element_lengths(uint32_t scan_chunk); void write(FieldWriter &writer) { if (_wordNum != writer.getSparseWordNum()) { @@ -131,9 +133,10 @@ private: public: FieldReaderStripInfo(const IndexIterator &index, std::shared_ptr<FieldLengthScanner>); bool allowRawFeatures() override; + bool need_regenerate_interleaved_features_scan() override; bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) override; void read() override; - void scan_element_lengths(); + void scan_element_lengths(uint32_t scan_chunk) override; void getFeatureParams(PostingListParams ¶ms) override; }; diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index f1b5c487e40..bf62965719d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -4,9 +4,10 @@ #include "bitvectorfile.h" #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/postinglistfile.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/bitcompression/countcompression.h> + +namespace search::index { class Schema; } namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index d76d1136f8d..e1c31c1ed8d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -31,32 +31,34 @@ FileHeader::taste(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) { vespalib::FileHeader header; - FastOS_File file; + uint32_t headerLen; + uint64_t fileSize; + { + FastOS_File file; - if (tuneFileRead.getWantDirectIO()) { - file.EnableDirectIO(); - } - bool res = file.OpenReadOnly(name.c_str()); - if (!res) { - return false; - } - uint32_t headerLen = 0u; - uint64_t fileSize = file.GetSize(); - try { - headerLen = header.readFile(file); - assert(headerLen >= header.getSize()); - (void) headerLen; - } catch (vespalib::IllegalHeaderException &e) { - if (e.getMessage() != "Failed to read header info." && - e.getMessage() != "Failed to verify magic bits.") { - LOG(error, "FileHeader::tastGeneric(\"%s\") exception: %s", - name.c_str(), e.getMessage().c_str()); + if (tuneFileRead.getWantDirectIO()) { + file.EnableDirectIO(); + } + bool res = file.OpenReadOnly(name.c_str()); + if (!res) { + return false; + } + + fileSize = file.GetSize(); + try { + headerLen = header.readFile(file); + assert(headerLen >= header.getSize()); + (void) headerLen; + } catch (vespalib::IllegalHeaderException &e) { + if (e.getMessage() != "Failed to read header info." && + e.getMessage() != "Failed to verify magic bits.") { + LOG(error, "FileHeader::tastGeneric(\"%s\") exception: %s", + name.c_str(), e.getMessage().c_str()); + } + return false; } - file.Close(); - return false; } - file.Close(); _version = 1; _headerLen = headerLen; diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 12552f09027..41b4b3cb794 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -3,13 +3,13 @@ #include "fusion.h" #include "fusion_input_index.h" #include "field_merger.h" +#include "field_mergers_state.h" #include <vespa/fastos/file.h> #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/count_down_latch.h> #include <vespa/vespalib/util/error.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> @@ -71,31 +71,20 @@ Fusion::Fusion(const Schema& schema, const vespalib::string& dir, Fusion::~Fusion() = default; bool -Fusion::mergeFields(vespalib::ThreadExecutor & executor, std::shared_ptr<IFlushToken> flush_token) +Fusion::mergeFields(vespalib::Executor& shared_executor, std::shared_ptr<IFlushToken> flush_token) { + FieldMergersState field_mergers_state(_fusion_out_index, shared_executor, flush_token); const Schema &schema = getSchema(); - std::atomic<uint32_t> failed(0); - uint32_t maxConcurrentThreads = std::max(1ul, executor.getNumThreads()/2); - document::Semaphore concurrent(maxConcurrentThreads); - vespalib::CountDownLatch done(schema.getNumIndexFields()); for (SchemaUtil::IndexIterator iter(schema); iter.isValid(); ++iter) { - concurrent.wait(); - executor.execute(vespalib::makeLambdaTask([this, index=iter.getIndex(), &failed, &done, &concurrent, flush_token]() { - FieldMerger merger(index, _fusion_out_index, flush_token); - if (!merger.merge_field()) { - failed++; - } - concurrent.post(); - done.countDown(); - })); + auto& field_merger = field_mergers_state.alloc_field_merger(iter.getIndex()); + field_mergers_state.schedule_task(field_merger); } LOG(debug, "Waiting for %u fields", schema.getNumIndexFields()); - done.await(); + field_mergers_state.wait_field_mergers_done(); LOG(debug, "Done waiting for %u fields", schema.getNumIndexFields()); - return (failed == 0u); + return (field_mergers_state.get_failed() == 0u); } - bool Fusion::checkSchemaCompat() { @@ -114,7 +103,7 @@ Fusion::readSchemaFiles() } bool -Fusion::merge(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> flush_token) +Fusion::merge(vespalib::Executor& shared_executor, std::shared_ptr<IFlushToken> flush_token) { FastOS_StatInfo statInfo; if (!FastOS_File::Stat(_fusion_out_index.get_path().c_str(), &statInfo)) { @@ -148,7 +137,7 @@ Fusion::merge(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> f if (!readSchemaFiles()) { throw IllegalArgumentException("Cannot read schema files for source indexes"); } - return mergeFields(executor, flush_token); + return mergeFields(shared_executor, flush_token); } catch (const std::exception & e) { LOG(error, "%s", e.what()); return false; diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index 1f5c4471950..d78bad097df 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -3,7 +3,7 @@ #pragma once #include "fusion_output_index.h" -#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/executor.h> namespace search { class IFlushToken; @@ -25,7 +25,7 @@ class Fusion private: using Schema = index::Schema; - bool mergeFields(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> flush_token); + bool mergeFields(vespalib::Executor& shared_executor, std::shared_ptr<IFlushToken> flush_token); bool readSchemaFiles(); bool checkSchemaCompat(); @@ -43,7 +43,7 @@ public: ~Fusion(); void set_dynamic_k_pos_index_format(bool dynamic_k_pos_index_format) { _fusion_out_index.set_dynamic_k_pos_index_format(dynamic_k_pos_index_format); } void set_force_small_merge_chunk(bool force_small_merge_chunk) { _fusion_out_index.set_force_small_merge_chunk(force_small_merge_chunk); } - bool merge(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> flush_token); + bool merge(vespalib::Executor& shared_executor, std::shared_ptr<IFlushToken> flush_token); }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index a5f49c2a3fe..4462c90f4c5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.pagedict4file"); @@ -15,9 +16,9 @@ namespace { vespalib::string myPId("PageDict4P.1"); vespalib::string mySPId("PageDict4SP.1"); vespalib::string mySSId("PageDict4SS.1"); -vespalib::string emptyId; -void assertOpenWriteOnly(bool ok, const vespalib::string &fileName) +void +assertOpenWriteOnly(bool ok, const vespalib::string &fileName) { if (!ok) { int osError = errno; @@ -28,6 +29,19 @@ void assertOpenWriteOnly(bool ok, const vespalib::string &fileName) } } +int64_t +getBitSizeAndAssertHeaders(const vespalib::FileHeader & header, vespalib::stringref id) { + assert(header.hasTag("frozen")); + assert(header.hasTag("fileBitSize")); + assert(header.hasTag("format.0")); + assert(!header.hasTag("format.1")); + assert(header.hasTag("endian")); + assert(header.getTag("frozen").asInteger() != 0); + assert(header.getTag("endian").asString() == "big"); + assert(header.getTag("format.0").asString() == id); + return header.getTag("fileBitSize").asInteger(); +} + } using search::common::FileHeaderContext; @@ -36,130 +50,98 @@ using vespalib::getLastErrorString; namespace search::diskindex { -PageDict4FileSeqRead::PageDict4FileSeqRead() - : _pReader(nullptr), - _ssReader(nullptr), - _ssd(), - _ssReadContext(_ssd), - _ssfile(), - _spd(), - _spReadContext(_spd), - _spfile(), - _pd(), - _pReadContext(_pd), - _pfile(), - _ssFileBitSize(0u), - _spFileBitSize(0u), - _pFileBitSize(0u), - _ssHeaderLen(0u), - _spHeaderLen(0u), - _pHeaderLen(0u), - _ssCompleted(false), - _spCompleted(false), - _pCompleted(false), - _wordNum(0u) -{ - _ssd.setReadContext(&_ssReadContext); - _spd.setReadContext(&_spReadContext); - _pd.setReadContext(&_pReadContext); -} - - -PageDict4FileSeqRead::~PageDict4FileSeqRead() +struct PageDict4FileSeqRead::DictFileReadContext { + DictFileReadContext(vespalib::stringref id, const vespalib::string & name, const TuneFileSeqRead &tune, bool read_all_upfront); + ~DictFileReadContext(); + vespalib::FileHeader readHeader(); + void readExtendedHeader(); + bool close(); + const vespalib::string _id; + uint64_t _fileBitSize; + uint32_t _headerLen; + bool _valid; + DC _dc; + ComprFileReadContext _readContext; + FastOS_File _file; +}; + +PageDict4FileSeqRead::DictFileReadContext::DictFileReadContext(vespalib::stringref id, const vespalib::string & name, + const TuneFileSeqRead &tune, bool read_all_upfront) + : _id(id), + _fileBitSize(0u), + _headerLen(0u), + _valid(false), + _dc(), + _readContext(_dc), + _file() { - delete _pReader; - delete _ssReader; + _dc.setReadContext(&_readContext); + if (tune.getWantDirectIO()) { + _file.EnableDirectIO(); + } + if (!_file.OpenReadOnly(name.c_str())) { + LOG(error, "could not open %s: %s", _file.GetFileName(), getLastErrorString().c_str()); + return; + } + uint64_t fileSize = _file.GetSize(); + _readContext.setFile(&_file); + _readContext.setFileSize(fileSize); + if (read_all_upfront) { + _readContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / sizeof(uint64_t), 32_Ki); + } else { + _readContext.allocComprBuf(64_Ki, 32_Ki); + } + _dc.emptyBuffer(0); + _readContext.readComprBuffer(); + if (read_all_upfront) { + assert(_readContext.getBufferEndFilePos() >= fileSize); + } + _valid = true; } +PageDict4FileSeqRead::DictFileReadContext::~DictFileReadContext() = default; -void -PageDict4FileSeqRead::readSSHeader() -{ - DC &ssd = _ssd; - +vespalib::FileHeader +PageDict4FileSeqRead::DictFileReadContext::readHeader() { vespalib::FileHeader header; - uint32_t headerLen = ssd.readHeader(header, _ssfile.getSize()); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(!header.hasTag("format.1")); - assert(header.hasTag("numWordIds")); - assert(header.hasTag("avgBitsPerDoc")); - assert(header.hasTag("minChunkDocs")); - assert(header.hasTag("docIdLimit")); - assert(header.hasTag("endian")); - _ssCompleted = header.getTag("frozen").asInteger() != 0; - _ssFileBitSize = header.getTag("fileBitSize").asInteger(); - assert(header.getTag("format.0").asString() == mySSId); - ssd._numWordIds = header.getTag("numWordIds").asInteger(); - ssd._avgBitsPerDoc = header.getTag("avgBitsPerDoc").asInteger(); - ssd._minChunkDocs = header.getTag("minChunkDocs").asInteger(); - ssd._docIdLimit = header.getTag("docIdLimit").asInteger(); - - assert(header.getTag("endian").asString() == "big"); - ssd.smallAlign(64); + uint32_t headerLen = _dc.readHeader(header, _file.getSize()); + _fileBitSize = getBitSizeAndAssertHeaders(header, _id); + _dc.smallAlign(64); uint32_t minHeaderLen = header.getSize(); minHeaderLen += (-minHeaderLen & 7); assert(headerLen >= minHeaderLen); - assert(ssd.getReadOffset() == headerLen * 8); - _ssHeaderLen = headerLen; + assert(_dc.getReadOffset() == headerLen * 8); + _headerLen = headerLen; + return header; } +PageDict4FileSeqRead::PageDict4FileSeqRead() + : _pReader(), + _ssReader(), + _ss(), + _sp(), + _p(), + _wordNum(0u) +{ } -void -PageDict4FileSeqRead::readSPHeader() -{ - DC &spd = _spd; - - vespalib::FileHeader header; - uint32_t headerLen = spd.readHeader(header, _spfile.getSize()); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(!header.hasTag("format.1")); - assert(header.hasTag("endian")); - _spCompleted = header.getTag("frozen").asInteger() != 0; - _spFileBitSize = header.getTag("fileBitSize").asInteger(); - assert(header.getTag("format.0").asString() == mySPId); - assert(header.getTag("endian").asString() == "big"); - spd.smallAlign(64); - uint32_t minHeaderLen = header.getSize(); - minHeaderLen += (-minHeaderLen & 7); - assert(headerLen >= minHeaderLen); - assert(spd.getReadOffset() == headerLen * 8); - _spHeaderLen = headerLen; -} - +PageDict4FileSeqRead::~PageDict4FileSeqRead() = default; void -PageDict4FileSeqRead::readPHeader() +PageDict4FileSeqRead::DictFileReadContext::readExtendedHeader() { - DC &pd = _pd; - - vespalib::FileHeader header; - uint32_t headerLen = pd.readHeader(header, _pfile.getSize()); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(!header.hasTag("format.1")); - assert(header.hasTag("endian")); - _pCompleted = header.getTag("frozen").asInteger() != 0; - _pFileBitSize = header.getTag("fileBitSize").asInteger(); - assert(header.getTag("format.0").asString() == myPId); - assert(header.getTag("endian").asString() == "big"); - pd.smallAlign(64); - uint32_t minHeaderLen = header.getSize(); - minHeaderLen += (-minHeaderLen & 7); - assert(headerLen >= minHeaderLen); - assert(pd.getReadOffset() == headerLen * 8); - _pHeaderLen = headerLen; + vespalib::FileHeader header = readHeader(); + assert(header.hasTag("numWordIds")); + assert(header.hasTag("avgBitsPerDoc")); + assert(header.hasTag("minChunkDocs")); + assert(header.hasTag("docIdLimit")); + _dc._numWordIds = header.getTag("numWordIds").asInteger(); + _dc._avgBitsPerDoc = header.getTag("avgBitsPerDoc").asInteger(); + _dc._minChunkDocs = header.getTag("minChunkDocs").asInteger(); + _dc._docIdLimit = header.getTag("docIdLimit").asInteger(); } - void -PageDict4FileSeqRead::readWord(vespalib::string &word, - uint64_t &wordNum, - PostingListCounts &counts) +PageDict4FileSeqRead::readWord(vespalib::string &word, uint64_t &wordNum, PostingListCounts &counts) { // Map to external ids and filter by what's present in the schema. uint64_t checkWordNum = 0; @@ -173,455 +155,295 @@ PageDict4FileSeqRead::readWord(vespalib::string &word, } } +bool +PageDict4FileSeqRead::DictFileReadContext::close() { + _readContext.dropComprBuf(); + _readContext.setFile(nullptr); + return _file.Close(); +} bool PageDict4FileSeqRead::open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) { - if (tuneFileRead.getWantDirectIO()) { - _ssfile.EnableDirectIO(); - _spfile.EnableDirectIO(); - _pfile.EnableDirectIO(); - } - - vespalib::string pname = name + ".pdat"; - vespalib::string spname = name + ".spdat"; - vespalib::string ssname = name + ".ssdat"; - - if (!_ssfile.OpenReadOnly(ssname.c_str())) { - LOG(error, "could not open %s: %s", - _ssfile.GetFileName(), getLastErrorString().c_str()); - return false; - } - if (!_spfile.OpenReadOnly(spname.c_str())) { - LOG(error, "could not open %s: %s", - _spfile.GetFileName(), getLastErrorString().c_str()); - return false; - } - if (!_pfile.OpenReadOnly(pname.c_str())) { - LOG(error, "could not open %s: %s", - _pfile.GetFileName(), getLastErrorString().c_str()); + _ss = std::make_unique<DictFileReadContext>(mySSId, name + ".ssdat", tuneFileRead, true); + _sp = std::make_unique<DictFileReadContext>(mySPId, name + ".spdat", tuneFileRead, false); + _p = std::make_unique<DictFileReadContext>(myPId, name + ".pdat", tuneFileRead, false); + if ( !_ss->_valid || !_sp->_valid || !_p->_valid ) { return false; } - _spReadContext.setFile(&_spfile); - _spReadContext.setFileSize(_spfile.GetSize()); - _spReadContext.allocComprBuf(64_Ki, 32_Ki); - _spd.emptyBuffer(0); - - _pReadContext.setFile(&_pfile); - _pReadContext.setFileSize(_pfile.GetSize()); - _pReadContext.allocComprBuf(64_Ki, 32_Ki); - _pd.emptyBuffer(0); - - uint64_t fileSize = _ssfile.GetSize(); - _ssReadContext.setFile(&_ssfile); - _ssReadContext.setFileSize(fileSize); - _ssReadContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / - sizeof(uint64_t), - 32_Ki); - _ssd.emptyBuffer(0); - - _ssReadContext.readComprBuffer(); - assert(_ssReadContext.getBufferEndFilePos() >= fileSize); - readSSHeader(); - _spReadContext.readComprBuffer(); - readSPHeader(); - _pReadContext.readComprBuffer(); - readPHeader(); - - _ssReader = new SSReader(_ssReadContext, - _ssHeaderLen, - _ssFileBitSize, - _spHeaderLen, - _spFileBitSize, - _pHeaderLen, - _pFileBitSize); + _ss->readExtendedHeader(); + _sp->readHeader(); + _p->readHeader(); + + _ssReader = std::make_unique<SSReader>(_ss->_readContext, + _ss->_headerLen, _ss->_fileBitSize, + _sp->_headerLen, _sp->_fileBitSize, + _p->_headerLen, _p->_fileBitSize); // Instantiate helper class for reading - _pReader = new Reader(*_ssReader, - _spd, - _pd); + _pReader = std::make_unique<Reader>(*_ssReader, _sp->_dc, _p->_dc); - _ssReader->setup(_ssd); + _ssReader->setup(_ss->_dc); _pReader->setup(); _wordNum = 0; return true; } - bool PageDict4FileSeqRead::close() { - delete _pReader; - delete _ssReader; - _pReader = nullptr; - _ssReader = nullptr; - - _ssReadContext.dropComprBuf(); - _spReadContext.dropComprBuf(); - _pReadContext.dropComprBuf(); - _ssReadContext.setFile(nullptr); - _spReadContext.setFile(nullptr); - _pReadContext.setFile(nullptr); - _ssfile.Close(); - _spfile.Close(); - _pfile.Close(); + _pReader.reset(); + _ssReader.reset(); + if (_ss) { + return _ss->close() && _sp->close() && _p->close(); + } return true; } - void PageDict4FileSeqRead::getParams(PostingListParams ¶ms) { params.clear(); - params.set("avgBitsPerDoc", _ssd._avgBitsPerDoc); - params.set("minChunkDocs", _ssd._minChunkDocs); - params.set("docIdLimit", _ssd._docIdLimit); - params.set("numWordIds", _ssd._numWordIds); - params.set("numCounts", _ssd._numWordIds); + if (_ss) { + const DC &dc = _ss->_dc; + params.set("avgBitsPerDoc", dc._avgBitsPerDoc); + params.set("minChunkDocs", dc._minChunkDocs); + params.set("docIdLimit", dc._docIdLimit); + params.set("numWordIds", dc._numWordIds); + params.set("numCounts", dc._numWordIds); + } } +struct PageDict4FileSeqWrite::DictFileContext { + DictFileContext(bool extended, vespalib::stringref id, vespalib::stringref desc, + const vespalib::string &name, const TuneFileSeqWrite &tune); + ~DictFileContext(); + void makeHeader(const FileHeaderContext &fileHeaderContext); + bool updateHeader(uint64_t fileBitSize, uint64_t wordNum); + void writeExtendedHeader(vespalib::GenericHeader &header); + bool close(); + const vespalib::string _id; + const vespalib::string _desc; + const bool _extended; + uint32_t _headerLen; + bool _valid; + EC _ec; + ComprFileWriteContext _writeContext; + FastOS_File _file; +}; + +PageDict4FileSeqWrite::DictFileContext::DictFileContext(bool extended, vespalib::stringref id, vespalib::stringref desc, + const vespalib::string & name, const TuneFileSeqWrite &tune) + : _id(id), + _desc(desc), + _extended(extended), + _headerLen(0u), + _valid(false), + _ec(), + _writeContext(_ec), + _file() +{ + _ec.setWriteContext(&_writeContext); + if (tune.getWantSyncWrites()) { + _file.EnableSyncWrites(); + } + if (tune.getWantDirectIO()) { + _file.EnableDirectIO(); + } + bool ok = _file.OpenWriteOnly(name.c_str()); + assertOpenWriteOnly(ok, name); + _writeContext.setFile(&_file); + _writeContext.allocComprBuf(64_Ki, 32_Ki); + uint64_t fileSize = _file.GetSize(); + uint64_t bufferStartFilePos = _writeContext.getBufferStartFilePos(); + assert(fileSize >= bufferStartFilePos); + _file.SetSize(bufferStartFilePos); + assert(bufferStartFilePos == static_cast<uint64_t>(_file.GetPosition())); + + _ec.setupWrite(_writeContext); + assert(_ec.getWriteOffset() == 0); + _valid = true; +} + +bool +PageDict4FileSeqWrite::DictFileContext::DictFileContext::close() { + //uint64_t usedPBits = _ec.getWriteOffset(); + _ec.flush(); + _writeContext.writeComprBuffer(true); + + _writeContext.dropComprBuf(); + bool success = _file.Sync(); + success &= _file.Close(); + _writeContext.setFile(nullptr); + return success; +} + +PageDict4FileSeqWrite::DictFileContext::~DictFileContext() = default; PageDict4FileSeqWrite::PageDict4FileSeqWrite() - : _pWriter(), + : _params(), + _pWriter(), _spWriter(), _ssWriter(), - _pe(), - _pWriteContext(_pe), - _pfile(), - _spe(), - _spWriteContext(_spe), - _spfile(), - _sse(), - _ssWriteContext(_sse), - _ssfile(), - _pHeaderLen(0), - _spHeaderLen(0), - _ssHeaderLen(0) -{ - _pe.setWriteContext(&_pWriteContext); - _spe.setWriteContext(&_spWriteContext); - _sse.setWriteContext(&_ssWriteContext); -} - + _ss(), + _sp(), + _p() +{ } PageDict4FileSeqWrite::~PageDict4FileSeqWrite() = default; - void PageDict4FileSeqWrite::writeWord(vespalib::stringref word, const PostingListCounts &counts) { _pWriter->addCounts(word, counts); } - bool PageDict4FileSeqWrite::open(const vespalib::string &name, - const TuneFileSeqWrite &tuneFileWrite, + const TuneFileSeqWrite &tune, const FileHeaderContext &fileHeaderContext) { assert( ! _pWriter); assert( ! _spWriter); assert( ! _ssWriter); - - vespalib::string pname = name + ".pdat"; - vespalib::string spname = name + ".spdat"; - vespalib::string ssname = name + ".ssdat"; - - if (tuneFileWrite.getWantSyncWrites()) { - _pfile.EnableSyncWrites(); - _spfile.EnableSyncWrites(); - _ssfile.EnableSyncWrites(); - } - if (tuneFileWrite.getWantDirectIO()) { - _pfile.EnableDirectIO(); - _spfile.EnableDirectIO(); - _ssfile.EnableDirectIO(); - } - bool ok = _pfile.OpenWriteOnly(pname.c_str()); - assertOpenWriteOnly(ok, pname); - _pWriteContext.setFile(&_pfile); - - ok = _spfile.OpenWriteOnly(spname.c_str()); - assertOpenWriteOnly(ok, spname); - _spWriteContext.setFile(&_spfile); - - ok = _ssfile.OpenWriteOnly(ssname.c_str()); - assertOpenWriteOnly(ok, ssname); - _ssWriteContext.setFile(&_ssfile); - - _pWriteContext.allocComprBuf(64_Ki, 32_Ki); - _spWriteContext.allocComprBuf(64_Ki, 32_Ki); - _ssWriteContext.allocComprBuf(64_Ki, 32_Ki); - - uint64_t pFileSize = _pfile.GetSize(); - uint64_t spFileSize = _spfile.GetSize(); - uint64_t ssFileSize = _ssfile.GetSize(); - uint64_t pBufferStartFilePos = _pWriteContext.getBufferStartFilePos(); - uint64_t spBufferStartFilePos = _spWriteContext.getBufferStartFilePos(); - uint64_t ssBufferStartFilePos = _ssWriteContext.getBufferStartFilePos(); - assert(pFileSize >= pBufferStartFilePos); - assert(spFileSize >= spBufferStartFilePos); - assert(ssFileSize >= ssBufferStartFilePos); - (void) pFileSize; - (void) spFileSize; - (void) ssFileSize; - _pfile.SetSize(pBufferStartFilePos); - _spfile.SetSize(spBufferStartFilePos); - _ssfile.SetSize(ssBufferStartFilePos); - assert(pBufferStartFilePos == static_cast<uint64_t>(_pfile.GetPosition())); - assert(spBufferStartFilePos == - static_cast<uint64_t>(_spfile.GetPosition())); - assert(ssBufferStartFilePos == - static_cast<uint64_t>(_ssfile.GetPosition())); - - _pe.setupWrite(_pWriteContext); - _spe.setupWrite(_spWriteContext); - _sse.setupWrite(_ssWriteContext); - assert(_pe.getWriteOffset() == 0); - assert(_spe.getWriteOffset() == 0); - assert(_sse.getWriteOffset() == 0); - _spe.copyParams(_sse); - _pe.copyParams(_sse); + _ss = std::make_unique<DictFileContext>(true, mySSId, "Dictionary sparse sparse file", name + ".ssdat", tune); + _sp = std::make_unique<DictFileContext>(false, mySPId, "Dictionary sparse page file", name + ".spdat", tune); + _p = std::make_unique<DictFileContext>(false, myPId, "Dictionary page file", name + ".pdat", tune); + activateParams(_params); // Write initial file headers - makePHeader(fileHeaderContext); - makeSPHeader(fileHeaderContext); - makeSSHeader(fileHeaderContext); + _p->makeHeader(fileHeaderContext); + _sp->makeHeader(fileHeaderContext); + _ss->makeHeader(fileHeaderContext); - _ssWriter = std::make_unique<SSWriter>(_sse); - _spWriter = std::make_unique<SPWriter>(*_ssWriter, _spe); - _pWriter = std::make_unique<PWriter>(*_spWriter, _pe); + _ssWriter = std::make_unique<SSWriter>(_ss->_ec); + _spWriter = std::make_unique<SPWriter>(*_ssWriter, _sp->_ec); + _pWriter = std::make_unique<PWriter>(*_spWriter, _p->_ec); _spWriter->setup(); _pWriter->setup(); - return true; } - bool PageDict4FileSeqWrite::close() { + bool success = true; _pWriter->flush(); - uint64_t usedPBits = _pe.getWriteOffset(); - uint64_t usedSPBits = _spe.getWriteOffset(); - uint64_t usedSSBits = _sse.getWriteOffset(); - _pe.flush(); - _pWriteContext.writeComprBuffer(true); - _spe.flush(); - _spWriteContext.writeComprBuffer(true); - _sse.flush(); - _ssWriteContext.writeComprBuffer(true); - - _pWriteContext.dropComprBuf(); - _pfile.Sync(); - _pfile.Close(); - _pWriteContext.setFile(nullptr); - _spWriteContext.dropComprBuf(); - _spfile.Sync(); - _spfile.Close(); - _spWriteContext.setFile(nullptr); - _ssWriteContext.dropComprBuf(); - _ssfile.Sync(); - _ssfile.Close(); - _ssWriteContext.setFile(nullptr); + uint64_t usedPBits = _p->_ec.getWriteOffset(); + uint64_t usedSPBits = _sp->_ec.getWriteOffset(); + uint64_t usedSSBits = _ss->_ec.getWriteOffset(); + success &= _p->close(); + success &= _sp->close(); + success &= _ss->close(); + uint64_t wordNum = _pWriter->getWordNum(); // Update file headers - updatePHeader(usedPBits); - updateSPHeader(usedSPBits); - updateSSHeader(usedSSBits); + success &= _p->updateHeader(usedPBits, wordNum); + success &= _sp->updateHeader(usedSPBits, wordNum); + success &= _ss->updateHeader(usedSSBits, wordNum); _pWriter.reset(); _spWriter.reset(); _ssWriter.reset(); - return true; + return success; } - void -PageDict4FileSeqWrite::writeSSSubHeader(vespalib::GenericHeader &header) +PageDict4FileSeqWrite::DictFileContext::writeExtendedHeader(vespalib::GenericHeader &header) { - SSEC &e = _sse; typedef vespalib::GenericHeader::Tag Tag; - header.putTag(Tag("numWordIds", e._numWordIds)); - header.putTag(Tag("avgBitsPerDoc", e._avgBitsPerDoc)); - header.putTag(Tag("minChunkDocs", e._minChunkDocs)); - header.putTag(Tag("docIdLimit", e._docIdLimit)); + header.putTag(Tag("numWordIds", _ec._numWordIds)); + header.putTag(Tag("avgBitsPerDoc", _ec._avgBitsPerDoc)); + header.putTag(Tag("minChunkDocs", _ec._minChunkDocs)); + header.putTag(Tag("docIdLimit", _ec._docIdLimit)); } - void -PageDict4FileSeqWrite::makePHeader(const FileHeaderContext &fileHeaderContext) +PageDict4FileSeqWrite::DictFileContext::makeHeader(const FileHeaderContext &fileHeaderContext) { - PEC &e = _pe; - ComprFileWriteContext &wc = _pWriteContext; - - // subheader only written to SS file. - typedef vespalib::GenericHeader::Tag Tag; vespalib::FileHeader header(FileSettings::DIRECTIO_ALIGNMENT); - fileHeaderContext.addTags(header, _pfile.GetFileName()); + fileHeaderContext.addTags(header, _file.GetFileName()); header.putTag(Tag("frozen", 0)); header.putTag(Tag("fileBitSize", 0)); - header.putTag(Tag("format.0", myPId)); + header.putTag(Tag("format.0", _id)); header.putTag(Tag("endian", "big")); - header.putTag(Tag("desc", "Dictionary page file")); - e.setupWrite(wc); - e.writeHeader(header); - e.smallAlign(64); - e.flush(); - uint32_t headerLen = header.getSize(); - headerLen += (-headerLen & 7); - assert(e.getWriteOffset() == headerLen * 8); - assert((e.getWriteOffset() & 63) == 0); // Header must be word aligned - if (_pHeaderLen != 0) { - assert(_pHeaderLen == headerLen); + header.putTag(Tag("desc", _desc)); + if (_extended) { + writeExtendedHeader(header); } - _pHeaderLen = headerLen; -} - - -void -PageDict4FileSeqWrite::makeSPHeader(const FileHeaderContext &fileHeaderContext) -{ - SPEC &e = _spe; - ComprFileWriteContext &wc = _spWriteContext; - - // subheader only written to SS file. - - typedef vespalib::GenericHeader::Tag Tag; - vespalib::FileHeader header(FileSettings::DIRECTIO_ALIGNMENT); - - fileHeaderContext.addTags(header, _spfile.GetFileName()); - header.putTag(Tag("frozen", 0)); - header.putTag(Tag("fileBitSize", 0)); - header.putTag(Tag("format.0", mySPId)); - header.putTag(Tag("endian", "big")); - header.putTag(Tag("desc", "Dictionary sparse page file")); - e.setupWrite(wc); - e.writeHeader(header); - e.smallAlign(64); - e.flush(); + _ec.setupWrite(_writeContext); + _ec.writeHeader(header); + _ec.smallAlign(64); + _ec.flush(); uint32_t headerLen = header.getSize(); headerLen += (-headerLen & 7); - assert(e.getWriteOffset() == headerLen * 8); - assert((e.getWriteOffset() & 63) == 0); // Header must be word aligned - if (_spHeaderLen != 0) { - assert(_spHeaderLen == headerLen); + assert(_ec.getWriteOffset() == headerLen * 8); + assert((_ec.getWriteOffset() & 63) == 0); // Header must be word aligned + if (_headerLen != 0) { + assert(_headerLen == headerLen); } - _spHeaderLen = headerLen; + _headerLen = headerLen; } - -void -PageDict4FileSeqWrite::makeSSHeader(const FileHeaderContext &fileHeaderContext) -{ - SSEC &e = _sse; - ComprFileWriteContext &wc = _ssWriteContext; - - typedef vespalib::GenericHeader::Tag Tag; - vespalib::FileHeader header(FileSettings::DIRECTIO_ALIGNMENT); - - fileHeaderContext.addTags(header, _ssfile.GetFileName()); - header.putTag(Tag("frozen", 0)); - header.putTag(Tag("fileBitSize", 0)); - header.putTag(Tag("format.0", mySSId)); - header.putTag(Tag("endian", "big")); - header.putTag(Tag("desc", "Dictionary sparse sparse file")); - writeSSSubHeader(header); - - e.setupWrite(wc); - e.writeHeader(header); - e.smallAlign(64); - e.flush(); - uint32_t headerLen = header.getSize(); - headerLen += (-headerLen & 7); - assert(e.getWriteOffset() == headerLen * 8); - assert((e.getWriteOffset() & 63) == 0); // Header must be word aligned - if (_ssHeaderLen != 0) { - assert(_ssHeaderLen == headerLen); - } - _ssHeaderLen = headerLen; -} - - -void -PageDict4FileSeqWrite::updatePHeader(uint64_t fileBitSize) -{ - vespalib::FileHeader h(FileSettings::DIRECTIO_ALIGNMENT); - FastOS_File f; - f.OpenReadWrite(_pfile.GetFileName()); - h.readFile(f); - FileHeaderContext::setFreezeTime(h); - typedef vespalib::GenericHeader::Tag Tag; - h.putTag(Tag("frozen", 1)); - h.putTag(Tag("fileBitSize", fileBitSize)); - h.rewriteFile(f); - f.Sync(); - f.Close(); -} - - -void -PageDict4FileSeqWrite::updateSPHeader(uint64_t fileBitSize) +bool +PageDict4FileSeqWrite::DictFileContext::updateHeader(uint64_t fileBitSize, uint64_t wordNum) { vespalib::FileHeader h(FileSettings::DIRECTIO_ALIGNMENT); FastOS_File f; - f.OpenReadWrite(_spfile.GetFileName()); + f.OpenReadWrite(_file.GetFileName()); h.readFile(f); FileHeaderContext::setFreezeTime(h); typedef vespalib::GenericHeader::Tag Tag; h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", fileBitSize)); + if (_extended) { + assert(wordNum <= _ec._numWordIds); + h.putTag(Tag("numWordIds", wordNum)); + } h.rewriteFile(f); - f.Sync(); - f.Close(); + bool success = f.Sync(); + success &= f.Close(); + return success; } - void -PageDict4FileSeqWrite::updateSSHeader(uint64_t fileBitSize) -{ - vespalib::FileHeader h(FileSettings::DIRECTIO_ALIGNMENT); - FastOS_File f; - f.OpenReadWrite(_ssfile.GetFileName()); - h.readFile(f); - FileHeaderContext::setFreezeTime(h); - typedef vespalib::GenericHeader::Tag Tag; - h.putTag(Tag("frozen", 1)); - h.putTag(Tag("fileBitSize", fileBitSize)); - uint64_t wordNum = _pWriter->getWordNum(); - assert(wordNum <= _sse._numWordIds); - h.putTag(Tag("numWordIds", wordNum)); - h.rewriteFile(f); - f.Sync(); - f.Close(); +PageDict4FileSeqWrite::setParams(const PostingListParams ¶ms) { + _params.add(params); + if (_ss) { + activateParams(_params); + } } - void -PageDict4FileSeqWrite::setParams(const PostingListParams ¶ms) -{ - params.get("avgBitsPerDoc", _sse._avgBitsPerDoc); - params.get("minChunkDocs", _sse._minChunkDocs); - params.get("docIdLimit", _sse._docIdLimit); - params.get("numWordIds", _sse._numWordIds); - _spe.copyParams(_sse); - _pe.copyParams(_sse); +PageDict4FileSeqWrite::activateParams(const PostingListParams ¶ms) { + assert(_ss); + EC & ec = _ss->_ec; + params.get("avgBitsPerDoc", ec._avgBitsPerDoc); + params.get("minChunkDocs", ec._minChunkDocs); + params.get("docIdLimit", ec._docIdLimit); + params.get("numWordIds", ec._numWordIds); + _sp->_ec.copyParams(_ss->_ec); + _p->_ec.copyParams(_ss->_ec); } - void PageDict4FileSeqWrite::getParams(PostingListParams ¶ms) { params.clear(); - params.set("avgBitsPerDoc", _sse._avgBitsPerDoc); - params.set("minChunkDocs", _sse._minChunkDocs); - params.set("docIdLimit", _sse._docIdLimit); - params.set("numWordIds", _sse._numWordIds); + if (_ss) { + EC &ec = _ss->_ec; + params.set("avgBitsPerDoc", ec._avgBitsPerDoc); + params.set("minChunkDocs", ec._minChunkDocs); + params.set("docIdLimit", ec._docIdLimit); + params.set("numWordIds", ec._numWordIds); + } else { + params = _params; + } } } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index 0a68be7ae1b..1c43c20a219 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -2,10 +2,8 @@ #pragma once #include <vespa/searchlib/index/dictionaryfile.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/countcompression.h> +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/searchlib/bitcompression/pagedict4.h> -#include <vespa/fastos/file.h> namespace vespalib { class GenericHeader; } @@ -16,45 +14,19 @@ namespace search::diskindex { */ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead { - typedef bitcompression::PostingListCountFileDecodeContext DC; - typedef bitcompression::PageDict4SSReader SSReader; - typedef bitcompression::PageDict4Reader Reader; - - typedef index::PostingListCounts PostingListCounts; - - Reader *_pReader; - SSReader *_ssReader; - - DC _ssd; - ComprFileReadContext _ssReadContext; - FastOS_File _ssfile; - - DC _spd; - ComprFileReadContext _spReadContext; - FastOS_File _spfile; - - DC _pd; - ComprFileReadContext _pReadContext; - FastOS_File _pfile; - - uint64_t _ssFileBitSize; - uint64_t _spFileBitSize; - uint64_t _pFileBitSize; - uint32_t _ssHeaderLen; - uint32_t _spHeaderLen; - uint32_t _pHeaderLen; - - bool _ssCompleted; - bool _spCompleted; - bool _pCompleted; - + using DC = bitcompression::PostingListCountFileDecodeContext; + using SSReader = bitcompression::PageDict4SSReader; + using Reader = bitcompression::PageDict4Reader; + using PostingListCounts = index::PostingListCounts; + struct DictFileReadContext; + + std::unique_ptr<Reader> _pReader; + std::unique_ptr<SSReader> _ssReader; + std::unique_ptr<DictFileReadContext> _ss; + std::unique_ptr<DictFileReadContext> _sp; + std::unique_ptr<DictFileReadContext> _p; uint64_t _wordNum; - - void readSSHeader(); - void readSPHeader(); - void readPHeader(); public: - PageDict4FileSeqRead(); ~PageDict4FileSeqRead() override; @@ -73,44 +45,23 @@ public: */ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite { - typedef bitcompression::PostingListCountFileEncodeContext EC; - typedef EC SPEC; - typedef EC PEC; - typedef EC SSEC; - typedef bitcompression::PageDict4SSWriter SSWriter; - typedef bitcompression::PageDict4SPWriter SPWriter; - typedef bitcompression::PageDict4PWriter PWriter; - - typedef index::PostingListCounts PostingListCounts; + using EC = bitcompression::PostingListCountFileEncodeContext; + using SSWriter = bitcompression::PageDict4SSWriter; + using SPWriter = bitcompression::PageDict4SPWriter; + using PWriter = bitcompression::PageDict4PWriter; + using PostingListCounts = index::PostingListCounts; using FileHeaderContext = common::FileHeaderContext; + struct DictFileContext; + index::PostingListParams _params; std::unique_ptr<PWriter> _pWriter; std::unique_ptr<SPWriter> _spWriter; std::unique_ptr<SSWriter> _ssWriter; + std::unique_ptr<DictFileContext> _ss; + std::unique_ptr<DictFileContext> _sp; + std::unique_ptr<DictFileContext> _p; - EC _pe; - ComprFileWriteContext _pWriteContext; - FastOS_File _pfile; - - EC _spe; - ComprFileWriteContext _spWriteContext; - FastOS_File _spfile; - - EC _sse; - ComprFileWriteContext _ssWriteContext; - FastOS_File _ssfile; - - uint32_t _pHeaderLen; // Length of header for page file (bytes) - uint32_t _spHeaderLen; // Length of header for sparse page file (bytes) - uint32_t _ssHeaderLen; // Length of header for sparse sparse file (bytes) - - void writeSSSubHeader(vespalib::GenericHeader &header); - void makePHeader(const FileHeaderContext &fileHeaderContext); - void makeSPHeader(const FileHeaderContext &fileHeaderContext); - void makeSSHeader(const FileHeaderContext &fileHeaderContext); - void updatePHeader(uint64_t fileBitSize); - void updateSPHeader(uint64_t fileBitSize); - void updateSSHeader(uint64_t fileBitSize); + void activateParams(const index::PostingListParams ¶ms); public: PageDict4FileSeqWrite(); ~PageDict4FileSeqWrite(); @@ -121,7 +72,7 @@ public: * Open dictionary file for sequential write. The index with most * words should be first for optimal compression. */ - bool open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite, + bool open(const vespalib::string &name, const TuneFileSeqWrite &tune, const FileHeaderContext &fileHeaderContext) override; bool close() override; diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp index 30fc85d47f0..c8374610dea 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp @@ -255,10 +255,10 @@ PageDict4RandRead::close() _ssReadContext.dropComprBuf(); _ssReadContext.setFile(nullptr); - _ssfile->Close(); - _spfile->Close(); - _pfile->Close(); - return true; + bool ok = _ssfile->Close(); + ok &= _spfile->Close(); + ok &= _pfile->Close(); + return ok; } uint64_t diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp index 49e175ba102..3c74ab5ff07 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp @@ -31,9 +31,8 @@ WordNumMapping::readMappingFile(const vespalib::string &name, map.resize(tempfileentries + 2); _oldDictSize = tempfileentries; - old2newwordfile.Read(&map[1], - static_cast<size_t>(tempfilesize)); - old2newwordfile.Close(); + ssize_t has_read = old2newwordfile.Read(&map[1], static_cast<size_t>(tempfilesize)); + assert(has_read == tempfilesize); map[0] = noWordNum(); map[tempfileentries + 1] = noWordNumHigh(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp index a14c880a214..3f44b56706a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -2,6 +2,7 @@ #include "zc4_posting_writer_base.h" #include <vespa/searchlib/index/postinglistcounts.h> +#include <vespa/searchlib/index/postinglistparams.h> using search::index::PostingListCounts; using search::index::PostingListParams; @@ -225,9 +226,7 @@ Zc4PostingWriterBase::Zc4PostingWriterBase(PostingListCounts &counts) _l4Skip.maybeExpand(); } -Zc4PostingWriterBase::~Zc4PostingWriterBase() -{ -} +Zc4PostingWriterBase::~Zc4PostingWriterBase() = default; #define L1SKIPSTRIDE 16 #define L2SKIPSTRIDE 8 diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp index 593d5567266..d0b7fb42692 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp @@ -3,9 +3,7 @@ #include "zcposocc.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> -#include <vespa/searchlib/index/postinglistfile.h> -#include <vespa/searchlib/index/docidandfeatures.h> - +#include <vespa/searchlib/index/postinglistparams.h> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp index 7be66db9fec..4ed72c2f8c3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp @@ -175,8 +175,7 @@ open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) bool ZcPosOccRandRead::close() { - _file->Close(); - return true; + return _file->Close(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index f4bee9f6344..1f399971406 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/index/postinglistcountfile.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/data/fileheader.h> @@ -15,7 +16,6 @@ namespace { vespalib::string myId5("Zc.5"); vespalib::string myId4("Zc.4"); -vespalib::string emptyId; vespalib::string interleaved_features("interleaved_features"); } @@ -47,9 +47,7 @@ Zc4PostingSeqRead::Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile, boo } -Zc4PostingSeqRead::~Zc4PostingSeqRead() -{ -} +Zc4PostingSeqRead::~Zc4PostingSeqRead() = default; void Zc4PostingSeqRead::readDocIdAndFeatures(DocIdAndFeatures &features) @@ -98,9 +96,8 @@ Zc4PostingSeqRead::close() { auto &readContext = _reader.get_read_context(); readContext.dropComprBuf(); - _file.Close(); readContext.setFile(nullptr); - return true; + return _file.Close(); } @@ -203,9 +200,7 @@ Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) } -Zc4PostingSeqWrite::~Zc4PostingSeqWrite() -{ -} +Zc4PostingSeqWrite::~Zc4PostingSeqWrite() = default; void @@ -258,7 +253,7 @@ Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) } -void +bool Zc4PostingSeqWrite::updateHeader() { vespalib::FileHeader h; @@ -271,8 +266,9 @@ Zc4PostingSeqWrite::updateHeader() h.putTag(Tag("fileBitSize", _fileBitSize)); h.putTag(Tag("numWords", _writer.get_num_words())); h.rewriteFile(f); - f.Sync(); - f.Close(); + bool success = f.Sync(); + success &= f.Close(); + return success; } @@ -320,11 +316,11 @@ Zc4PostingSeqWrite::close() _writer.on_close(); // flush and pad auto &writeContext = _writer.get_write_context(); writeContext.dropComprBuf(); - _file.Sync(); - _file.Close(); + bool success = _file.Sync(); + success &= _file.Close(); writeContext.setFile(nullptr); - updateHeader(); - return true; + success &= updateHeader(); + return success; } void diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 24fccee9b8d..3d7dee1a988 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -42,8 +42,6 @@ public: bool close() override; void getParams(PostingListParams ¶ms) override; void getFeatureParams(PostingListParams ¶ms) override; - void readWordStartWithSkip(); - void readWordStart(); void readHeader(); static const vespalib::string &getIdentifier(bool dynamic_k); }; @@ -61,6 +59,11 @@ protected: FastOS_File _file; uint64_t _fileBitSize; index::PostingListCountFileSeqWrite *const _countFile; + /** + * Make header using feature encode write context. + */ + void makeHeader(const search::common::FileHeaderContext &fileHeaderContext); + bool updateHeader(); public: Zc4PostingSeqWrite(index::PostingListCountFileSeqWrite *countFile); ~Zc4PostingSeqWrite(); @@ -81,12 +84,6 @@ public: void getParams(PostingListParams ¶ms) override; void setFeatureParams(const PostingListParams ¶ms) override; void getFeatureParams(PostingListParams ¶ms) override; - - /** - * Make header using feature encode write context. - */ - void makeHeader(const search::common::FileHeaderContext &fileHeaderContext); - void updateHeader(); }; class ZcPostingSeqWrite : public Zc4PostingSeqWrite diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 26fb79f8a4e..33ed84ff4d0 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -2,6 +2,7 @@ #include "compacter.h" #include "logdatastore.h" +#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/log/log.h> @@ -11,6 +12,10 @@ namespace search::docstore { using vespalib::alloc::Alloc; +namespace { + static constexpr size_t INITIAL_BACKING_BUFFER_SIZE = 64_Mi; +} + void Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) { (void) chunkId; @@ -18,7 +23,8 @@ Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *bu _ds.write(std::move(guard), fileId, lid, buffer, sz); } -BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) : +BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, + Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) : _unSignificantBucketBits((maxSignificantBucketBits > 8) ? (maxSignificantBucketBits - 8) : 0), _sourceFileId(source), _destinationFileId(destination), @@ -28,7 +34,7 @@ BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const Compress _maxBucketGuardDuration(vespalib::duration::zero()), _lastSample(vespalib::steady_clock::now()), _lock(), - _backingMemory(Alloc::alloc(0x40000000), &_lock), + _backingMemory(Alloc::alloc(INITIAL_BACKING_BUFFER_SIZE), &_lock), _tmpStore(), _lidGuard(ds.getLidReadGuard()), _bucketizerGuard(), diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index cd4609ce33e..9c5775c0c4a 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -36,7 +36,7 @@ class BucketCompacter : public IWriteData, public StoreByBucket::IWrite public: using FileId = FileChunk::FileId; BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, - Executor & exeutor, const IBucketizer & bucketizer, FileId source, FileId destination); + Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination); void write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override ; void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override; void close() override; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 180d6aa8ced..a56360d0b53 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -10,8 +10,9 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/executor.h> +#include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/fastos/file.h> @@ -20,6 +21,7 @@ #include <vespa/log/log.h> LOG_SETUP(".search.filechunk"); +using vespalib::CpuUsage; using vespalib::GenericHeader; using vespalib::getErrorString; @@ -100,7 +102,6 @@ FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & base _diskFootprint += idxFile.GetSize(); _modificationTime = FileKit::getModificationTime(_idxFileName); } else { - dataFile.Close(); throw SummaryException("Failed opening idx file", idxFile, VESPA_STRLOC); } } @@ -304,8 +305,6 @@ FileChunk::getModificationTime() const namespace { -using FutureChunk = std::future<Chunk::UP>; - struct FixedParams { const IGetLid & db; IWriteData & dest; @@ -337,29 +336,38 @@ appendChunks(FixedParams * args, Chunk::UP chunk) } void -FileChunk::appendTo(vespalib::ThreadExecutor & executor, const IGetLid & db, IWriteData & dest, - uint32_t numChunks, IFileChunkVisitorProgress *visitorProgress) +FileChunk::appendTo(vespalib::Executor & executor, const IGetLid & db, IWriteData & dest, + uint32_t numChunks, IFileChunkVisitorProgress *visitorProgress, + vespalib::CpuUsage::Category cpu_category) { assert(frozen() || visitorProgress); vespalib::GenerationHandler::Guard lidReadGuard(db.getLidReadGuard()); assert(numChunks <= getNumChunks()); FixedParams fixedParams = {db, dest, lidReadGuard, getFileId().getId(), visitorProgress}; - vespalib::BlockingThreadStackExecutor singleExecutor(1, 64_Ki, executor.getNumThreads()*2); + size_t limit = std::thread::hardware_concurrency(); + vespalib::ArrayQueue<std::future<Chunk::UP>> queue; for (size_t chunkId(0); chunkId < numChunks; chunkId++) { std::promise<Chunk::UP> promisedChunk; std::future<Chunk::UP> futureChunk = promisedChunk.get_future(); - executor.execute(vespalib::makeLambdaTask([promise = std::move(promisedChunk), chunkId, this]() mutable { + auto task = vespalib::makeLambdaTask([promise = std::move(promisedChunk), chunkId, this]() mutable { const ChunkInfo & cInfo(_chunkInfo[chunkId]); vespalib::DataBuffer whole(0ul, ALIGNMENT); FileRandRead::FSP keepAlive(_file->read(cInfo.getOffset(), whole, cInfo.getSize())); promise.set_value(std::make_unique<Chunk>(chunkId, whole.getData(), whole.getDataLen())); - })); + }); + executor.execute(CpuUsage::wrap(std::move(task), cpu_category)); + + while (queue.size() >= limit) { + appendChunks(&fixedParams, queue.front().get()); + queue.pop(); + } - singleExecutor.execute(vespalib::makeLambdaTask([args = &fixedParams, chunk = std::move(futureChunk)]() mutable { - appendChunks(args, chunk.get()); - })); + queue.push(std::move(futureChunk)); + } + while ( ! queue.empty() ) { + appendChunks(&fixedParams, queue.front().get()); + queue.pop(); } - singleExecutor.sync(); dest.close(); } diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index eee69602f8b..60c788cf9c7 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -7,10 +7,11 @@ #include "lid_info.h" #include "randread.h" #include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/vespalib/util/memoryusage.h> -#include <vespa/vespalib/util/ptrholder.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/memoryusage.h> +#include <vespa/vespalib/util/ptrholder.h> #include <vespa/vespalib/util/time.h> class FastOS_FileInterface; @@ -18,7 +19,7 @@ class FastOS_FileInterface; namespace vespalib { class DataBuffer; class GenericHeader; - class ThreadExecutor; + class Executor; } namespace search { @@ -163,8 +164,9 @@ public: virtual bool frozen() const { return true; } const vespalib::string & getName() const { return _name; } void compact(const IGetLid & iGetLid); - void appendTo(vespalib::ThreadExecutor & executor, const IGetLid & db, IWriteData & dest, - uint32_t numChunks, IFileChunkVisitorProgress *visitorProgress); + void appendTo(vespalib::Executor & executor, const IGetLid & db, IWriteData & dest, + uint32_t numChunks, IFileChunkVisitorProgress *visitorProgress, + vespalib::CpuUsage::Category cpu_category); /** * Must be called after chunk has been created to allow correct * underlying file object to be created. Must be called before diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 6a9ae40cc93..2d62e1db6fd 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -3,10 +3,11 @@ #include "storebybucket.h" #include "compacter.h" #include "logdatastore.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/vespalib/util/size_literals.h> @@ -22,17 +23,20 @@ namespace { constexpr uint32_t DEFAULT_MAX_LIDS_PER_FILE = 32_Mi; } -using vespalib::getLastErrorString; -using vespalib::getErrorString; -using vespalib::GenerationHandler; -using vespalib::make_string; -using vespalib::IllegalStateException; using common::FileHeaderContext; -using std::runtime_error; -using document::BucketId; -using docstore::StoreByBucket; using docstore::BucketCompacter; +using docstore::StoreByBucket; +using document::BucketId; using namespace std::literals; +using std::runtime_error; +using vespalib::CpuUsage; +using vespalib::GenerationHandler; +using vespalib::IllegalStateException; +using vespalib::getErrorString; +using vespalib::getLastErrorString; +using vespalib::make_string; + +using CpuCategory = CpuUsage::Category; LogDataStore::Config::Config() : _maxFileSize(DEFAULT_MAX_FILESIZE), @@ -54,7 +58,7 @@ LogDataStore::Config::operator == (const Config & rhs) const { (_fileConfig == rhs._fileConfig); } -LogDataStore::LogDataStore(vespalib::ThreadExecutor &executor, const vespalib::string &dirName, const Config &config, +LogDataStore::LogDataStore(vespalib::Executor &executor, const vespalib::string &dirName, const Config &config, const GrowStrategy &growStrategy, const TuneFileSummary &tune, const FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, IBucketizer::SP bucketizer, bool readOnly) @@ -180,29 +184,30 @@ LogDataStore::write(uint64_t serialNum, uint32_t lid, const void * buffer, size_ { std::unique_lock guard(_updateLock); WriteableFileChunk & active = getActive(guard); - write(std::move(guard), active, serialNum, lid, buffer, len); + write(std::move(guard), active, serialNum, lid, buffer, len, CpuCategory::WRITE); } void LogDataStore::write(MonitorGuard guard, FileId destinationFileId, uint32_t lid, const void * buffer, size_t len) { auto & destination = static_cast<WriteableFileChunk &>(*_fileChunks[destinationFileId.getId()]); - write(std::move(guard), destination, destination.getSerialNum(), lid, buffer, len); + write(std::move(guard), destination, destination.getSerialNum(), lid, buffer, len, CpuCategory::COMPACT); } void LogDataStore::write(MonitorGuard guard, WriteableFileChunk & destination, - uint64_t serialNum, uint32_t lid, const void * buffer, size_t len) + uint64_t serialNum, uint32_t lid, const void * buffer, size_t len, + CpuUsage::Category cpu_category) { - LidInfo lm = destination.append(serialNum, lid, buffer, len); + LidInfo lm = destination.append(serialNum, lid, buffer, len, cpu_category); setLid(guard, lid, lm); if (destination.getFileId() == getActiveFileId(guard)) { - requireSpace(std::move(guard), destination); + requireSpace(std::move(guard), destination, cpu_category); } } void -LogDataStore::requireSpace(MonitorGuard guard, WriteableFileChunk & active) +LogDataStore::requireSpace(MonitorGuard guard, WriteableFileChunk & active, CpuUsage::Category cpu_category) { assert(active.getFileId() == getActiveFileId(guard)); size_t oldSz(active.getDiskFootprint()); @@ -216,13 +221,13 @@ LogDataStore::requireSpace(MonitorGuard guard, WriteableFileChunk & active) guard.unlock(); // Write chunks to old .dat file // Note: Feed latency spike - active.flush(true, active.getSerialNum()); + active.flush(true, active.getSerialNum(), cpu_category); // Sync transaction log _tlSyncer.sync(active.getSerialNum()); // sync old active .dat file, write pending chunks to old .idx file // and sync old .idx file to disk. active.flushPendingChunks(active.getSerialNum()); - active.freeze(); + active.freeze(cpu_category); // TODO: Delay create of new file LOG(debug, "Closed file %s of size %ld and %u lids due to maxsize of %ld or maxlids %u reached. Bloat is %ld", active.getName().c_str(), active.getDiskFootprint(), active.getNumLids(), @@ -278,7 +283,7 @@ LogDataStore::remove(uint64_t serialNum, uint32_t lid) if (lm.valid()) { _fileChunks[lm.getFileId()]->remove(lid, lm.size()); } - lm = getActive(guard).append(serialNum, lid, nullptr, 0); + lm = getActive(guard).append(serialNum, lid, nullptr, 0, CpuCategory::WRITE); assert( lm.empty() ); _lidInfo[lid] = lm; } @@ -311,7 +316,9 @@ LogDataStore::flush(uint64_t syncToken) { MonitorGuard guard(_updateLock); // Note: Feed latency spike - getActive(guard).flush(true, syncToken); + // This is executed by an IFlushTarget, + // but is a fundamental part of the WRITE pipeline of the data store. + getActive(guard).flush(true, syncToken, CpuCategory::WRITE); active = &getActive(guard); activeHolder = holdFileChunk(active->getFileId()); } @@ -330,21 +337,6 @@ LogDataStore::initFlush(uint64_t syncToken) return syncToken; } -double -LogDataStore::getMaxBucketSpread() const -{ - double maxSpread(1.0); - MonitorGuard guard(_updateLock); - for (const auto & fc : _fileChunks) { - if (fc) { - if (_bucketizer && fc->frozen()) { - maxSpread = std::max(maxSpread, fc->getBucketSpread()); - } - } - } - return maxSpread; -} - std::pair<bool, LogDataStore::FileId> LogDataStore::findNextToCompact(bool dueToBloat) { @@ -403,18 +395,21 @@ LogDataStore::compactWorst(uint64_t syncToken, bool compactDiskBloat) { } } -SerialNum LogDataStore::flushFile(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken) { +SerialNum LogDataStore::flushFile(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken, + CpuUsage::Category cpu_category) +{ (void) guard; uint64_t lastSerial(file.getSerialNum()); if (lastSerial > syncToken) { syncToken = lastSerial; } - file.flush(false, syncToken); + file.flush(false, syncToken, cpu_category); return syncToken; } void LogDataStore::flushFileAndWait(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken) { - syncToken = flushFile(std::move(guard), file, syncToken); + // This function is always called in the context of compaction. + syncToken = flushFile(std::move(guard), file, syncToken, CpuCategory::COMPACT); file.waitForDiskToCatchUpToNow(); _tlSyncer.sync(syncToken); file.flushPendingChunks(syncToken); @@ -423,7 +418,9 @@ void LogDataStore::flushFileAndWait(MonitorGuard guard, WriteableFileChunk & fil SerialNum LogDataStore::flushActive(SerialNum syncToken) { MonitorGuard guard(_updateLock); WriteableFileChunk &active = getActive(guard); - return flushFile(std::move(guard), active, syncToken); + // This is executed by an IFlushTarget (via initFlush), + // but is a fundamental part of the WRITE pipeline of the data store. + return flushFile(std::move(guard), active, syncToken, CpuCategory::WRITE); } void LogDataStore::flushActiveAndWait(SerialNum syncToken) { @@ -465,7 +462,7 @@ void LogDataStore::compactFile(FileId fileId) compacter = std::make_unique<docstore::Compacter>(*this); } - fc->appendTo(_executor, *this, *compacter, fc->getNumChunks(), nullptr); + fc->appendTo(_executor, *this, *compacter, fc->getNumChunks(), nullptr, CpuCategory::COMPACT); if (destinationFileId.isActive()) { flushActiveAndWait(0); @@ -473,7 +470,7 @@ void LogDataStore::compactFile(FileId fileId) MonitorGuard guard(_updateLock); auto & compactTo = dynamic_cast<WriteableFileChunk &>(*_fileChunks[destinationFileId.getId()]); flushFileAndWait(std::move(guard), compactTo, 0); - compactTo.freeze(); + compactTo.freeze(CpuCategory::COMPACT); } compacter.reset(); @@ -579,6 +576,22 @@ LogDataStore::getDiskHeaderFootprint() const return sz; } +double +LogDataStore::getMaxBucketSpread() const +{ + double maxSpread(1.0); + MonitorGuard guard(_updateLock); + for (FileId i(0); i < FileId(_fileChunks.size()); i = i.next()) { + /// Ignore the the active file as it is never considered for reordering until completed and frozen. + if (i != _active) { + const auto & fc = _fileChunks[i.getId()]; + if (fc && _bucketizer && fc->frozen()) { + maxSpread = std::max(maxSpread, fc->getBucketSpread()); + } + } + } + return maxSpread; +} size_t LogDataStore::getDiskBloat() const @@ -586,7 +599,8 @@ LogDataStore::getDiskBloat() const MonitorGuard guard(_updateLock); size_t sz(0); for (FileId i(0); i < FileId(_fileChunks.size()); i = i.next()) { - /// Do not count the holes in the last file as bloat + /// Do not count the holes in the last file as bloat as it is + /// never considered for compaction until completed and frozen. if (i != _active) { const auto & chunk = _fileChunks[i.getId()]; if (chunk) { @@ -1069,7 +1083,9 @@ LogDataStore::accept(IDataStoreVisitor &visitor, WrapVisitorProgress wrapProgress(visitorProgress, totalChunks); for (FileId fcId : fileChunks) { FileChunk & fc = *_fileChunks[fcId.getId()]; - fc.appendTo(_executor, *this, wrap, fc.getNumChunks(), &wrapProgress); + // accept() is used when reprocessing all documents stored (e.g. when adding attribute to a field). + // We tag this work as WRITE, as the alternative to reprocessing would be to re-feed the data. + fc.appendTo(_executor, *this, wrap, fc.getNumChunks(), &wrapProgress, CpuCategory::WRITE); if (prune) { internalFlushAll(); FileChunk::UP toDie; @@ -1080,7 +1096,7 @@ LogDataStore::accept(IDataStoreVisitor &visitor, toDie->erase(); } } - lfc.appendTo(_executor, *this, wrap, lastChunks, &wrapProgress); + lfc.appendTo(_executor, *this, wrap, lastChunks, &wrapProgress, CpuCategory::WRITE); if (prune) { internalFlushAll(); } diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 62f87076759..fc985c57ce4 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -5,12 +5,13 @@ #include "idatastore.h" #include "lid_info.h" #include "writeablefilechunk.h" -#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/transactionlog/syncproxy.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/rcuvector.h> -#include <vespa/vespalib/util/threadexecutor.h> #include <set> @@ -84,7 +85,7 @@ public: * The caller must keep it alive for the semantic * lifetime of the log data store. */ - LogDataStore(vespalib::ThreadExecutor &executor, const vespalib::string &dirName, const Config & config, + LogDataStore(vespalib::Executor &executor, const vespalib::string &dirName, const Config & config, const GrowStrategy &growStrategy, const TuneFileSummary &tune, const search::common::FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, IBucketizer::SP bucketizer, bool readOnly = false); @@ -114,7 +115,8 @@ public: const Config & getConfig() const { return _config; } Config & getConfig() { return _config; } - void write(MonitorGuard guard, WriteableFileChunk & destination, uint64_t serialNum, uint32_t lid, const void * buffer, size_t len); + void write(MonitorGuard guard, WriteableFileChunk & destination, uint64_t serialNum, uint32_t lid, + const void * buffer, size_t len, vespalib::CpuUsage::Category cpu_category); void write(MonitorGuard guard, FileId destinationFileId, uint32_t lid, const void * buffer, size_t len); /** @@ -215,7 +217,7 @@ private: vespalib::string createDatFileName(NameId id) const; vespalib::string createIdxFileName(NameId id) const; - void requireSpace(MonitorGuard guard, WriteableFileChunk & active); + void requireSpace(MonitorGuard guard, WriteableFileChunk & active, vespalib::CpuUsage::Category cpu_category); bool isReadOnly() const { return _readOnly; } void updateSerialNum(); @@ -232,7 +234,8 @@ private: */ void unholdFileChunk(FileId fileId); - SerialNum flushFile(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken); + SerialNum flushFile(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken, + vespalib::CpuUsage::Category cpu_category); SerialNum flushActive(SerialNum syncToken); void flushActiveAndWait(SerialNum syncToken); void flushFileAndWait(MonitorGuard guard, WriteableFileChunk & file, SerialNum syncToken); @@ -256,7 +259,7 @@ private: FileId _prevActive; mutable std::mutex _updateLock; bool _readOnly; - vespalib::ThreadExecutor &_executor; + vespalib::Executor &_executor; SerialNum _initFlushSyncToken; transactionlog::SyncProxy &_tlSyncer; IBucketizer::SP _bucketizer; diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp index a9bc8559f3c..d18726dbecb 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp @@ -13,7 +13,7 @@ LogDocumentStore::Config::operator == (const Config & rhs) const { return DocumentStore::Config::operator ==(rhs) && (_logConfig == rhs._logConfig); } -LogDocumentStore::LogDocumentStore(vespalib::ThreadExecutor & executor, +LogDocumentStore::LogDocumentStore(vespalib::Executor & executor, const vespalib::string & baseDir, const Config & config, const GrowStrategy & growStrategy, diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h index 2931f8bce2d..2b7d7365c1e 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h @@ -44,7 +44,7 @@ public: * The caller must keep it alive for the semantic * lifetime of the log data store. */ - LogDocumentStore(vespalib::ThreadExecutor & executor, const vespalib::string & baseDir, const Config & config, + LogDocumentStore(vespalib::Executor & executor, const vespalib::string & baseDir, const Config & config, const GrowStrategy & growStrategy, const TuneFileSummary &tuneFileSummary, const common::FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, IBucketizer::SP bucketizer); diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index 37732fe46cb..651ff111f4e 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -1,14 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "storebybucket.h" -#include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/lambdatask.h> #include <algorithm> namespace search::docstore { using document::BucketId; +using vespalib::CpuUsage; using vespalib::makeLambdaTask; StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) noexcept @@ -35,9 +37,10 @@ StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void Chunk::UP tmpChunk = createChunk(); _current.swap(tmpChunk); incChunksPosted(); - _executor.execute(makeLambdaTask([this, chunk=std::move(tmpChunk)]() mutable { + auto task = makeLambdaTask([this, chunk=std::move(tmpChunk)]() mutable { closeChunk(std::move(chunk)); - })); + }); + _executor.execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::COMPACT)); } Index idx(bucketId, _current->getId(), chunkId, lid); _current->append(lid, buffer, sz); @@ -88,9 +91,10 @@ void StoreByBucket::drain(IWrite & drainer) { incChunksPosted(); - _executor.execute(makeLambdaTask([this, chunk=std::move(_current)]() mutable { + auto task = makeLambdaTask([this, chunk=std::move(_current)]() mutable { closeChunk(std::move(chunk)); - })); + }); + _executor.execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::COMPACT)); waitAllProcessed(); std::vector<Chunk::UP> chunks; chunks.resize(_chunks.size()); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 8c76a4477f5..78b247ffd46 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -10,19 +10,21 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> LOG_SETUP(".search.writeablefilechunk"); -using vespalib::makeLambdaTask; +using search::common::FileHeaderContext; +using vespalib::CpuUsage; using vespalib::FileHeader; +using vespalib::GenerationHandler; +using vespalib::IllegalHeaderException; +using vespalib::makeLambdaTask; using vespalib::make_string; using vespalib::nbostream; -using vespalib::IllegalHeaderException; -using vespalib::GenerationHandler; -using search::common::FileHeaderContext; namespace search { @@ -135,7 +137,9 @@ WriteableFileChunk(vespalib::Executor &executor, _idxHeaderLen = writeIdxHeader(fileHeaderContext, _docIdLimit, *idxFile); } _idxFileSize = idxFile->GetSize(); - idxFile->Sync(); + if ( ! idxFile->Sync()) { + throw SummaryException("Failed syncing idx file", *idxFile, VESPA_STRLOC); + } } else { throw SummaryException("Failed opening data file", _dataFile, VESPA_STRLOC); } @@ -159,9 +163,9 @@ WriteableFileChunk::~WriteableFileChunk() { if (!frozen()) { if (_active->size() || _active->count()) { - flush(true, _serialNum); + flush(true, _serialNum, CpuUsage::Category::WRITE); } - freeze(); + freeze(CpuUsage::Category::WRITE); } // This is a wild stab at fixing bug 6348143. // If it works it indicates something bad with the filesystem. @@ -186,9 +190,10 @@ WriteableFileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t } void -WriteableFileChunk::restart(uint32_t nextChunkId) +WriteableFileChunk::restart(uint32_t nextChunkId, CpuUsage::Category cpu_category) { - _executor.execute(makeLambdaTask([this, nextChunkId] {fileWriter(nextChunkId);})); + auto task = makeLambdaTask([this, nextChunkId] {fileWriter(nextChunkId);}); + _executor.execute(CpuUsage::wrap(std::move(task), cpu_category)); } namespace { @@ -280,7 +285,7 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer } void -WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) +WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum, CpuUsage::Category cpu_category) { Chunk * active(nullptr); { @@ -303,11 +308,11 @@ WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) std::lock_guard innerGuard(_lock); setDiskFootprint(FileChunk::getDiskFootprint() + tmp->getBuf().getDataLen()); } - enque(std::move(tmp)); + enque(std::move(tmp), cpu_category); } void -WriteableFileChunk::enque(ProcessedChunkUP tmp) +WriteableFileChunk::enque(ProcessedChunkUP tmp, CpuUsage::Category cpu_category) { LOG(debug, "enqueing %p", tmp.get()); std::unique_lock guard(_writeMonitor); @@ -317,7 +322,7 @@ WriteableFileChunk::enque(ProcessedChunkUP tmp) uint32_t nextChunkId = _firstChunkIdToBeWritten; guard.unlock(); _writeCond.notify_one(); - restart(nextChunkId); + restart(nextChunkId, cpu_category); } else { _writeCond.notify_one(); } @@ -339,9 +344,14 @@ getAlignedStartPos(FastOS_File & file) ssize_t toWrite(Alignment - (startPos & (Alignment-1))); ssize_t written = align.Write2(&Padding[0], toWrite); if (written == toWrite) { - align.Sync(); - file.SetPosition(align.GetSize()); - startPos = file.GetPosition(); + if ( align.Sync() ) { + file.SetPosition(align.GetSize()); + startPos = file.GetPosition(); + } else { + throw SummaryException( + make_string("Failed syncing dat file."), + align, VESPA_STRLOC); + } } else { throw SummaryException( make_string("Failed writing %ld bytes to dat file. Only %ld written", toWrite, written), @@ -556,11 +566,11 @@ WriteableFileChunk::getModificationTime() const } void -WriteableFileChunk::freeze() +WriteableFileChunk::freeze(CpuUsage::Category cpu_category) { if (!frozen()) { waitForAllChunksFlushedToDisk(); - enque(ProcessedChunkUP()); + enque(ProcessedChunkUP(), cpu_category); { std::unique_lock guard(_writeMonitor); while (_writeTaskIsRunning) { @@ -574,7 +584,8 @@ WriteableFileChunk::freeze() setDiskFootprint(getDiskFootprint(guard)); _frozen = true; } - _dataFile.Close(); + bool sync_and_close_ok = _dataFile.Sync() && _dataFile.Close(); + assert(sync_and_close_ok); _bucketMap = BucketDensityComputer(_bucketizer); } } @@ -657,12 +668,15 @@ int32_t WriteableFileChunk::flushLastIfNonEmpty(bool force) } void -WriteableFileChunk::flush(bool block, uint64_t syncToken) +WriteableFileChunk::flush(bool block, uint64_t syncToken, CpuUsage::Category cpu_category) { int32_t chunkId = flushLastIfNonEmpty(syncToken > _serialNum); if (chunkId >= 0) { setSerialNum(syncToken); - _executor.execute(makeLambdaTask([this, chunkId, serialNum=_serialNum] { internalFlush(chunkId, serialNum); })); + auto task = makeLambdaTask([this, chunkId, serialNum=_serialNum, cpu_category] { + internalFlush(chunkId, serialNum, cpu_category); + }); + _executor.execute(CpuUsage::wrap(std::move(task), cpu_category)); } else { if (block) { std::lock_guard guard(_lock); @@ -708,11 +722,12 @@ WriteableFileChunk::waitForAllChunksFlushedToDisk() const } LidInfo -WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len) +WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len, + CpuUsage::Category cpu_category) { assert( !frozen() ); if ( ! _active->hasRoom(len)) { - flush(false, _serialNum); + flush(false, _serialNum, cpu_category); } assert(serialNum >= _serialNum); _serialNum = serialNum; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index f1b69a5f1f9..846a9a9035e 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -53,13 +53,14 @@ public: ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const override; void read(LidInfoWithLidV::const_iterator begin, size_t count, IBufferVisitor & visitor) const override; - LidInfo append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len); - void flush(bool block, uint64_t syncToken); + LidInfo append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len, + vespalib::CpuUsage::Category cpu_category); + void flush(bool block, uint64_t syncToken, vespalib::CpuUsage::Category cpu_category); uint64_t getSerialNum() const { return _serialNum; } void setSerialNum(uint64_t serialNum) { _serialNum = std::max(_serialNum, serialNum); } vespalib::system_time getModificationTime() const override; - void freeze(); + void freeze(vespalib::CpuUsage::Category cpu_category); size_t getDiskFootprint() const override; size_t getMemoryFootprint() const override; size_t getMemoryMetaFootprint() const override; @@ -80,11 +81,11 @@ private: void waitForChunkFlushedToDisk(uint32_t chunkId) const; void waitForAllChunksFlushedToDisk() const; void fileWriter(const uint32_t firstChunkId); - void internalFlush(uint32_t, uint64_t serialNum); - void enque(ProcessedChunkUP); + void internalFlush(uint32_t, uint64_t serialNum, vespalib::CpuUsage::Category cpu_category); + void enque(ProcessedChunkUP, vespalib::CpuUsage::Category cpu_category); int32_t flushLastIfNonEmpty(bool force); // _writeMonitor should not be held when calling restart - void restart(uint32_t nextChunkId); + void restart(uint32_t nextChunkId, vespalib::CpuUsage::Category cpu_category); ProcessedChunkQ drainQ(unique_lock & guard); void readDataHeader(); void readIdxHeader(FastOS_FileInterface & idxFile); diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 9d4119a7faa..88531a46cb1 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(searchlib_features OBJECT debug_wait.cpp dense_tensor_attribute_executor.cpp direct_tensor_attribute_executor.cpp + great_circle_distance_feature.cpp distancefeature.cpp distancetopathfeature.cpp documenttestutils.cpp diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index d5a27d04216..7429f32cf4f 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -135,10 +135,15 @@ DistanceExecutor::DistanceExecutor(GeoLocationSpecPtrs locations, void DistanceExecutor::execute(uint32_t docId) { - outputs().set_number(0, calculateDistance(docId)); + static constexpr double earth_mean_radius = 6371.0088; + static constexpr double deg_to_rad = M_PI / 180.0; + static constexpr double km_from_internal = 1.0e-6 * deg_to_rad * earth_mean_radius; + feature_t internal_d = calculateDistance(docId); + outputs().set_number(0, internal_d); outputs().set_number(1, _best_index); outputs().set_number(2, _best_y * 1.0e-6); // latitude outputs().set_number(3, _best_x * 1.0e-6); // longitude + outputs().set_number(4, internal_d * km_from_internal); // km } const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0); @@ -146,6 +151,7 @@ const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0); DistanceBlueprint::DistanceBlueprint() : Blueprint("distance"), + _field_name(), _arg_string(), _attr_id(search::index::Schema::UNKNOWN_FIELD_ID), _use_geo_pos(false), @@ -178,6 +184,7 @@ DistanceBlueprint::setup_geopos(const IIndexEnvironment & env, describeOutput("index", "Index in array of closest point"); describeOutput("latitude", "Latitude of closest point"); describeOutput("longitude", "Longitude of closest point"); + describeOutput("km", "Distance in kilometer units"); env.hintAttributeAccess(_arg_string); return true; } @@ -202,7 +209,7 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, bool allow_bad_field = true; if (params.size() == 2) { // params[0] = field / label - // params[0] = attribute name / label value + // params[1] = attribute name / label value if (arg == "label") { _arg_string = params[1].getValue(); _use_item_label = true; @@ -212,12 +219,18 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, arg = params[1].getValue(); allow_bad_field = false; } else { - LOG(error, "first argument must be 'field' or 'label', but was '%s'", - arg.c_str()); + LOG(error, "first argument must be 'field' or 'label', but was '%s'", arg.c_str()); return false; } } - const FieldInfo *fi = env.getFieldByName(arg); + _field_name = arg; + vespalib::string z = document::PositionDataType::getZCurveFieldName(arg); + const FieldInfo *fi = env.getFieldByName(z); + if (fi != nullptr && fi->hasAttribute()) { + // can't check anything here because streaming has wrong information + return setup_geopos(env, z); + } + fi = env.getFieldByName(arg); if (fi != nullptr && fi->hasAttribute()) { auto dt = fi->get_data_type(); auto ct = fi->collection(); @@ -225,21 +238,17 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, _attr_id = fi->id(); return setup_nns(env, arg); } - // could check if dt is DataType::INT64 // could check if ct is CollectionType::SINGLE or CollectionType::ARRAY) - return setup_geopos(env, arg); - } - vespalib::string z = document::PositionDataType::getZCurveFieldName(arg); - fi = env.getFieldByName(z); - if (fi != nullptr && fi->hasAttribute()) { - return setup_geopos(env, z); + if (dt == DataType::INT64) { + return setup_geopos(env, arg); + } } if (allow_bad_field) { // TODO remove on Vespa 8 // backwards compatibility fallback: return setup_geopos(env, arg); } - if (env.getFieldByName(arg) == nullptr && fi == nullptr) { + if (env.getFieldByName(arg) == nullptr) { LOG(error, "unknown field '%s' for rank feature %s\n", arg.c_str(), getName().c_str()); } else { LOG(error, "field '%s' must be an attribute for rank feature %s\n", arg.c_str(), getName().c_str()); @@ -263,7 +272,9 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash for (auto loc_ptr : env.getAllLocations()) { if (_use_geo_pos && loc_ptr && loc_ptr->location.valid()) { - if (loc_ptr->field_name == _arg_string) { + if (loc_ptr->field_name == _arg_string || + loc_ptr->field_name == _field_name) + { LOG(debug, "found loc from query env matching '%s'", _arg_string.c_str()); matching_locs.push_back(loc_ptr); } else { diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.h b/searchlib/src/vespa/searchlib/features/distancefeature.h index b60072cf872..6eff0380c3a 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.h +++ b/searchlib/src/vespa/searchlib/features/distancefeature.h @@ -44,6 +44,7 @@ public: */ class DistanceBlueprint : public fef::Blueprint { private: + vespalib::string _field_name; vespalib::string _arg_string; uint32_t _attr_id; bool _use_geo_pos; diff --git a/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp new file mode 100644 index 00000000000..39a82ca8237 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp @@ -0,0 +1,202 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "great_circle_distance_feature.h" +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/document/datatype/positiondatatype.h> +#include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/util/stash.h> +#include <cmath> +#include <limits> +#include "utils.h" + +#include <vespa/log/log.h> +LOG_SETUP(".features.great_circle_distance_feature"); + +using namespace search::fef; +using namespace search::index::schema; +using vespalib::Issue; + +namespace search::features { + +feature_t GCDExecutor::calculateGCD(uint32_t docId) { + feature_t dist = std::numeric_limits<feature_t>::max(); + _best_lat = 90.0; + _best_lng = -180.0; + if (_locations.empty()) { + return dist; + } + _intBuf.fill(*_pos, docId); + uint32_t numValues = _intBuf.size(); + int32_t docx = 0; + int32_t docy = 0; + for (auto loc : _locations) { + for (uint32_t i = 0; i < numValues; ++i) { + vespalib::geo::ZCurve::decode(_intBuf[i], &docx, &docy); + double lat = docy / 1.0e6; + double lng = docx / 1.0e6; + double d = loc.km_great_circle_distance(lat, lng); + if (d < dist) { + dist = d; + _best_lat = lat; + _best_lng = lng; + } + } + } + return dist; +} + +GCDExecutor::GCDExecutor(GeoLocationSpecPtrs locations, const attribute::IAttributeVector * pos) + : FeatureExecutor(), + _locations(), + _pos(pos), + _intBuf() +{ + if (_pos == nullptr) { + return; + } + _intBuf.allocate(_pos->getMaxValueCount()); + for (const auto * p : locations) { + if (p && p->location.valid()) { + double lat = p->location.point.y * 1.0e-6; + double lng = p->location.point.x * 1.0e-6; + _locations.emplace_back(search::common::GeoGcd{lat, lng}); + } + } +} + +void +GCDExecutor::execute(uint32_t docId) +{ + outputs().set_number(0, calculateGCD(docId)); + outputs().set_number(1, _best_lat); // latitude + outputs().set_number(2, _best_lng); // longitude +} + + +GreatCircleDistanceBlueprint::GreatCircleDistanceBlueprint() : + Blueprint("great_circle_distance"), + _attr_name() +{ +} + +GreatCircleDistanceBlueprint::~GreatCircleDistanceBlueprint() = default; + +void GreatCircleDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const +{ +} + +Blueprint::UP +GreatCircleDistanceBlueprint::createInstance() const +{ + return std::make_unique<GreatCircleDistanceBlueprint>(); +} + +bool +GreatCircleDistanceBlueprint::setup_geopos(const IIndexEnvironment & env, const vespalib::string &attr) +{ + _attr_name = attr; + describeOutput("km", "The distance (in km) from the query position."); + describeOutput("latitude", "Latitude of closest point"); + describeOutput("longitude", "Longitude of closest point"); + env.hintAttributeAccess(_attr_name); + return true; +} + + +bool +GreatCircleDistanceBlueprint::setup(const IIndexEnvironment & env, + const ParameterList & params) +{ + if (params.size() == 1) { + _field_name = params[0].getValue(); + } else if (params.size() == 2) { + // params[0] = "field" + // params[1] = attribute name + if (params[0].getValue() == "field") { + _field_name = params[1].getValue(); + } else { + LOG(error, "first argument must be 'field' but was '%s'", params[0].getValue().c_str()); + return false; + } + } else { + LOG(error, "Wants 2 parameters, but got %zd", params.size()); + return false; + } + vespalib::string z = document::PositionDataType::getZCurveFieldName(_field_name); + const auto *fi = env.getFieldByName(z); + if (fi != nullptr && fi->hasAttribute()) { + auto dt = fi->get_data_type(); + auto ct = fi->collection(); + LOG(spam, "index env has attribute for field '%s' which is: %s%s", + z.c_str(), + (ct == CollectionType::SINGLE ? "" : + (ct == CollectionType::ARRAY ? "array of " : "collection of ")), + (dt == DataType::INT64 ? "int64" : + (dt == DataType::DOUBLE ? "double" : "something"))); + /* we can't check these because streaming has wrong information + if (dt == DataType::INT64) { + if (ct == CollectionType::SINGLE || ct == CollectionType::ARRAY) { + return setup_geopos(env, z); + } + } + */ + return setup_geopos(env, z); + } + if (env.getFieldByName(_field_name) == nullptr && fi == nullptr) { + LOG(error, "unknown field '%s' for rank feature %s\n", _field_name.c_str(), getName().c_str()); + } else { + LOG(error, "field '%s' must be type position and attribute for rank feature %s\n", _field_name.c_str(), getName().c_str()); + } + return false; +} + +FeatureExecutor & +GreatCircleDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const +{ + // expect geo pos: + const search::attribute::IAttributeVector * pos = nullptr; + GeoLocationSpecPtrs matching_locs; + GeoLocationSpecPtrs other_locs; + + for (auto loc_ptr : env.getAllLocations()) { + if (loc_ptr && loc_ptr->location.valid()) { + if (loc_ptr->field_name == _attr_name || + loc_ptr->field_name == _field_name) + { + LOG(debug, "found loc from query env matching '%s'", _attr_name.c_str()); + matching_locs.push_back(loc_ptr); + } else { + LOG(debug, "found loc(%s) from query env not matching arg(%s)", + loc_ptr->field_name.c_str(), _attr_name.c_str()); + other_locs.push_back(loc_ptr); + } + } + } + if (matching_locs.empty() && other_locs.empty()) { + LOG(debug, "createExecutor: no valid locations"); + return stash.create<GCDExecutor>(matching_locs, nullptr); + } + LOG(debug, "createExecutor: valid location, attribute='%s'", _attr_name.c_str()); + pos = env.getAttributeContext().getAttribute(_attr_name); + if (pos != nullptr) { + if (!pos->isIntegerType()) { + Issue::report("distance feature: The position attribute '%s' is not an integer attribute.", + pos->getName().c_str()); + pos = nullptr; + } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { + Issue::report("distance feature: The position attribute '%s' is a weighted set attribute.", + pos->getName().c_str()); + pos = nullptr; + } + } else { + Issue::report("distance feature: The position attribute '%s' was not found.", _attr_name.c_str()); + } + LOG(debug, "use '%s' locations with pos=%p", matching_locs.empty() ? "other" : "matching", pos); + return stash.create<GCDExecutor>(matching_locs.empty() ? other_locs : matching_locs, pos); +} + +} diff --git a/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h new file mode 100644 index 00000000000..22a464ed5fa --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/common/geo_gcd.h> + +namespace search::features { + +/** Convenience typedef. */ +using GeoLocationSpecPtrs = std::vector<const search::common::GeoLocationSpec *>; + +/** + * Implements the executor for the great circle distance feature. + */ +class GCDExecutor : public fef::FeatureExecutor { +private: + std::vector<search::common::GeoGcd> _locations; + const attribute::IAttributeVector * _pos; + attribute::IntegerContent _intBuf; + feature_t _best_lat; + feature_t _best_lng; + + feature_t calculateGCD(uint32_t docId); +public: + /** + * Constructs an executor for the GCD feature. + * + * @param locations location objects associated with the query environment. + * @param pos the attribute to use for positions (expects zcurve encoding). + */ + GCDExecutor(GeoLocationSpecPtrs locations, const attribute::IAttributeVector * pos); + void execute(uint32_t docId) override; +}; + +/** + * Implements the blueprint for the GCD executor. + */ +class GreatCircleDistanceBlueprint : public fef::Blueprint { +private: + vespalib::string _field_name; + vespalib::string _attr_name; + bool setup_geopos(const fef::IIndexEnvironment & env, const vespalib::string &attr); +public: + GreatCircleDistanceBlueprint(); + ~GreatCircleDistanceBlueprint(); + void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; + fef::Blueprint::UP createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override { + return fef::ParameterDescriptions().desc().string().desc().string().string(); + } + bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp index c28abe10b4a..cdeb0515659 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp @@ -150,6 +150,7 @@ OnnxBlueprint::setup(const IIndexEnvironment &env, return fail("undefined input: %s (->%s)", input_feature.value().c_str(), model_input.name.c_str()); } } + planner.prepare_output_types(*_model); for (size_t i = 0; i < _model->outputs().size(); ++i) { const auto &model_output = _model->outputs()[i]; auto output_name = model_cfg->output_name(model_output.name); diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 069eab190ec..434040c7dad 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -127,6 +127,24 @@ public: void execute(uint32_t docId) override; }; +/** + * Implements the executor for interpreted ranking expressions (with + * tensor support) that will unbox the result. + **/ +class UnboxingInterpretedRankingExpressionExecutor : public fef::FeatureExecutor +{ +private: + const InterpretedFunction &_function; + InterpretedFunction::Context _context; + MyLazyParams _params; + +public: + UnboxingInterpretedRankingExpressionExecutor(const InterpretedFunction &function, + ConstArrayRef<char> input_is_object); + bool isPure() override { return true; } + void execute(uint32_t docId) override; +}; + //----------------------------------------------------------------------------- FastForestExecutor::FastForestExecutor(ArrayRef<float> param_space, const FastForest &forest) @@ -215,6 +233,22 @@ InterpretedRankingExpressionExecutor::execute(uint32_t) //----------------------------------------------------------------------------- +UnboxingInterpretedRankingExpressionExecutor::UnboxingInterpretedRankingExpressionExecutor(const InterpretedFunction &function, + ConstArrayRef<char> input_is_object) + : _function(function), + _context(function), + _params(inputs(), input_is_object) +{ +} + +void +UnboxingInterpretedRankingExpressionExecutor::execute(uint32_t) +{ + outputs().set_number(0, _function.eval(_context, _params).as_double()); +} + +//----------------------------------------------------------------------------- + RankingExpressionBlueprint::RankingExpressionBlueprint() : RankingExpressionBlueprint(std::make_shared<rankingexpression::NullExpressionReplacer>()) {} @@ -225,7 +259,8 @@ RankingExpressionBlueprint::RankingExpressionBlueprint(rankingexpression::Expres _fast_forest(), _interpreted_function(), _compile_token(), - _input_is_object() + _input_is_object(), + _should_unbox(false) { } @@ -328,9 +363,10 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, } else { _interpreted_function.reset(new InterpretedFunction(FastValueBuilderFactory::get(), *rank_function, node_types)); + _should_unbox = root_type.is_double(); } } - FeatureType output_type = do_compile + FeatureType output_type = (do_compile || _should_unbox) ? FeatureType::number() : FeatureType::object(root_type); describeOutput("out", "The result of running the contained ranking expression.", output_type); @@ -359,7 +395,11 @@ RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, ve } if (_interpreted_function) { ConstArrayRef<char> input_is_object = stash.copy_array<char>(_input_is_object); - return stash.create<InterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); + if (_should_unbox) { + return stash.create<UnboxingInterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); + } else { + return stash.create<InterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); + } } if (_fast_forest) { ArrayRef<float> param_space = stash.create_array<float>(_input_is_object.size(), 0.0); diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h index 43aafe67da1..a759c7855d6 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h @@ -24,6 +24,7 @@ private: vespalib::eval::InterpretedFunction::UP _interpreted_function; vespalib::eval::CompileCache::Token::UP _compile_token; std::vector<char> _input_is_object; + bool _should_unbox; public: RankingExpressionBlueprint(); diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index f2d5bd745ac..2bc8a349d1b 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -10,6 +10,7 @@ #include "debug_attribute_wait.h" #include "debug_wait.h" #include "distancefeature.h" +#include "great_circle_distance_feature.h" #include "distancetopathfeature.h" #include "dotproductfeature.h" #include "element_completeness_feature.h" @@ -126,6 +127,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(std::make_shared<GlobalSequenceBlueprint>()); registry.addPrototype(std::make_shared<OnnxBlueprint>("onnx")); registry.addPrototype(std::make_shared<OnnxBlueprint>("onnxModel")); + registry.addPrototype(std::make_shared<GreatCircleDistanceBlueprint>()); // Ranking Expression auto replacers = std::make_unique<ListExpressionReplacer>(); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 808e0d81b20..bf5598f0a5e 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -21,8 +21,8 @@ public: } // namespace <unnamed> namespace search::fef { - -using namespace indexproperties; + +using namespace indexproperties; RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv) : _factory(factory), diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp index a82f4287d00..83df24ac543 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp @@ -14,10 +14,18 @@ struct UnboxExecutor : FeatureExecutor { } }; +struct ForwardExecutor : FeatureExecutor { + bool isPure() override { return true; } + void execute(uint32_t) override { + outputs().set_number(0, inputs().get_number(0)); + } +}; + } // namespace search::fef::test::<unnamed> UnboxBlueprint::UnboxBlueprint() - : Blueprint("unbox") + : Blueprint("unbox"), + _was_object(false) { } @@ -41,15 +49,22 @@ UnboxBlueprint::getDescriptions() const bool UnboxBlueprint::setup(const IIndexEnvironment &, const ParameterList ¶ms) { - defineInput(params[0].getValue(), AcceptInput::OBJECT); - describeOutput("value", "unboxed value", FeatureType::number()); - return true; + if (auto input = defineInput(params[0].getValue(), AcceptInput::ANY)) { + _was_object = input.value().is_object(); + describeOutput("value", "unboxed value", FeatureType::number()); + return true; + } + return false; // dependency error } FeatureExecutor & UnboxBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const { - return stash.create<UnboxExecutor>(); + if (_was_object) { + return stash.create<UnboxExecutor>(); + } else { + return stash.create<ForwardExecutor>(); + } } } // namespace search::fef::test diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.h b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.h index 8e311d0a173..0ce6b07d830 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.h +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.h @@ -8,6 +8,7 @@ namespace search::fef::test { struct UnboxBlueprint : Blueprint { + bool _was_object; UnboxBlueprint(); void visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const override; Blueprint::UP createInstance() const override; diff --git a/searchlib/src/vespa/searchlib/index/dictionaryfile.h b/searchlib/src/vespa/searchlib/index/dictionaryfile.h index 5063143d323..6c8535f8563 100644 --- a/searchlib/src/vespa/searchlib/index/dictionaryfile.h +++ b/searchlib/src/vespa/searchlib/index/dictionaryfile.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "postinglistcounts.h" #include "postinglisthandle.h" #include "postinglistcountfile.h" #include <vespa/searchlib/common/tunefileinfo.h> diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp index f62a4bc7997..4341bcb9a46 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docidandfeatures.h" -#include <vespa/log/log.h> -LOG_SETUP(".index.docidandfeatures"); +#include <cassert> namespace search::index { @@ -23,4 +22,21 @@ DocIdAndFeatures::DocIdAndFeatures(const DocIdAndFeatures &) = default; DocIdAndFeatures & DocIdAndFeatures::operator = (const DocIdAndFeatures &) = default; DocIdAndFeatures::~DocIdAndFeatures() = default; +void +DocIdAndPosOccFeatures::addNextOcc(uint32_t elementId, uint32_t wordPos, int32_t elementWeight, uint32_t elementLen) +{ + assert(wordPos < elementLen); + if (_elements.empty() || elementId > _elements.back().getElementId()) { + _elements.emplace_back(elementId, elementWeight, elementLen); + } else { + assert(elementId == _elements.back().getElementId()); + assert(elementWeight == _elements.back().getWeight()); + assert(elementLen == _elements.back().getElementLen()); + } + assert(_elements.back().getNumOccs() == 0 || + wordPos > _word_positions.back().getWordPos()); + _elements.back().incNumOccs(); + _word_positions.emplace_back(wordPos); +} + } diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 6b1659771fa..e595ec833ef 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -163,4 +163,10 @@ public: void set_has_raw_data(bool val) { _has_raw_data = val; } }; +class DocIdAndPosOccFeatures : public DocIdAndFeatures +{ +public: + void addNextOcc(uint32_t elementId, uint32_t wordPos, int32_t elementWeight, uint32_t elementLen); +}; + } diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp b/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp index f4c38636d01..edf4f8c43b2 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp @@ -1,16 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "postinglistcountfile.h" +#include <vespa/searchlib/index/postinglistparams.h> namespace search::index { -PostingListCountFileSeqRead::PostingListCountFileSeqRead() -{ -} - -PostingListCountFileSeqRead::~PostingListCountFileSeqRead() -{ -} +PostingListCountFileSeqRead::PostingListCountFileSeqRead() = default; +PostingListCountFileSeqRead::~PostingListCountFileSeqRead() = default; void PostingListCountFileSeqRead:: @@ -19,13 +15,8 @@ getParams(PostingListParams ¶ms) params.clear(); } -PostingListCountFileSeqWrite::PostingListCountFileSeqWrite() -{ -} - -PostingListCountFileSeqWrite::~PostingListCountFileSeqWrite() -{ -} +PostingListCountFileSeqWrite::PostingListCountFileSeqWrite() = default; +PostingListCountFileSeqWrite::~PostingListCountFileSeqWrite() = default; void PostingListCountFileSeqWrite:: diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h index 47ec202dad1..7e17fc5bb9e 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "postinglistparams.h" #include "postinglistcounts.h" #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/vespalib/stllike/string.h> namespace search::common { class FileHeaderContext; } @@ -11,6 +11,7 @@ namespace search::index { class PostingListCounts; class PostingListHandle; +class PostingListParams; /** * Interface for count files describing where in a posting list file diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp index 15412fcd5f1..4d53790bd73 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "postinglistfile.h" +#include <vespa/searchlib/index/postinglistparams.h> #include <vespa/fastos/file.h> namespace search::index { -PostingListFileSeqRead::PostingListFileSeqRead() -{ -} - +PostingListFileSeqRead::PostingListFileSeqRead() = default; PostingListFileSeqRead::~PostingListFileSeqRead() = default; void @@ -37,9 +35,7 @@ PostingListFileSeqWrite::PostingListFileSeqWrite() { } -PostingListFileSeqWrite::~PostingListFileSeqWrite() -{ -} +PostingListFileSeqWrite::~PostingListFileSeqWrite() = default; void PostingListFileSeqWrite:: @@ -75,9 +71,7 @@ PostingListFileRandRead() { } -PostingListFileRandRead::~PostingListFileRandRead() -{ -} +PostingListFileRandRead::~PostingListFileRandRead() = default; void PostingListFileRandRead::afterOpen(FastOS_FileInterface &file) @@ -117,8 +111,7 @@ readPostingList(const PostingListCounts &counts, uint32_t numSegments, PostingListHandle &handle) { - _lower->readPostingList(counts, firstSegment, numSegments, - handle); + _lower->readPostingList(counts, firstSegment, numSegments,handle); } bool diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index d731b3f0f67..a33319e1d4f 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -3,8 +3,8 @@ #include "postinglistcounts.h" #include "postinglisthandle.h" -#include "postinglistparams.h" #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/vespalib/stllike/string.h> class FastOS_FileInterface; @@ -14,6 +14,7 @@ namespace search::index { class DocIdAndFeatures; class FieldLengthInfo; +class PostingListParams; /** * Interface for posting list files containing document ids and features diff --git a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp index 74e8f731f6f..6275399c498 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp @@ -49,6 +49,12 @@ PostingListParams::clear() } void +PostingListParams::add(const PostingListParams & toAdd) +{ + _map.insert(toAdd._map.begin(), toAdd._map.end()); +} + +void PostingListParams::erase(const vespalib::string &key) { _map.erase(key); @@ -62,8 +68,7 @@ PostingListParams::operator!=(const PostingListParams &rhs) const template <typename TYPE> void -PostingListParams::set(const vespalib::string &key, - const TYPE &val) +PostingListParams::set(const vespalib::string &key, const TYPE &val) { std::ostringstream os; @@ -73,8 +78,7 @@ PostingListParams::set(const vespalib::string &key, template <typename TYPE> void -PostingListParams::get(const vespalib::string &key, - TYPE &val) const +PostingListParams::get(const vespalib::string &key, TYPE &val) const { std::istringstream is; Map::const_iterator it; @@ -87,35 +91,27 @@ PostingListParams::get(const vespalib::string &key, } template void -PostingListParams::set<bool>(const vespalib::string &key, - const bool &val); +PostingListParams::set<bool>(const vespalib::string &key, const bool &val); template void -PostingListParams::get<bool>(const vespalib::string &key, - bool &val) const; +PostingListParams::get<bool>(const vespalib::string &key, bool &val) const; template void -PostingListParams::set<int32_t>(const vespalib::string &key, - const int32_t &val); +PostingListParams::set<int32_t>(const vespalib::string &key, const int32_t &val); template void -PostingListParams::get<int32_t>(const vespalib::string &key, - int32_t &val) const; +PostingListParams::get<int32_t>(const vespalib::string &key, int32_t &val) const; template void -PostingListParams::set<uint32_t>(const vespalib::string &key, - const uint32_t &val); +PostingListParams::set<uint32_t>(const vespalib::string &key, const uint32_t &val); template void -PostingListParams::get<uint32_t>(const vespalib::string &key, - uint32_t &val) const; +PostingListParams::get<uint32_t>(const vespalib::string &key, uint32_t &val) const; template void -PostingListParams::set<uint64_t>(const vespalib::string &key, - const uint64_t &val); +PostingListParams::set<uint64_t>(const vespalib::string &key, const uint64_t &val); template void -PostingListParams::get<uint64_t>(const vespalib::string &key, - uint64_t &val) const; +PostingListParams::get<uint64_t>(const vespalib::string &key, uint64_t &val) const; } diff --git a/searchlib/src/vespa/searchlib/index/postinglistparams.h b/searchlib/src/vespa/searchlib/index/postinglistparams.h index 9797eef5278..42da5855c23 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistparams.h +++ b/searchlib/src/vespa/searchlib/index/postinglistparams.h @@ -7,34 +7,22 @@ namespace search::index { class PostingListParams { - typedef std::map<vespalib::string, vespalib::string> Map; + using Map = std::map<vespalib::string, vespalib::string>; Map _map; public: template <typename TYPE> - void - set(const vespalib::string &key, const TYPE &val); + void set(const vespalib::string &key, const TYPE &val); template <typename TYPE> - void - get(const vespalib::string &key, TYPE &val) const; - - bool - isSet(const vespalib::string &key) const; - - void - setStr(const vespalib::string &key, const vespalib::string &val); - - const vespalib::string & - getStr(const vespalib::string &key) const; - - void - clear(); - - void - erase(const vespalib::string &key); - - bool - operator!=(const PostingListParams &rhs) const; + void get(const vespalib::string &key, TYPE &val) const; + + bool isSet(const vespalib::string &key) const; + void setStr(const vespalib::string &key, const vespalib::string &val); + const vespalib::string & getStr(const vespalib::string &key) const; + void clear(); + void erase(const vespalib::string &key); + bool operator!=(const PostingListParams &rhs) const; + void add(const PostingListParams & toAdd); }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index fdb2de8fb59..c55de3890cd 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/common/schedule_sequenced_task_callback.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/retain_guard.h> +#include <cassert> namespace search::memoryindex { @@ -28,8 +29,7 @@ DocumentInverter::DocumentInverter(DocumentInverterContext& context) { auto& schema = context.get_schema(); auto& field_indexes = context.get_field_indexes(); - for (uint32_t fieldId = 0; fieldId < schema.getNumIndexFields(); - ++fieldId) { + for (uint32_t fieldId = 0; fieldId < schema.getNumIndexFields(); ++fieldId) { auto &remover(field_indexes.get_remover(fieldId)); auto &inserter(field_indexes.get_inserter(fieldId)); auto &calculator(field_indexes.get_calculator(fieldId)); diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index 2ad1fd78f07..9f17d369208 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/index/docidandfeatures.h> -#include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/bitcompression/posocc_fields_params.h> #include <vespa/vespalib/datastore/datastore.h> diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index 5f4d02d23db..c606b9b6340 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -5,6 +5,7 @@ #include "ordered_field_index_inserter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/index/i_field_length_inspector.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index d96b8491027..a443e994559 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -13,6 +13,7 @@ #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> #include <vespa/vespalib/text/utf8.h> @@ -443,6 +444,17 @@ FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) } void +FieldInverter::startDoc(uint32_t docId) { + assert(_docId == 0); + assert(docId != 0); + abortPendingDoc(docId); + _removeDocs.push_back(docId); + _docId = docId; + _elem = 0; + _wpos = 0; +} + +void FieldInverter::invertNormalDocTextField(const FieldValue &val) { const vespalib::Identifiable::RuntimeClass & cInfo(val.getClass()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 56cb1677f67..36dd6339b54 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -6,13 +6,14 @@ #include <vespa/document/annotation/span.h> #include <vespa/document/datatype/datatypes.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/allocator.h> #include <limits> -namespace search::index { class FieldLengthCalculator; } +namespace search::index { + class FieldLengthCalculator; + class Schema; +} namespace search::memoryindex { @@ -310,15 +311,7 @@ public: _removeDocs.push_back(docId); } - void startDoc(uint32_t docId) { - assert(_docId == 0); - assert(docId != 0); - abortPendingDoc(docId); - _removeDocs.push_back(docId); - _docId = docId; - _elem = 0; - _wpos = 0; - } + void startDoc(uint32_t docId); void endDoc(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index 10918a83c50..326b7b0967a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -389,7 +389,7 @@ UrlFieldInverter::pushDocuments() _hostname->pushDocuments(); } -UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, +UrlFieldInverter::UrlFieldInverter(index::schema::CollectionType collectionType, FieldInverter *all, FieldInverter *scheme, FieldInverter *host, diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.h b/searchlib/src/vespa/searchlib/query/query_term_simple.h index cfdba03855c..433ab7d56dd 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.h +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.h @@ -21,7 +21,8 @@ public: SUBSTRINGTERM = 2, EXACTSTRINGTERM = 3, SUFFIXTERM = 4, - REGEXP = 5 + REGEXP = 5, + GEO_LOCATION = 6 }; template <typename N> @@ -59,6 +60,7 @@ public: bool isSuffix() const { return (_type == Type::SUFFIXTERM); } bool isWord() const { return (_type == Type::WORD); } bool isRegex() const { return (_type == Type::REGEXP); } + bool isGeoLoc() const { return (_type == Type::GEO_LOCATION); } bool empty() const { return _term.empty(); } virtual void visitMembers(vespalib::ObjectVisitor &visitor) const; vespalib::string getClassName() const; diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 060573861d9..77fc97913a4 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -71,11 +71,12 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor qn = std::make_unique<FalseNode>(); break; case ParseItem::ITEM_GEO_LOCATION_TERM: - // TODO implement this: - // vespalib::string field = queryRep.getIndexName(); - // vespalib::stringref location_term = queryRep.getTerm(); - // qn = std::make_unique<LocationQueryNode> ...something .... - // break; + // just keep the string representation here; parsed in vsm::GeoPosFieldSearcher + qn = std::make_unique<QueryTerm>(factory.create(), + queryRep.getTerm(), + queryRep.getIndexName(), + QueryTerm::Type::GEO_LOCATION); + break; case ParseItem::ITEM_NUMTERM: case ParseItem::ITEM_TERM: case ParseItem::ITEM_PREFIXTERM: diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 9a35e4a2b05..ae34cdd66c8 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchlib_tensor OBJECT SOURCES angular_distance.cpp bitvector_visited_tracker.cpp + blob_sequence_reader.cpp default_nearest_neighbor_index_factory.cpp dense_tensor_attribute.cpp dense_tensor_attribute_saver.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp new file mode 100644 index 00000000000..0d86af2f3a5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "blob_sequence_reader.h" +#include <vespa/fastos/file.h> + +namespace search::tensor { + +void +BlobSequenceReader::readBlob(void *buf, size_t len) { + _datFile.file().ReadBuf(buf, len); +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h index 5b7efc73a02..45fcf5524d2 100644 --- a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h +++ b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastlib/io/bufferedfile.h> +#pragma once + #include <vespa/searchlib/attribute/readerbase.h> -#include <vespa/searchlib/util/fileutil.h> namespace search::tensor { @@ -20,7 +20,7 @@ public: _sizeReader(_datFile.file()) { } uint32_t getNextSize() { return _sizeReader.readHostOrder(); } - void readBlob(void *buf, size_t len) { _datFile.file().ReadBuf(buf, len); } + void readBlob(void *buf, size_t len); }; } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 113883a307f..d376fb020be 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/memory_allocator.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> @@ -21,6 +22,7 @@ LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); using search::attribute::LoadUtils; +using vespalib::CpuUsage; using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::slime::ObjectInserter; @@ -97,17 +99,6 @@ BlobSequenceReader::is_present() { return true; } - - -std::unique_ptr<vespalib::alloc::MemoryAllocator> -make_memory_allocator(const vespalib::string& name, bool swappable) -{ - if (swappable) { - return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); - } - return {}; -} - } void @@ -159,7 +150,7 @@ DenseTensorAttribute::populate_address_space_usage(AddressSpaceUsage& usage) con DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.paged())), + _denseTensorStore(cfg.tensorType(), get_memory_allocator()), _index() { if (cfg.hnsw_index_params().has_value()) { @@ -324,7 +315,7 @@ DenseTensorAttribute::ThreadedLoader::load(uint32_t lid, vespalib::datastore::En // Then we can issue a new one ++_pending; - _shared_executor.execute(vespalib::makeLambdaTask([this, ref, lid]() { + auto task = vespalib::makeLambdaTask([this, ref, lid]() { auto prepared = _attr._index->prepare_add_document(lid, _attr._denseTensorStore.get_typed_cells(ref), _attr.getGenerationHandler().takeGuard()); std::unique_lock guard(_mutex); @@ -332,7 +323,8 @@ DenseTensorAttribute::ThreadedLoader::load(uint32_t lid, vespalib::datastore::En if (_queue.size() == 1) { _cond.notify_all(); } - })); + }); + _shared_executor.execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::SETUP)); } class DenseTensorAttribute::ForegroundLoader : public Loader { public: diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index ed3fb737b7d..6435ba6f27c 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -45,7 +45,7 @@ DenseTensorStore::TensorSizeCalc::TensorSizeCalc(const ValueType &type) _aligned_size = my_align(buf_size, alignment); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator) : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), _allocator(std::move(allocator)) {} @@ -65,7 +65,7 @@ DenseTensorStore::BufferType::get_memory_allocator() const return _allocator.get(); } -DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) +DenseTensorStore::DenseTensorStore(const ValueType &type, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 47932fbff7e..7176edbcf08 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -37,9 +37,9 @@ public: class BufferType : public vespalib::datastore::BufferType<char> { using CleanContext = vespalib::datastore::BufferType<char>::CleanContext; - std::unique_ptr<vespalib::alloc::MemoryAllocator> _allocator; + std::shared_ptr<vespalib::alloc::MemoryAllocator> _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; @@ -55,7 +55,7 @@ private: TensorStore::EntryRef setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + DenseTensorStore(const ValueType &type, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; const ValueType &type() const { return _type; } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index 0cb23fe2ae9..0ae55a670da 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -5,7 +5,6 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value.h> -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/util/array.h> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 6eb215273ad..8beb111ca59 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -10,8 +10,8 @@ namespace search::tensor { HnswGraph::HnswGraph() : node_refs(), node_refs_size(1u), - nodes(HnswIndex::make_default_node_store_config()), - links(HnswIndex::make_default_link_store_config()), + nodes(HnswIndex::make_default_node_store_config(), {}), + links(HnswIndex::make_default_link_store_config(), {}), entry_docid_and_level() { node_refs.ensure_size(1, AtomicEntryRef()); diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 16d9f2e3bbd..a0ec04b98cb 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -3,10 +3,7 @@ #include "serialized_fast_value_attribute.h" #include "streamed_value_saver.h" #include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/fast_value.h> #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/searchlib/attribute/readerbase.h> -#include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp index 0901e643afa..2e6d771a870 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp @@ -11,7 +11,6 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/typify.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.streamed_value_store"); @@ -22,6 +21,7 @@ using namespace vespalib::eval; using vespalib::ConstArrayRef; using vespalib::MemoryUsage; using vespalib::string_id; +using vespalib::StringIdVector; namespace search::tensor { @@ -61,12 +61,12 @@ struct MyFastValueView final : Value { const ValueType &my_type; FastValueIndex my_index; TypedCells my_cells; - MyFastValueView(const ValueType &type_ref, const std::vector<string_id> &handle_view, TypedCells cells, size_t num_mapped, size_t num_spaces) + MyFastValueView(const ValueType &type_ref, const StringIdVector &handle_view, TypedCells cells, size_t num_mapped, size_t num_spaces) : my_type(type_ref), my_index(num_mapped, handle_view, num_spaces), my_cells(cells) { - const std::vector<string_id> &labels = handle_view; + const StringIdVector &labels = handle_view; for (size_t i = 0; i < num_spaces; ++i) { ConstArrayRef<string_id> addr(&labels[i * num_mapped], num_mapped); my_index.map.add_mapping(FastAddrMap::hash_labels(addr)); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 088019749af..c6646f2e61f 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -5,6 +5,8 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/bitcompression/posocc_fields_params.h> +#include <vespa/searchcommon/common/schema.h> + namespace vespalib { class Rand48; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 87efc8132ee..8d5f6d6db4e 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/diskindex/zc4_posting_params.h> #include <vespa/searchlib/diskindex/zc4_posting_reader.h> #include <vespa/searchlib/diskindex/zc4_posting_writer.h> +#include <vespa/searchlib/index/postinglistparams.h> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 2ca2f15545d..eea9610c7ab 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -124,7 +124,9 @@ handleReadError(const char *text, FastOS_FileInterface &file, ssize_t len, ssize e += fmt(" Truncate to %" PRId64 " and continue", lastKnownGoodPos); LOG(error, "%s", e.c_str()); FastOS_File truncateFile(file.GetFileName()); - file.Close(); + if ( ! file.Close()) { + e += getError(file); + } if ( truncateFile.OpenWriteOnlyExisting()) { if (truncateFile.SetSize(lastKnownGoodPos)) { if (truncateFile.Close()) { @@ -243,7 +245,6 @@ DomainPart::buildPacketMapping(bool allowTruncate) } currPos = transLog.GetPosition(); } - transLog.Close(); return currPos; } @@ -429,7 +430,8 @@ void DomainPart::write(FastOS_FileInterface &file, SerialNumRange range, vespalib::ConstBufferRef buf) { std::lock_guard guard(_writeLock); - if ( ! file.CheckedWrite(buf.data(), buf.size()) ) { + size_t written = file.Write2(buf.data(), buf.size()); + if ( written != buf.size() ) { throw runtime_error(handleWriteError("Failed writing the entry.", file, byteSize(), range, buf.size())); } LOG(debug, "Wrote chunk with and %zu bytes, range[%" PRIu64 ", %" PRIu64 "]", buf.size(), range.from(), range.to()); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index db2cf2a255d..ccef92c0802 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -2,27 +2,29 @@ #include "translogserver.h" #include "domain.h" #include "client_common.h" -#include <vespa/vespalib/util/destructor_callbacks.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/transport.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fnet/frt/supervisor.h> -#include <vespa/fnet/frt/rpcrequest.h> -#include <vespa/fnet/transport.h> +#include <vespa/vespalib/util/stringfmt.h> #include <fstream> #include <thread> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.server"); -using vespalib::make_string; -using vespalib::stringref; -using vespalib::IllegalArgumentException; using search::common::FileHeaderContext; using std::make_shared; using std::runtime_error; +using vespalib::CpuUsage; +using vespalib::IllegalArgumentException; +using vespalib::make_string; +using vespalib::stringref; using namespace std::chrono_literals; namespace search::transactionlog { @@ -97,7 +99,7 @@ TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, con _name(name), _baseDir(baseDir), _domainConfig(cfg), - _executor(maxThreads, 128_Ki, tls_executor), + _executor(maxThreads, 128_Ki, CpuUsage::wrap(tls_executor, CpuUsage::Category::WRITE)), _threadPool(std::make_unique<FastOS_ThreadPool>(120_Ki)), _transport(std::make_unique<FNET_Transport>()), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())), diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp index 9ca3b678054..927047cecaa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp @@ -2,6 +2,7 @@ #include "translogserverapp.h" #include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> #include <vespa/vespalib/util/time.h> #include <vespa/log/log.h> @@ -16,11 +17,11 @@ TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri, : _lock(), _tls(), _tlsConfig(), - _tlsConfigFetcher(tlsConfigUri.getContext()), + _tlsConfigFetcher(std::make_unique<config::ConfigFetcher>(tlsConfigUri.getContext())), _fileHeaderContext(fileHeaderContext) { - _tlsConfigFetcher.subscribe<searchlib::TranslogserverConfig>(tlsConfigUri.getConfigId(), this); - _tlsConfigFetcher.start(); + _tlsConfigFetcher->subscribe<searchlib::TranslogserverConfig>(tlsConfigUri.getConfigId(), this); + _tlsConfigFetcher->start(); } namespace { @@ -78,22 +79,29 @@ logReconfig(const searchlib::TranslogserverConfig & cfg, const DomainConfig & dc dcfg.getPartSizeLimit(), dcfg.getChunkSizeLimit()); } +size_t +derive_num_threads(uint32_t configured_cores, uint32_t actual_cores) { + return (configured_cores > 0) + ? configured_cores + : std::max(1u, std::min(4u, actual_cores/8)); +} + } void -TransLogServerApp::start() +TransLogServerApp::start(uint32_t num_cores) { std::lock_guard<std::mutex> guard(_lock); auto c = _tlsConfig.get(); DomainConfig domainConfig = getDomainConfig(*c); logReconfig(*c, domainConfig); _tls = std::make_shared<TransLogServer>(c->servername, c->listenport, c->basedir, _fileHeaderContext, - domainConfig, c->maxthreads); + domainConfig, derive_num_threads(c->maxthreads, num_cores)); } TransLogServerApp::~TransLogServerApp() { - _tlsConfigFetcher.close(); + _tlsConfigFetcher->close(); } void diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h index 7c3ebdcf22c..ccd207ed6d1 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.h @@ -3,10 +3,13 @@ #include "translogserver.h" #include <vespa/searchlib/config/config-translogserver.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/vespalib/util/ptrholder.h> -namespace config { class ConfigUri; } +namespace config { + class ConfigFetcher; + class ConfigUri; +} namespace search::common { class FileHeaderContext; } namespace search::transactionlog { @@ -17,7 +20,7 @@ private: mutable std::mutex _lock; TransLogServer::SP _tls; vespalib::PtrHolder<searchlib::TranslogserverConfig> _tlsConfig; - config::ConfigFetcher _tlsConfigFetcher; + std::unique_ptr<config::ConfigFetcher> _tlsConfigFetcher; const common::FileHeaderContext & _fileHeaderContext; void configure(std::unique_ptr<searchlib::TranslogserverConfig> cfg) override ; @@ -29,7 +32,7 @@ public: TransLogServer::SP getTransLogServer() const; - void start(); + void start(uint32_t num_cores); }; } diff --git a/searchlib/src/vespa/searchlib/util/file_with_header.cpp b/searchlib/src/vespa/searchlib/util/file_with_header.cpp index b004f2b29d5..f1b66a63bd7 100644 --- a/searchlib/src/vespa/searchlib/util/file_with_header.cpp +++ b/searchlib/src/vespa/searchlib/util/file_with_header.cpp @@ -5,6 +5,7 @@ #include "filesizecalculator.h" #include <vespa/fastos/file.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> namespace search { @@ -30,7 +31,8 @@ FileWithHeader::FileWithHeader(std::unique_ptr<FastOS_FileInterface> file_in) _header_len = _header.readFile(*_file); _file->SetPosition(_header_len); if (!extract_file_size(_header, *_file, _file_size)) { - _file->Close(); + bool close_ok = _file->Close(); + assert(close_ok); } } } @@ -52,7 +54,8 @@ FileWithHeader::rewind() void FileWithHeader::close() { - _file->Close(); + bool close_ok = _file->Close(); + assert(close_ok); } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index 5c213d4e0f0..e85e792f492 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,7 +4,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> -#include <cassert> +#include <vespa/fastlib/io/bufferedfile.h> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> @@ -19,12 +19,11 @@ using vespalib::GenericHeader; using vespalib::FileDescriptor; using vespalib::getLastErrorString; -namespace search { -namespace fileutil { +namespace search::fileutil { LoadedMmap::LoadedMmap(const vespalib::string &fileName) - : LoadedBuffer(NULL, 0), - _mapBuffer(NULL), + : LoadedBuffer(nullptr, 0), + _mapBuffer(nullptr), _mapSize(0) { FileDescriptor fd(open(fileName.c_str(), O_RDONLY, 0664)); @@ -35,7 +34,7 @@ LoadedMmap::LoadedMmap(const vespalib::string &fileName) uint64_t fileSize = stbuf.st_size; size_t sz = fileSize; if (sz) { - void *tmpBuffer = mmap(NULL, sz, PROT_READ, MAP_PRIVATE, fd.fd(), 0); + void *tmpBuffer = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd.fd(), 0); if (tmpBuffer != MAP_FAILED) { _mapSize = sz; _mapBuffer = tmpBuffer; @@ -78,14 +77,15 @@ LoadedMmap::~LoadedMmap() { } +namespace search { + std::unique_ptr<FastOS_FileInterface> FileUtil::openFile(const vespalib::string &fileName) { - std::unique_ptr<Fast_BufferedFile> file(new Fast_BufferedFile()); + auto file = std::make_unique<Fast_BufferedFile>(); file->EnableDirectIO(); if (!file->OpenReadOnly(fileName.c_str())) { LOG(error, "could not open %s: %s", file->GetFileName(), getLastErrorString().c_str()); - file->Close(); throw IllegalStateException(make_string("Failed opening '%s' for direct IO reading.", file->GetFileName())); } return file; @@ -97,12 +97,11 @@ using fileutil::LoadedMmap; LoadedBuffer::UP FileUtil::loadFile(const vespalib::string &fileName) { - LoadedBuffer::UP data(new LoadedMmap(fileName)); + auto data = std::make_unique<LoadedMmap>(fileName); FastOS_File file(fileName.c_str()); if (!file.OpenReadOnly()) { LOG(error, "could not open %s: %s", file.GetFileName(), getLastErrorString().c_str()); } - file.Close(); return data; } @@ -125,44 +124,6 @@ void FileWriterBase::handleError(ssize_t numRead, size_t wanted) } } -SequentialFileArray::SequentialFileArray(const vespalib::string & fname) : - _backingFile(), - _name(fname) -{ - _backingFile->EnableDirectIO(); -} - -SequentialFileArray::~SequentialFileArray() -{ - close(); -} - -void SequentialFileArray::rewind() -{ - assert(_backingFile->SetPosition(0)); -} - -void SequentialFileArray::close() -{ - _backingFile->Close(); -} - -void SequentialFileArray::erase() -{ - close(); - FastOS_File::Delete(_backingFile->GetFileName()); -} - -void SequentialFileArray::openReadOnly() -{ - _backingFile->ReadOpen(_name.c_str()); -} - -void SequentialFileArray::openWriteOnly() -{ - _backingFile->OpenWriteOnlyTruncate(_name.c_str()); -} - ssize_t FileReaderBase::read(void *buf, size_t sz) { ssize_t numRead = _file.Read(buf, sz); diff --git a/searchlib/src/vespa/searchlib/util/fileutil.h b/searchlib/src/vespa/searchlib/util/fileutil.h index 27761403b96..8271265aa33 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.h +++ b/searchlib/src/vespa/searchlib/util/fileutil.h @@ -8,7 +8,6 @@ #include <vespa/vespalib/stllike/string.h> using vespalib::GenericHeader; -class Fast_BufferedFile; namespace search { @@ -104,67 +103,6 @@ public: } }; -class SequentialFileArray -{ -public: - SequentialFileArray(const vespalib::string & fname); - virtual ~SequentialFileArray(); - const vespalib::string & getName() const { return _name; } - void rewind(); - void close(); - void erase(); -protected: - void openReadOnly(); - void openWriteOnly(); - std::unique_ptr<Fast_BufferedFile> _backingFile; - vespalib::string _name; -}; - -template <typename T> -class SequentialFileArrayRead : public SequentialFileArray -{ -public: - SequentialFileArrayRead(const vespalib::string & fname); - ~SequentialFileArrayRead(); - T getNext() const { return _fileReader.readHostOrder(); } - bool hasNext() const; - size_t size() const; -private: - mutable FileReader<T> _fileReader; -}; - -template <typename T> -class SequentialFileArrayWrite : public SequentialFileArray -{ -public: - SequentialFileArrayWrite(const vespalib::string & fname); - void push_back(const T & v) { _count++; _fileWriter.write(&v, sizeof(v)); } - size_t size() const { return _count; } - bool empty() const { return _count == 0; } -private: - size_t _count; - FileWriterBase _fileWriter; -}; - -template <typename T, typename S> -class MergeSorter -{ -public: - MergeSorter(const vespalib::string & name, size_t chunkSize); - void push_back(const T & v); - void commit() { sortChunk(); merge(); } - const vespalib::string & getName() const { return _name; } - void rewind() { } -private: - vespalib::string genName(size_t n); - void merge(); - void sortChunk(); - - std::vector<T> _chunk; - size_t _chunkCount; - vespalib::string _name; -}; - template <typename T> class SequentialReadModifyWriteInterface { @@ -199,32 +137,5 @@ private: size_t _wp; }; -template <typename T, typename R, typename W> -class SequentialReaderWriter : public SequentialReadModifyWriteInterface<T> -{ -public: - SequentialReaderWriter(R & reader, W & writer); - ~SequentialReaderWriter(); - virtual const T & read() { return _lastRead; } - virtual void write(const T & v) { _writer.push_back(v); } - virtual bool next() { - bool hasMore(_reader.hasNext()); - if (hasMore) { - _lastRead = _reader.getNext(); - } - return hasMore; - } - virtual size_t size() const { return _reader.size(); } - virtual void rewind() { - _reader.rewind(); - next(); - _writer.rewind(); - } -private: - T _lastRead; - R & _reader; - W & _writer; -}; - } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.hpp b/searchlib/src/vespa/searchlib/util/fileutil.hpp index eb0d62b195e..5b5303ef169 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.hpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.hpp @@ -2,127 +2,10 @@ #pragma once #include "fileutil.h" -#include <vespa/fastlib/io/bufferedfile.h> namespace search { template <typename T> -SequentialFileArrayRead<T>::SequentialFileArrayRead(const vespalib::string & fname) : - SequentialFileArray(fname), - _fileReader(std::make_unique<Fast_BufferedFile>(_backingFile)) -{ - openReadOnly(); -} - -template <typename T> -bool -SequentialFileArrayRead<T>::hasNext() const { - return _backingFile->BytesLeft() >= sizeof(T); - -} - -template <typename T> -size_t SequentialFileArrayRead<T>::size() const -{ - return _backingFile->GetSize()/sizeof(T); -} - -template <typename T> -SequentialFileArrayWrite<T>::SequentialFileArrayWrite(const vespalib::string & fname) : - SequentialFileArray(fname), - _count(0), - _fileWriter(_backingFile) -{ - openWriteOnly(); -} - -template <typename T, typename S> -MergeSorter<T, S>::MergeSorter(const vespalib::string & name, size_t chunkSize) : - _chunk(), - _chunkCount(0), - _name(name + ".sorted") -{ - _chunk.reserve(chunkSize); -} - -template <typename T, typename S> -void MergeSorter<T, S>::push_back(const T & v) -{ - if (_chunk.size() < _chunk.capacity()) { - _chunk.push_back(v); - if (_chunk.size() == _chunk.capacity()) { - sortChunk(); - } - } -} - -template <typename T, typename S> -vespalib::string MergeSorter<T, S>::genName(size_t n) -{ - char tmp[32]; - sprintf(tmp, ".%zd", n); - vespalib::string fname(_name); - fname += tmp; - return fname; -} - -template <typename T, typename S> -void MergeSorter<T, S>::merge() -{ - S sorter; - std::vector< SequentialFileArrayRead<T> *> fileParts; - size_t count(0); - for(size_t i(0); i < _chunkCount; i++) { - std::unique_ptr< SequentialFileArrayRead<T> > part(new SequentialFileArrayRead<T>(genName(i))); - size_t sz = part->size(); - if (sz > 0) { - fileParts.push_back(part.release()); - } else { - part->erase(); - } - count += sz; - } - - std::vector<T> cachedValue; - for(size_t i(0), m(fileParts.size()); i < m; i++) { - cachedValue.push_back(fileParts[i]->getNext()); - } - SequentialFileArrayWrite<T> merged(_name); - for(size_t j(0); j < count; j++) { - size_t firstIndex(0); - for(size_t i(1), m(cachedValue.size()); i < m; i++) { - if (sorter.cmp(cachedValue[i], cachedValue[firstIndex])) { - firstIndex = i; - } - } - merged.push_back(cachedValue[firstIndex]); - if ( ! fileParts[firstIndex]->hasNext() ) { - fileParts[firstIndex]->erase(); - delete fileParts[firstIndex]; - fileParts.erase(fileParts.begin()+firstIndex); - cachedValue.erase(cachedValue.begin()+firstIndex); - } else { - cachedValue[firstIndex] = fileParts[firstIndex]->getNext(); - } - } -} - -template <typename T, typename S> -void MergeSorter<T, S>::sortChunk() -{ - S sorter; - sorter.sort(&_chunk[0], _chunk.size()); - FastOS_File chunkFile(genName(_chunkCount).c_str()); - chunkFile.EnableDirectIO(); - if (chunkFile.OpenWriteOnlyTruncate()) { - chunkFile.CheckedWrite(&_chunk[0], _chunk.size()*sizeof(_chunk[0])); - } - chunkFile.Close(); - _chunkCount++; - _chunk.clear(); -} - -template <typename T> SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector() : Vector(), _rp(0), @@ -137,17 +20,6 @@ SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector(size_t sz) { } template <typename T> -SequentialReadModifyWriteVector<T>::~SequentialReadModifyWriteVector() { } - -template <typename T, typename R, typename W> -SequentialReaderWriter<T, R, W>::SequentialReaderWriter(R & reader, W & writer) - : _reader(reader), - _writer(writer) -{ - next(); -} - -template <typename T, typename R, typename W> -SequentialReaderWriter<T, R, W>::~SequentialReaderWriter() { } +SequentialReadModifyWriteVector<T>::~SequentialReadModifyWriteVector() = default; } diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp index 01c4d4e785e..116e400083a 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ b/searchlib/src/vespa/searchlib/util/stringenum.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "stringenum.h" -#include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/stllike/hashtable.hpp> #include <cassert> @@ -10,28 +9,6 @@ LOG_SETUP(".seachlib.util.stringenum"); namespace search::util { -static inline char * -StripString(char *str) -{ - char *first = NULL; // first non-space char - char *last = NULL; // last non-space char - - if (str == NULL) - return NULL; - - for (; *str != '\0' && isspace(*str); str++); - first = str; - - for (; *str != '\0'; str++) - if (!isspace(*str)) - last = str; - - if (last != NULL) - *(last + 1) = '\0'; - - return first; -} - StringEnum::StringEnum() : _numEntries(0), _mapping(), @@ -56,80 +33,6 @@ StringEnum::CreateReverseMapping() const } } - -bool -StringEnum::Save(const char *filename) -{ - char str[1024]; - - Fast_BufferedFile file; - file.WriteOpen(filename); - if (!file.IsOpened()) - return false; - - file.SetSize(0); - sprintf(str, "%d\n", _numEntries); - file.WriteString(str); - - for (uint32_t i = 0; i < _numEntries; i++) { - file.WriteString(Lookup(i)); - file.WriteString("\n"); - } - - file.Close(); - return true; -} - - -bool -StringEnum::Load(const char *filename) -{ - char line[1024]; - char *pt; - uint32_t entries; // from first line of file - uint32_t lineNumber; // current line in file - uint32_t entryCnt; // # entries obtained from file - - Clear(); - - Fast_BufferedFile file; - if (!file.OpenReadOnly(filename)) - return false; - - lineNumber = 0; - entryCnt = 0; - - pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == NULL || *pt == '\0') - return false; - lineNumber++; - - entries = atoi(pt); - - while (!file.Eof()) { - pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == NULL) // end of input ? - break; - lineNumber++; - if (*pt == '\0') // empty line ? - continue; - - uint32_t tmp = _numEntries; - if (static_cast<uint32_t>(Add(pt)) != tmp) { - LOG(error, "(%s:%d) duplicate enum entry: %s", filename, lineNumber, pt); - } - entryCnt++; - } - - file.Close(); - if (entries != _numEntries - || entries != entryCnt) { - Clear(); - return false; - } - return true; -} - void StringEnum::Clear() { @@ -162,7 +65,7 @@ const char * StringEnum::Lookup(uint32_t value) const { if (value >= _numEntries) - return NULL; + return nullptr; if (_numEntries > _reverseMap.size()) CreateReverseMapping(); diff --git a/searchlib/src/vespa/searchlib/util/stringenum.h b/searchlib/src/vespa/searchlib/util/stringenum.h index 85f97de48ad..0da79db323a 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.h +++ b/searchlib/src/vespa/searchlib/util/stringenum.h @@ -80,25 +80,6 @@ public: * @return current number of entries. **/ uint32_t GetNumEntries() const { return _numEntries; } - - - /** - * Save the enumeration currently held by this object to file. - * - * @return success(true)/fail(false). - * @param filename name of save file. - **/ - bool Save(const char *filename); - - - /** - * Load an enumeration from file. The loaded enumeration will - * replace the one currently held by this object. - * - * @return success(true)/fail(false). - * @param filename name of file to load. - **/ - bool Load(const char *filename); }; } diff --git a/searchsummary/src/tests/docsumformat/docsum-pack.cpp b/searchsummary/src/tests/docsumformat/docsum-pack.cpp index 240abe6bae7..a80c030ef6e 100644 --- a/searchsummary/src/tests/docsumformat/docsum-pack.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-pack.cpp @@ -46,7 +46,6 @@ public: void TestFailShort(); void TestFailOrder(); void TestBasicInplace(); - void TestCompressInplace(); int Main() override; }; @@ -337,60 +336,6 @@ MyApp::TestBasicInplace() delete gres; } - -void -MyApp::TestCompressInplace() -{ - const char *buf; - uint32_t buflen; - - search::RawBuf field1(32_Ki); - search::RawBuf field2(32_Ki); - const ResultClass *resClass; - GeneralResult *gres; - - const char *lstrval = "string string string"; - const char *ldatval = "data data data"; - - RTR(__LINE__, _packer.Init(2)); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, _packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - - resClass = _config.LookupResultClass(_config.GetClassID(buf, buflen)); - if (resClass == nullptr) { - gres = nullptr; - } else { - DocsumStoreValue value(buf, buflen); - gres = new GeneralResult(resClass); - if (!gres->inplaceUnpack(value)) { - delete gres; - gres = nullptr; - } - } - - ResEntry *e1 = (gres == nullptr) ? nullptr : gres->GetEntry("text"); - ResEntry *e2 = (gres == nullptr) ? nullptr : gres->GetEntry("data"); - - if (e1 != nullptr) - e1->_extract_field(&field1); - if (e2 != nullptr) - e2->_extract_field(&field2); - - RTR(__LINE__, gres != nullptr); - RTR(__LINE__, e1 != nullptr); - RTR(__LINE__, e2 != nullptr); - RTR(__LINE__, strcmp(field1.GetDrainPos(), lstrval) == 0); - RTR(__LINE__, strcmp(field2.GetDrainPos(), ldatval) == 0); - RTR(__LINE__, strlen(lstrval) == field1.GetUsedLen()); - RTR(__LINE__, strlen(ldatval) == field2.GetUsedLen()); - RTR(__LINE__, (gres != nullptr && - gres->GetClass()->GetNumEntries() == 2)); - RTR(__LINE__, (gres != nullptr && - gres->GetClass()->GetClassID() == 2)); - delete gres; -} - int MyApp::Main() { @@ -423,7 +368,6 @@ MyApp::Main() TestFailShort(); TestFailOrder(); TestBasicInplace(); - TestCompressInplace(); LOG(info, "CONCLUSION: %s", (_rc) ? "SUCCESS" : "FAIL"); return (_rc ? 0 : 1); diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 52a363c9888..b9469ca2102 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -131,8 +131,7 @@ void checkWritePositionField(Test &test, AttrType &attr, } MyAttributeManager attribute_man(attr); - PositionsDFW::UP writer = - createPositionsDFW(attr.getName().c_str(), &attribute_man); + PositionsDFW::UP writer = PositionsDFW::create(attr.getName().c_str(), &attribute_man, false); ASSERT_TRUE(writer.get()); ResType res_type = RES_JSONSTRING; MyGetDocsumsStateCallback callback; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index a1bc690e042..f55a13a9604 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -72,19 +72,19 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } else if (overrideName == "absdist") { if (getEnvironment()) { IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = createAbsDistanceDFW(argument.c_str(), am); + fieldWriter = AbsDistanceDFW::create(argument.c_str(), am); rc = fieldWriter.get(); } } else if (overrideName == "positions") { if (getEnvironment()) { IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = createPositionsDFW(argument.c_str(), am); + fieldWriter = PositionsDFW::create(argument.c_str(), am, resultConfig.useV8geoPositions()); rc = fieldWriter.get(); } } else if (overrideName == "geopos") { if (getEnvironment()) { IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = GeoPositionDFW::create(argument.c_str(), am); + fieldWriter = GeoPositionDFW::create(argument.c_str(), am, resultConfig.useV8geoPositions()); rc = fieldWriter.get(); } } else if (overrideName == "attribute") { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index fada441f718..47d94a716f7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -78,7 +78,7 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) } void -CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *state, ResType type, +CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *, ResType type, vespalib::slime::Inserter &target) { int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); @@ -128,7 +128,7 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *s uint32_t len; const char *spt; // resolve field - entry->_resolve_field(&spt, &len, &state->_docSumFieldSpace); + entry->_resolve_field(&spt, &len); vespalib::Memory value(spt, len); target.insertString(value); break; } @@ -139,7 +139,7 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *s uint32_t len; const char *dpt; // resolve field - entry->_resolve_field(&dpt, &len, &state->_docSumFieldSpace); + entry->_resolve_field(&dpt, &len); vespalib::Memory value(dpt, len); target.insertData(value); break; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 31d174c72a8..4f1634042c1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -30,8 +30,6 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _keywords(nullptr), _callback(callback), _dynteaser(), - _docSumFieldSpaceStore(), - _docSumFieldSpace(_docSumFieldSpaceStore, sizeof(_docSumFieldSpaceStore)), // only alloc buffer if needed _attrCtx(), _attributes(), _fieldWriterStates(), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index cbb6925c1e0..85d24d25c62 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -71,8 +71,6 @@ public: } _dynteaser; - char _docSumFieldSpaceStore[2048]; - search::RawBuf _docSumFieldSpace; std::unique_ptr<search::attribute::IAttributeContext> _attrCtx; std::vector<const search::attribute::IAttributeVector *> _attributes; std::vector<std::unique_ptr<DocsumFieldWriterState>> _fieldWriterStates; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 666c5098ca4..e7a9840c47f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -84,8 +84,7 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const constexpr uint32_t default_32bits_int = search::attribute::getUndefined<int32_t>(); constexpr uint64_t default_64bits_int = search::attribute::getUndefined<int64_t>(); -static void convertEntry(GetDocsumsState *state, - const ResConfigEntry *resCfg, +static void convertEntry(const ResConfigEntry *resCfg, const ResEntry *entry, Inserter &inserter, Slime &slime) @@ -120,7 +119,7 @@ static void convertEntry(GetDocsumsState *state, case RES_STRING: case RES_LONG_STRING: case RES_FEATUREDATA: - entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); + entry->_resolve_field(&ptr, &len); if (len != 0) { inserter.insertString(Memory(ptr, len)); } @@ -128,13 +127,13 @@ static void convertEntry(GetDocsumsState *state, case RES_DATA: case RES_TENSOR: case RES_LONG_DATA: - entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); + entry->_resolve_field(&ptr, &len); if (len != 0) { inserter.insertData(Memory(ptr, len)); } break; case RES_JSONSTRING: - entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); + entry->_resolve_field(&ptr, &len); if (len != 0) { // note: 'JSONSTRING' really means 'structured data' size_t d = BinaryFormat::decode_into(Memory(ptr, len), slime, inserter); @@ -185,7 +184,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, } } else { if (rci.inputClass == rci.outputClass) { - convertEntry(state, outCfg, gres.GetEntry(i), inserter, slime); + convertEntry(outCfg, gres.GetEntry(i), inserter, slime); } else { int inIdx = rci.inputClass->GetIndexFromEnumValue(outCfg->_enumValue); const ResConfigEntry *inCfg = rci.inputClass->GetEntry(inIdx); @@ -193,7 +192,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, // copy field const ResEntry *entry = gres.GetEntry(inIdx); LOG_ASSERT(entry != nullptr); - convertEntry(state, outCfg, entry, inserter, slime); + convertEntry(outCfg, entry, inserter, slime); } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index f9301557c0c..ef1ffded941 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -345,13 +345,13 @@ JuniperTeaserDFW::Init( } vespalib::stringref -DynamicTeaserDFW::getJuniperInput(GeneralResult *gres, GetDocsumsState *state) { +DynamicTeaserDFW::getJuniperInput(GeneralResult *gres) { int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); ResEntry *entry = gres->GetEntry(idx); if (entry != nullptr) { const char *buf; uint32_t buflen; - entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); + entry->_resolve_field(&buf, &buflen); return vespalib::stringref(buf, buflen); } return vespalib::stringref(); @@ -428,7 +428,7 @@ void DynamicTeaserDFW::insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { - vespalib::stringref input = getJuniperInput(gres, state); + vespalib::stringref input = getJuniperInput(gres); if (input.length() > 0) { vespalib::string teaser = makeDynamicTeaser(docid, input, state); vespalib::Memory value(teaser.c_str(), teaser.size()); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index ceb358e5859..be6664e41a3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -4,7 +4,6 @@ #include "resultconfig.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/datatype.h> -#include <zlib.h> #include <cassert> #include <vespa/log/log.h> @@ -264,9 +263,13 @@ GeneralResult::unpack(const char *buf, const size_t buflen) memcpy(&flen, p, sizeof(flen)); p += sizeof(flen); lslen = flen & 0x7fffffff; + if (lslen != flen) { + LOG(error, "GeneralResult::_inplace_unpack: compressed data"); + rc = false; + } if (p + lslen <= ebuf) { _entries[i]._stringval = const_cast<char *>(p); - _entries[i]._stringlen = flen; // with compression flag + _entries[i]._stringlen = lslen; _entries[i]._type = RES_STRING; // type normalization p += lslen; } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 1f73872af6e..8f627ac1b9a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -19,14 +19,19 @@ using attribute::IAttributeVector; using attribute::IAttributeContext; using vespalib::Issue; -GeoPositionDFW::GeoPositionDFW(const vespalib::string & attrName) : - AttrDFW(attrName) +GeoPositionDFW::GeoPositionDFW(const vespalib::string & attrName, bool useV8geoPositions) : + AttrDFW(attrName), + _useV8geoPositions(useV8geoPositions) { } namespace { -void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target) -{ +double to_degrees(int32_t microDegrees) { + double d = microDegrees / 1.0e6; + return d; +} + +void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target, bool useV8geoPositions) { int32_t docx = 0; int32_t docy = 0; vespalib::geo::ZCurve::decode(zval, &docx, &docy); @@ -34,8 +39,15 @@ void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target) LOG(spam, "skipping empty zcurve value"); } else { vespalib::slime::Cursor &obj = target.insertObject(); - obj.setLong("y", docy); - obj.setLong("x", docx); + if (useV8geoPositions) { + double degrees_ns = to_degrees(docy); + double degrees_ew = to_degrees(docx); + obj.setDouble("lat", degrees_ns); + obj.setDouble("lng", degrees_ew); + } else { + obj.setLong("y", docy); + obj.setLong("x", docx); + } } } @@ -52,6 +64,7 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, const auto& attribute = get_attribute(*dsState); if (attribute.hasMultiValue()) { uint32_t entries = attribute.getValueCount(docid); + if (entries == 0 && _useV8geoPositions) return; Cursor &arr = target.insertArray(); if (attribute.hasWeightedSetType()) { Symbol isym = arr.resolve("item"); @@ -62,7 +75,7 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, Cursor &elem = arr.addObject(); int64_t pos = elements[i].getValue(); ObjectSymbolInserter obj(elem, isym); - fmtZcurve(pos, obj); + fmtZcurve(pos, obj, _useV8geoPositions); elem.setLong(wsym, elements[i].getWeight()); } } else { @@ -76,18 +89,19 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, for (uint32_t i = 0; i < numValues; i++) { int64_t pos = elements[i]; ArrayInserter obj(arr); - fmtZcurve(pos, obj); + fmtZcurve(pos, obj, _useV8geoPositions); } } } else { int64_t pos = attribute.getInt(docid); - fmtZcurve(pos, target); + fmtZcurve(pos, target, _useV8geoPositions); } } GeoPositionDFW::UP GeoPositionDFW::create(const char *attribute_name, - IAttributeManager *attribute_manager) + IAttributeManager *attribute_manager, + bool useV8geoPositions) { GeoPositionDFW::UP ret; if (attribute_manager != nullptr) { @@ -106,8 +120,7 @@ GeoPositionDFW::create(const char *attribute_name, return ret; } } - ret.reset(new GeoPositionDFW(attribute_name)); - return ret; + return std::make_unique<GeoPositionDFW>(attribute_name, useV8geoPositions); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h index 020e864d902..5dcee853304 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h @@ -11,11 +11,13 @@ namespace search::docsummary { **/ class GeoPositionDFW : public AttrDFW { +private: + bool _useV8geoPositions; public: typedef std::unique_ptr<GeoPositionDFW> UP; - GeoPositionDFW(const vespalib::string & attrName); + GeoPositionDFW(const vespalib::string & attrName, bool useV8geoPositions); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; - static UP create(const char *attribute_name, IAttributeManager *attribute_manager); + static UP create(const char *attribute_name, IAttributeManager *attribute_manager, bool useV8geoPositions); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index f9a6a56e01a..d9a657038c4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -49,7 +49,7 @@ class DynamicTeaserDFW : public JuniperTeaserDFW public: DynamicTeaserDFW(juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } - vespalib::stringref getJuniperInput(GeneralResult *gres, GetDocsumsState *state); + vespalib::stringref getJuniperInput(GeneralResult *gres); vespalib::string makeDynamicTeaser(uint32_t docid, vespalib::stringref input, GetDocsumsState *state); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index 916f39a2b98..2c2ce860e39 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -65,11 +65,11 @@ MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; namespace { void -decode_input_field_to_slime(const ResEntry& entry, search::RawBuf& target_buf, Slime& input_field_as_slime) +decode_input_field_to_slime(const ResEntry& entry, Slime& input_field_as_slime) { const char* buf; uint32_t buf_len; - entry._resolve_field(&buf, &buf_len, &target_buf); + entry._resolve_field(&buf, &buf_len); BinaryFormat::decode(vespalib::Memory(buf, buf_len), input_field_as_slime); } @@ -91,12 +91,11 @@ filter_matching_elements_in_input_field_while_converting_to_slime(const FieldVal } bool -resolve_input_field_as_slime(GeneralResult& result, GetDocsumsState& state, - int entry_idx, Slime& input_field_as_slime) +resolve_input_field_as_slime(GeneralResult& result, int entry_idx, Slime& input_field_as_slime) { ResEntry* entry = result.GetEntry(entry_idx); if (entry != nullptr) { - decode_input_field_to_slime(*entry, state._docSumFieldSpace, input_field_as_slime); + decode_input_field_to_slime(*entry, input_field_as_slime); return true; } return false; @@ -127,7 +126,7 @@ MatchedElementsFilterDFW::insertField(uint32_t docid, GeneralResult* result, Get assert(type == ResType::RES_JSONSTRING); int entry_idx = result->GetClass()->GetIndexFromEnumValue(_input_field_enum); Slime input_field; - if (resolve_input_field_as_slime(*result, *state, entry_idx, input_field)) { + if (resolve_input_field_as_slime(*result, entry_idx, input_field)) { Slime output_field; filter_matching_elements_in_input_field(input_field, get_matching_elements(docid, *state), output_field); inject(output_field.get(), target); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index ce14069a135..1a2cf24a18d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -4,6 +4,7 @@ #include "docsumstate.h" #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/common/geo_gcd.h> #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/slime/cursor.h> @@ -16,11 +17,21 @@ LOG_SETUP(".searchlib.docsummary.positionsdfw"); namespace search::docsummary { +namespace { + +double to_degrees(int32_t microDegrees) { + double d = microDegrees / 1.0e6; + return d; +} + +} + using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::attribute::BasicType; using search::attribute::IntegerContent; using search::common::Location; +using search::common::GeoGcd; LocationAttrDFW::AllLocations LocationAttrDFW::getAllLocations(GetDocsumsState *state) @@ -50,10 +61,39 @@ LocationAttrDFW::getAllLocations(GetDocsumsState *state) return retval; } -AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) : - LocationAttrDFW(attrName) +AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) + : LocationAttrDFW(attrName) { } +double +AbsDistanceDFW::kmMinDistance(uint32_t docid, GetDocsumsState *state, + const std::vector<const GeoLoc *> &locations) +{ + double best = std::numeric_limits<double>::max(); + const auto& attribute = get_attribute(*state); + for (auto location : locations) { + double lat = to_degrees(location->point.y); + double lng = to_degrees(location->point.x); + GeoGcd point{lat, lng}; + int32_t docx = 0; + int32_t docy = 0; + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); + for (uint32_t i = 0; i < numValues; i++) { + int64_t docxy(pos[i]); + vespalib::geo::ZCurve::decode(docxy, &docx, &docy); + lat = to_degrees(docy); + lng = to_degrees(docx); + double dist = point.km_great_circle_distance(lat, lng); + if (dist < best) { + best = dist; + } + } + } + return best; +} + uint64_t AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state, const std::vector<const GeoLoc *> &locations) @@ -105,8 +145,9 @@ AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type //-------------------------------------------------------------------------- -PositionsDFW::PositionsDFW(const vespalib::string & attrName) : - AttrDFW(attrName) +PositionsDFW::PositionsDFW(const vespalib::string & attrName, bool useV8geoPositions) : + AttrDFW(attrName), + _useV8geoPositions(useV8geoPositions) { } @@ -127,10 +168,8 @@ insertPos(int64_t docxy, vespalib::slime::Inserter &target) obj.setLong("y", docy); obj.setLong("x", docx); - double degrees_ns = docy; - degrees_ns /= 1000000.0; - double degrees_ew = docx; - degrees_ew /= 1000000.0; + double degrees_ns = to_degrees(docy); + double degrees_ew = to_degrees(docx); vespalib::asciistream latlong; latlong << vespalib::FloatSpec::fixed; @@ -176,19 +215,70 @@ void checkExpected(ResType type) { LOG(error, "Unexpected summary field type %s", ResultConfig::GetResTypeName(type)); } +void insertPosV8(int64_t docxy, vespalib::slime::Inserter &target) { + int32_t docx = 0; + int32_t docy = 0; + vespalib::geo::ZCurve::decode(docxy, &docx, &docy); + if (docx == 0 && docy == INT_MIN) { + LOG(spam, "skipping empty zcurve value"); + return; + } + double degrees_ns = to_degrees(docy); + double degrees_ew = to_degrees(docx); + vespalib::slime::Cursor &obj = target.insertObject(); + obj.setDouble("lat", degrees_ns); + obj.setDouble("lng", degrees_ew); + vespalib::asciistream latlong; + latlong << vespalib::FloatSpec::fixed; + if (degrees_ns < 0) { + latlong << "S" << (-degrees_ns); + } else { + latlong << "N" << degrees_ns; + } + latlong << ";"; + if (degrees_ew < 0) { + latlong << "W" << (-degrees_ew); + } else { + latlong << "E" << degrees_ew; + } + obj.setString("latlong", vespalib::Memory(latlong.str())); +} + + +void insertV8FromAttr(const attribute::IAttributeVector &attribute, uint32_t docid, vespalib::slime::Inserter &target) { + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); + LOG(debug, "docid=%d, numValues=%d", docid, numValues); + if (numValues > 0) { + if (attribute.getCollectionType() == attribute::CollectionType::SINGLE) { + insertPosV8(pos[0], target); + } else { + vespalib::slime::Cursor &arr = target.insertArray(); + for (uint32_t i = 0; i < numValues; i++) { + vespalib::slime::ArrayInserter ai(arr); + insertPosV8(pos[i], ai); + } + } + } +} + } // namespace void PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { checkExpected(type); - insertFromAttr(get_attribute(*dsState), docid, target); + if (_useV8geoPositions) { + insertV8FromAttr(get_attribute(*dsState), docid, target); + } else { + insertFromAttr(get_attribute(*dsState), docid, target); + } } //-------------------------------------------------------------------------- -PositionsDFW::UP createPositionsDFW(const char *attribute_name, IAttributeManager *attribute_manager) -{ +PositionsDFW::UP PositionsDFW::create(const char *attribute_name, IAttributeManager *attribute_manager, bool useV8geoPositions) { PositionsDFW::UP ret; if (attribute_manager != nullptr) { if (!attribute_name) { @@ -206,11 +296,10 @@ PositionsDFW::UP createPositionsDFW(const char *attribute_name, IAttributeManage return ret; } } - return std::make_unique<PositionsDFW>(attribute_name); + return std::make_unique<PositionsDFW>(attribute_name, useV8geoPositions); } -AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, IAttributeManager *attribute_manager) -{ +AbsDistanceDFW::UP AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { AbsDistanceDFW::UP ret; if (attribute_manager != nullptr) { if (!attribute_name) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index eeab2f33b5b..7f6bfe22816 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -3,6 +3,7 @@ #pragma once #include "attributedfw.h" +#include "resultconfig.h" #include <vespa/searchlib/common/geo_location_spec.h> namespace search::docsummary { @@ -35,6 +36,8 @@ public: class AbsDistanceDFW : public LocationAttrDFW { private: + double kmMinDistance(uint32_t docid, GetDocsumsState *state, + const std::vector<const GeoLoc *> &locations); uint64_t findMinDistance(uint32_t docid, GetDocsumsState *state, const std::vector<const GeoLoc *> &locations); public: @@ -43,22 +46,24 @@ public: bool IsGenerated() const override { return true; } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; + + static UP create(const char *attribute_name, IAttributeManager *index_man); + }; //-------------------------------------------------------------------------- class PositionsDFW : public AttrDFW { +private: + bool _useV8geoPositions; public: typedef std::unique_ptr<PositionsDFW> UP; - - PositionsDFW(const vespalib::string & attrName); - + PositionsDFW(const vespalib::string & attrName, bool useV8geoPositions); bool IsGenerated() const override { return true; } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; + static UP create(const char *attribute_name, IAttributeManager *index_man, bool useV8geoPositions); }; -PositionsDFW::UP createPositionsDFW(const char *attribute_name, IAttributeManager *index_man); -AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, IAttributeManager *index_man); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index da0106cd7ad..37be8e0a1b2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -4,7 +4,6 @@ #include "resultconfig.h" #include <vespa/vespalib/stllike/hashtable.hpp> #include <cassert> -#include <zlib.h> namespace search::docsummary { @@ -45,7 +44,6 @@ ResultClass::AddConfigEntry(const char *name, ResType type) return true; } - void ResultClass::CreateEnumMap() { @@ -59,52 +57,4 @@ ResultClass::CreateEnumMap() } } - -bool -ResEntry::_extract_field(search::RawBuf *target) const -{ - bool rc = true; - target->reset(); - - if (ResultConfig::IsVariableSize(_type)) { - if (_is_compressed()) { // COMPRESSED - - uint32_t len = _get_length(); - uint32_t realLen = 0; - - if (len >= sizeof(uint32_t)) - realLen = _get_real_length(); - else - rc = false; - - if (realLen > 0) { - uLongf rlen = realLen; - char *fillPos = target->GetWritableFillPos(realLen + 1 < 32000 ? - 32000 : realLen + 1); - if ((uncompress((Bytef *)fillPos, &rlen, - (const Bytef *)(_get_compressed()), - len - sizeof(realLen)) == Z_OK) && - rlen == realLen) { - fillPos[realLen] = '\0'; - target->Fill(realLen); - } else { - rc = false; - } - } - } else { // UNCOMPRESSED - uint32_t len = _len; - if (len + 1 < 32000) - target->preAlloc(32000); - else - target->preAlloc(len + 1); - char *fillPos = target->GetWritableFillPos(len + 1 < 32000 ? - 32000 : len + 1); - memcpy(fillPos, _pt, len); - fillPos[len] = '\0'; - target->Fill(len); - } - } - return rc; -} - } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 858fdea2404..be3bb7570d2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -71,10 +71,7 @@ struct ResEntry void *_pt; }; - bool _extract_field(search::RawBuf *target) const; - - uint32_t _get_length() const { return (_len & 0x7fffffff); } - bool _is_compressed() const { return (_len & 0x80000000) != 0; } + uint32_t _get_length() const { return _len; } uint32_t _get_real_length() const { // precond: IsVariableSize(_type) && _len >= sizeof(uint32_t) @@ -83,29 +80,11 @@ struct ResEntry memcpy(&rlen, _pt, sizeof(rlen)); return rlen; } - const void *_get_compressed() const - { - // precond: IsVariableSize(_type) && _len >= sizeof(uint32_t) - - return (const void *)(((const char *) _pt) + sizeof(uint32_t)); - } - void _resolve_field(const char **buf, uint32_t *buflen, - search::RawBuf *target) const + void _resolve_field(const char **buf, uint32_t *buflen) const { // precond: IsVariableSize(_type) - - if (_is_compressed()) { - if (_extract_field(target)) { - *buf = target->GetDrainPos(); - *buflen = target->GetUsedLen(); - } else { - *buf = NULL; - *buflen = 0; - } - } else { - *buf = (char *) _pt; - *buflen = _len; - } + *buf = (char *) _pt; + *buflen = _len; } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index d3c0caeec48..02bb5d25ca4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -3,6 +3,7 @@ #include "resultconfig.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <atomic> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.resultconfig"); @@ -117,6 +118,13 @@ ResultConfig::CreateEnumMaps() } } +namespace { +std::atomic<bool> global_useV8geoPositions = false; +} + +bool ResultConfig::wantedV8geoPositions() { + return global_useV8geoPositions; +} bool ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const char *configId) @@ -126,6 +134,7 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const int maxclassID = 0x7fffffff; // avoid negative classids _defaultSummaryId = cfg.defaultsummaryid; _useV8geoPositions = cfg.usev8geopositions; + global_useV8geoPositions = cfg.usev8geopositions; for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) { const auto& cfg_class = cfg.classes[i]; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 8a8bfabaaec..8c5895a779d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -173,6 +173,8 @@ public: return false; } + // whether last config seen wanted useV8geoPositions = true + static bool wantedV8geoPositions(); /** * @return the name of the given result field type. diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 31b953362dd..597cfe8eb40 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -2,12 +2,14 @@ #include "summaryfieldconverter.h" #include "linguisticsannotation.h" +#include "resultconfig.h" #include "searchdatatype.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/annotation/annotation.h> #include <vespa/document/annotation/spantree.h> #include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/boolfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> @@ -472,6 +474,18 @@ private: } void visit(const StructFieldValue &value) override { + if (value.getDataType() == &document::PositionDataType::getInstance() + && ResultConfig::wantedV8geoPositions()) + { + auto xv = value.getValue("x"); + auto yv = value.getValue("y"); + if (xv && yv) { + Cursor &c = _inserter.insertObject(); + c.setDouble("lat", double(yv->getAsInt()) / 1.0e6); + c.setDouble("lng", double(xv->getAsInt()) / 1.0e6); + return; + } + } if (*value.getDataType() == *SearchDataType::URI) { FieldValue::UP uriAllValue = value.getValue("all"); if (uriAllValue && diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp index c7c6984d792..c85a4fb5788 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp @@ -27,7 +27,7 @@ TextExtractorDFW::init(const vespalib::string & fieldName, const vespalib::strin } void -TextExtractorDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *state, ResType, +TextExtractorDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) { vespalib::string extracted; @@ -35,7 +35,7 @@ TextExtractorDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *st if (entry != nullptr) { const char * buf = nullptr; uint32_t buflen = 0; - entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); + entry->_resolve_field(&buf, &buflen); // extract the text Tokenizer tokenizer(buf, buflen); while (tokenizer.hasMoreTokens()) { diff --git a/security-tools/pom.xml b/security-tools/pom.xml index 7f5d22b4b2b..f4511110fb7 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -49,18 +49,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - <exclude>META-INF/versions/*/module-info.class</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java deleted file mode 100644 index e522b736290..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.duper; - -import com.yahoo.vespa.applicationmodel.InfrastructureApplication; - -/** - * @author mortent - */ -public class DevHostApplication extends HostAdminApplication { - public DevHostApplication() { - super(InfrastructureApplication.DEV_HOST); - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 67d54091adc..9215581a2b9 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -9,8 +9,6 @@ import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -45,7 +43,6 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); static final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); static final TenantHostApplication tenantHostApplication = new TenantHostApplication(); - static final DevHostApplication devHostApplication = new DevHostApplication(); private final Map<ApplicationId, InfraApplication> supportedInfraApplications; @@ -61,23 +58,18 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi private boolean infraApplicationsIsComplete = false; @Inject - public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { + public DuperModelManager(ConfigserverConfig configServerConfig, SuperModelProvider superModelProvider) { this(configServerConfig.multitenant(), configServerConfig.serverNodeType() == ConfigserverConfig.ServerNodeType.Enum.controller, - superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system())); + superModelProvider, new DuperModel()); } /** Non-private for testing */ public DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, - DuperModel duperModel, FlagSource flagSource, SystemName system) { + DuperModel duperModel) { this.duperModel = duperModel; - if (system == SystemName.dev) { - // TODO (mortent): Support controllerApplication in dev system - supportedInfraApplications = - Stream.of(devHostApplication, configServerApplication) - .collect(Collectors.toUnmodifiableMap(InfraApplication::getApplicationId, Function.identity())); - } else if (multitenant) { + if (multitenant) { supportedInfraApplications = (isController ? Stream.of(controllerHostApplication, controllerApplication) : diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index 8b0cc621358..a3af44ec911 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -33,13 +33,12 @@ public class DuperModelManagerTest { private final SuperModelProvider superModelProvider = mock(SuperModelProvider.class); private final SuperModel superModel = mock(SuperModel.class); private final DuperModel duperModel = mock(DuperModel.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private DuperModelManager manager; private SuperModelListener superModelListener; private void makeManager(boolean isController) { - manager = new DuperModelManager(true, isController, superModelProvider, duperModel, flagSource, SystemName.cd); + manager = new DuperModelManager(true, isController, superModelProvider, duperModel); when(superModelProvider.getSuperModel()).thenReturn(superModel); verify(duperModel, times(0)).add(any()); diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index 77705864f80..295906fe8c1 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -4,7 +4,6 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> -#include <vespa/config/config.h> #include <vespa/config/common/configcontext.h> #include <vespa/config-slobroks.h> #include <vespa/fnet/transport.h> diff --git a/slobrok/src/vespa/slobrok/cfg.cpp b/slobrok/src/vespa/slobrok/cfg.cpp index 557e94d56d8..12e41bac4fc 100644 --- a/slobrok/src/vespa/slobrok/cfg.cpp +++ b/slobrok/src/vespa/slobrok/cfg.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cfg.h" +#include <vespa/config/subscription/configsubscriber.hpp> namespace slobrok { @@ -21,7 +22,7 @@ extract(const cloud::config::SlobroksConfig &cfg) bool Configurator::poll() { - bool retval = _subscriber.nextGenerationNow(); + bool retval = _subscriber->nextGenerationNow(); if (retval) { std::unique_ptr<cloud::config::SlobroksConfig> cfg = _handle->getConfig(); _target.setup(extract(*cfg)); @@ -31,12 +32,21 @@ Configurator::poll() Configurator::Configurator(Configurable& target, const config::ConfigUri & uri) - : _subscriber(uri.getContext()), - _handle(_subscriber.subscribe<cloud::config::SlobroksConfig>(uri.getConfigId())), + : _subscriber(std::make_unique<config::ConfigSubscriber>(uri.getContext())), + _handle(_subscriber->subscribe<cloud::config::SlobroksConfig>(uri.getConfigId())), _target(target) { } +Configurator::~Configurator() = default; + + +int64_t +Configurator::getGeneration() const { + return _subscriber->getGeneration(); +} + + ConfiguratorFactory::ConfiguratorFactory(const config::ConfigUri& uri) : _uri(uri) { diff --git a/slobrok/src/vespa/slobrok/cfg.h b/slobrok/src/vespa/slobrok/cfg.h index d3150a6f941..bd7bffe1d15 100644 --- a/slobrok/src/vespa/slobrok/cfg.h +++ b/slobrok/src/vespa/slobrok/cfg.h @@ -3,8 +3,12 @@ #include <vespa/vespalib/util/ptrholder.h> #include <vespa/config-slobroks.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/subscription/confighandle.h> +namespace config { + class ConfigSubscriber; +} namespace slobrok { class Configurable { @@ -16,15 +20,16 @@ public: class Configurator { private: - config::ConfigSubscriber _subscriber; - config::ConfigHandle<cloud::config::SlobroksConfig>::UP _handle; + std::unique_ptr<config::ConfigSubscriber> _subscriber; + std::unique_ptr<config::ConfigHandle<cloud::config::SlobroksConfig>> _handle; Configurable &_target; public: Configurator(Configurable &target, const config::ConfigUri & uri); + ~Configurator(); bool poll(); typedef std::unique_ptr<Configurator> UP; - int64_t getGeneration() const { return _subscriber.getGeneration(); } + int64_t getGeneration() const; }; class ConfiguratorFactory { diff --git a/slobrok/src/vespa/slobrok/server/configshim.cpp b/slobrok/src/vespa/slobrok/server/configshim.cpp index d6922685009..cc4892e616c 100644 --- a/slobrok/src/vespa/slobrok/server/configshim.cpp +++ b/slobrok/src/vespa/slobrok/server/configshim.cpp @@ -18,11 +18,11 @@ ConfigShim::ConfigShim(uint32_t port, const std::string& cfgId) _factory(config::ConfigUri(_configId)) {} -ConfigShim::ConfigShim(uint32_t port, const std::string& cfgId, config::IConfigContext::SP cfgCtx) +ConfigShim::ConfigShim(uint32_t port, const std::string& cfgId, std::shared_ptr<config::IConfigContext> cfgCtx) : _port(port), _enableStateServer(false), _configId(cfgId), - _factory(config::ConfigUri(cfgId, cfgCtx)) + _factory(config::ConfigUri(cfgId, std::move(cfgCtx))) {} ConfigShim::~ConfigShim() = default; diff --git a/slobrok/src/vespa/slobrok/server/configshim.h b/slobrok/src/vespa/slobrok/server/configshim.h index aebbe2dece4..f31eeb5c463 100644 --- a/slobrok/src/vespa/slobrok/server/configshim.h +++ b/slobrok/src/vespa/slobrok/server/configshim.h @@ -17,7 +17,7 @@ private: public: ConfigShim(uint32_t port); ConfigShim(uint32_t port, const std::string& cfgId); - ConfigShim(uint32_t port, const std::string& cfgId, config::IConfigContext::SP cfgCtx); + ConfigShim(uint32_t port, const std::string& cfgId, std::shared_ptr<config::IConfigContext> cfgCtx); ~ConfigShim(); ConfigShim & enableStateServer(bool v) { _enableStateServer = v; return *this; } diff --git a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp index f2ebc7ce78d..1bf6596744d 100644 --- a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp +++ b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp @@ -3,11 +3,10 @@ #include "reconfigurable_stateserver.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/net/state_server.h> +#include <vespa/config/helper/configfetcher.hpp> #include <thread> #include <vespa/log/log.h> -#include <vespa/config/common/exceptions.h> - LOG_SETUP(".slobrok.server.reconfigurable_stateserver"); using namespace std::chrono_literals; diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index a27680365a0..644344285ad 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -39,9 +39,9 @@ private: std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; - ConfigShim _configShim; - Configurator::UP _configurator; - bool _shuttingDown; + ConfigShim _configShim; + std::unique_ptr<Configurator> _configurator; + bool _shuttingDown; SBEnv(const SBEnv &); // Not used SBEnv &operator=(const SBEnv &); // Not used diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp index a174ba3ecb6..21e374e4f62 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> -#include <iostream> using namespace vespalib; @@ -302,7 +301,7 @@ void Test::testBufferAccess() { DataBuffer buf; - uint32_t len = 0; + uint32_t len; { GenericHeader header; header.putTag(GenericHeader::Tag("foo", 6.9)); @@ -345,8 +344,6 @@ Test::testFileReader() buf[i] = (uint8_t)i; } EXPECT_EQUAL(256, file.Write2(buf, 256)); - - file.Close(); } { FastOS_File file; @@ -364,7 +361,7 @@ Test::testFileReader() } EXPECT_EQUAL(256u, sum); - file.Close(); + ASSERT_TRUE(file.Close()); file.Delete(); } } @@ -399,7 +396,7 @@ Test::testFileWriter() EXPECT_EQUAL(i, (uint32_t)buf[i]); } - file.Close(); + ASSERT_TRUE(file.Close()); file.Delete(); } } @@ -418,7 +415,6 @@ Test::testFileHeader() ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); len = header.writeFile(file); EXPECT_EQUAL(len, header.getSize()); - file.Close(); } { FastOS_File file; @@ -440,8 +436,6 @@ Test::testFileHeader() header.putTag(FileHeader::Tag("baz", "999666")); EXPECT_EQUAL(len, header.getSize()); EXPECT_EQUAL(len, header.rewriteFile(file)); - - file.Close(); } { FileHeader header; @@ -450,7 +444,7 @@ Test::testFileHeader() ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); EXPECT_EQUAL(len, header.readFile(file)); EXPECT_EQUAL(len, header.getSize()); - file.Close(); + ASSERT_TRUE(file.Close()); file.Delete(); EXPECT_TRUE(header.hasTag("foo")); @@ -575,34 +569,36 @@ Test::testRewriteErrors() header.putTag(FileHeader::Tag("foo", "bar")); uint32_t len = header.getSize(); - FastOS_File file; - ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); - EXPECT_EQUAL(len, header.writeFile(file)); - file.Close(); - - ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); - header.putTag(FileHeader::Tag("baz", "cox")); - EXPECT_TRUE(len != header.getSize()); - try { - header.rewriteFile(file); - EXPECT_TRUE(false); - } catch (IllegalHeaderException &e) { - EXPECT_EQUAL("Failed to rewrite resized header.", e.getMessage()); + { + FastOS_File file; + ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); + EXPECT_EQUAL(len, header.writeFile(file)); + } + { + FastOS_File file; + ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); + header.putTag(FileHeader::Tag("baz", "cox")); + EXPECT_TRUE(len != header.getSize()); + try { + header.rewriteFile(file); + EXPECT_TRUE(false); + } catch (IllegalHeaderException &e) { + EXPECT_EQUAL("Failed to rewrite resized header.", e.getMessage()); + } } - file.Close(); } void Test::testLayout() { - FastOS_File file; - const std::string fileName = TEST_PATH("fileheader.dat"); - ASSERT_TRUE(file.OpenReadOnly(fileName.c_str())); - FileHeader header; - uint32_t len = header.readFile(file); - EXPECT_EQUAL(len, header.getSize()); - file.Close(); + { + FastOS_File file; + const std::string fileName = TEST_PATH("fileheader.dat"); + ASSERT_TRUE(file.OpenReadOnly(fileName.c_str())); + uint32_t len = header.readFile(file); + EXPECT_EQUAL(len, header.getSize()); + } EXPECT_TRUE(header.hasTag("foo")); EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); @@ -621,7 +617,7 @@ Test::testReadSize(bool mapped) buf.writeInt32(21); buf.writeInt32(GenericHeader::VERSION); buf.writeInt32(1); - uint32_t headerLen = 0u; + uint32_t headerLen; if (mapped) { GenericHeader::MMapReader reader(buf.getData(), buf.getDataLen()); headerLen = FileHeader::readSize(reader); diff --git a/staging_vespalib/src/tests/json/json.cpp b/staging_vespalib/src/tests/json/json.cpp index 15bb841b8ca..1a707ae1776 100644 --- a/staging_vespalib/src/tests/json/json.cpp +++ b/staging_vespalib/src/tests/json/json.cpp @@ -358,7 +358,7 @@ JSONTest::testJsonStreamErrors() try{ vespalib::asciistream as; vespalib::JsonStream stream(as); - stream << Object() << Array(); + stream << Object() << jsonstream::Array(); } catch (vespalib::JsonStreamException& e) { EXPECT_EQUAL("Invalid state on call: An array value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); } @@ -423,7 +423,7 @@ JSONTest::testJsonStreamErrors() try{ vespalib::asciistream as; vespalib::JsonStream stream(as); - stream << Object() << End() << Array(); + stream << Object() << End() << jsonstream::Array(); } catch (vespalib::JsonStreamException& e) { EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't start a new array. (Finalized)", e.getReason()); } @@ -442,7 +442,7 @@ JSONTest::testJsonStreamStateReporting() using namespace vespalib::jsonstream; vespalib::asciistream as; vespalib::JsonStream stream(as); - stream << Array() << 13 + stream << jsonstream::Array() << 13 << "foo" << Object() << "key" << "value" << End() << false diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 5fc6d2a69ae..da31f1c1a79 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -18,7 +18,7 @@ class Fixture public: AdaptiveSequencedExecutor _threads; - Fixture() : _threads(2, 2, 0, 1000) { } + Fixture(bool is_max_pending_hard=true) : _threads(2, 2, 0, 1000, is_max_pending_hard) { } }; @@ -231,12 +231,12 @@ TEST_F("require that executeLambda works", Fixture) } TEST("require that you get correct number of executors") { - AdaptiveSequencedExecutor seven(7, 1, 0, 10); + AdaptiveSequencedExecutor seven(7, 1, 0, 10, true); EXPECT_EQUAL(7u, seven.getNumExecutors()); } TEST("require that you distribute well") { - AdaptiveSequencedExecutor seven(7, 1, 0, 10); + AdaptiveSequencedExecutor seven(7, 1, 0, 10, true); EXPECT_EQUAL(7u, seven.getNumExecutors()); for (uint32_t id=0; id < 1000; id++) { EXPECT_EQUAL(id%7, seven.getExecutorId(id).getId()); diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp index 3528cf74040..0f7c82ef988 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -55,12 +55,12 @@ int main(int argc, char **argv) { std::atomic<long> counter(0); std::unique_ptr<ISequencedTaskExecutor> executor; if (use_adaptive_executor) { - executor = std::make_unique<AdaptiveSequencedExecutor>(num_strands, num_threads, max_waiting, task_limit); + executor = std::make_unique<AdaptiveSequencedExecutor>(num_strands, num_threads, max_waiting, task_limit, true); } else { auto optimize = optimize_for_throughput ? vespalib::Executor::OptimizeFor::THROUGHPUT : vespalib::Executor::OptimizeFor::LATENCY; - executor = SequencedTaskExecutor::create(sequenced_executor, num_strands, task_limit, optimize); + executor = SequencedTaskExecutor::create(sequenced_executor, num_strands, task_limit, true, optimize); } vespalib::Timer timer; for (size_t task_id = 0; task_id < num_tasks; ++task_id) { diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index ef7f8bfb0f6..705d6346e8c 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -2,7 +2,8 @@ #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/adaptive_sequenced_executor.h> - +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/singleexecutor.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -22,7 +23,10 @@ class Fixture public: std::unique_ptr<ISequencedTaskExecutor> _threads; - Fixture() : _threads(SequencedTaskExecutor::create(sequenced_executor, 2)) { } + Fixture(bool is_task_limit_hard = true) : + _threads(SequencedTaskExecutor::create(sequenced_executor, 2, 1000, is_task_limit_hard, + Executor::OptimizeFor::LATENCY)) + { } }; @@ -96,6 +100,23 @@ TEST_F("require that task with same component id are serialized", Fixture) EXPECT_EQUAL(42, tv->_val); } +TEST_F("require that task with same component id are serialized when executed with list", Fixture) +{ + std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); + EXPECT_EQUAL(0, tv->_val); + ISequencedTaskExecutor::ExecutorId executorId = f._threads->getExecutorId(0); + ISequencedTaskExecutor::TaskList list; + list.template emplace_back(executorId, makeLambdaTask([=]() { usleep(2000); tv->modify(0, 14); })); + list.template emplace_back(executorId, makeLambdaTask([=]() { tv->modify(14, 42); })); + f._threads->executeTasks(std::move(list)); + tv->wait(2); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); + f._threads->sync_all(); + EXPECT_EQUAL(0, tv->_fail); + EXPECT_EQUAL(42, tv->_val); +} + TEST_F("require that task with different component ids are not serialized", Fixture) { int tryCnt = 0; @@ -136,7 +157,8 @@ TEST_F("require that task with same string component id are serialized", Fixture namespace { -int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) +int +detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) { int tryCnt = 0; for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { @@ -158,7 +180,8 @@ int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int t return tryCnt; } -vespalib::string makeAltComponentId(Fixture &f) +vespalib::string +makeAltComponentId(Fixture &f) { int tryCnt = 0; char altComponentId[20]; @@ -239,6 +262,28 @@ TEST("require that you get correct number of executors") { EXPECT_EQUAL(7u, seven->getNumExecutors()); } +void verifyHardLimitForLatency(bool expect_hard) { + auto sequenced = SequencedTaskExecutor::create(sequenced_executor, 1, 100, expect_hard, Executor::OptimizeFor::LATENCY); + const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*sequenced); + EXPECT_EQUAL(expect_hard,nullptr != dynamic_cast<const BlockingThreadStackExecutor *>(seq.first_executor())); +} + +void verifyHardLimitForThroughput(bool expect_hard) { + auto sequenced = SequencedTaskExecutor::create(sequenced_executor, 1, 100, expect_hard, Executor::OptimizeFor::THROUGHPUT); + const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*sequenced); + const SingleExecutor * first = dynamic_cast<const SingleExecutor *>(seq.first_executor()); + EXPECT_TRUE(first != nullptr); + EXPECT_EQUAL(expect_hard, first->isBlocking()); +} + +TEST("require that you can get executor with both hard and soft limit") { + verifyHardLimitForLatency(true); + verifyHardLimitForLatency(false); + verifyHardLimitForThroughput(true); + verifyHardLimitForThroughput(false); +} + + TEST("require that you distribute well") { auto seven = SequencedTaskExecutor::create(sequenced_executor, 7); const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*seven); @@ -288,15 +333,15 @@ TEST("Test creation of different types") { auto * seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); ASSERT_TRUE(seq != nullptr); - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, Executor::OptimizeFor::LATENCY); + iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::LATENCY); seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); ASSERT_TRUE(seq != nullptr); - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, Executor::OptimizeFor::THROUGHPUT); + iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::THROUGHPUT); seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); ASSERT_TRUE(seq != nullptr); - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, Executor::OptimizeFor::ADAPTIVE, 17); + iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::ADAPTIVE, 17); auto aseq = dynamic_cast<AdaptiveSequencedExecutor *>(iseq.get()); ASSERT_TRUE(aseq != nullptr); } diff --git a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp index 79777cdd53f..348e9bbd503 100644 --- a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp +++ b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/shutdownguard.h> +#include <vespa/vespalib/util/malloc_mmap_guard.h> #include <thread> #include <unistd.h> #include <sys/wait.h> @@ -8,12 +9,8 @@ using namespace vespalib; -TEST_SETUP(Test); - -int -Test::Main() +TEST("test shutdown guard") { - TEST_INIT("shutdownguard_test"); { ShutdownGuard farFuture(1000000s); std::this_thread::sleep_for(20ms); @@ -37,5 +34,10 @@ Test::Main() } EXPECT_TRUE(i < 800); } - TEST_DONE(); } + +TEST("test malloc mmap guard") { + MallocMmapGuard guard(0x100000); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index dd71380f64a..56352ff3c0d 100644 --- a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -30,6 +30,28 @@ TEST("test that all tasks are executed") { EXPECT_EQUAL(10000u, counter); } +TEST("test that executor can overflow") { + constexpr size_t NUM_TASKS = 1000; + std::atomic<uint64_t> counter(0); + vespalib::Gate gate; + SingleExecutor executor(sequenced_executor, 10, false, 1, 1ms); + executor.execute(makeLambdaTask([&gate] { gate.await();})); + + for(size_t i(0); i < NUM_TASKS; i++) { + executor.execute(makeLambdaTask([&counter, i] { + EXPECT_EQUAL(i, counter); + counter++; + })); + } + EXPECT_EQUAL(0u, counter); + ExecutorStats stats = executor.getStats(); + EXPECT_EQUAL(NUM_TASKS + 1, stats.acceptedTasks); + EXPECT_EQUAL(NUM_TASKS, stats.queueSize.max()); + gate.countDown(); + executor.sync(); + EXPECT_EQUAL(NUM_TASKS, counter); +} + void verifyResizeTaskLimit(bool up) { std::mutex lock; std::condition_variable cond; @@ -38,7 +60,7 @@ void verifyResizeTaskLimit(bool up) { constexpr uint32_t INITIAL = 20; const uint32_t INITIAL_2inN = roundUp2inN(INITIAL); double waterMarkRatio = 0.5; - SingleExecutor executor(sequenced_executor, INITIAL, INITIAL*waterMarkRatio, 10ms); + SingleExecutor executor(sequenced_executor, INITIAL, true, INITIAL*waterMarkRatio, 10ms); EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); EXPECT_EQUAL(uint32_t(INITIAL_2inN*waterMarkRatio), executor.get_watermark()); diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/staging_vespalib/src/vespa/vespalib/stllike/cache.h index 0f4349eb15a..181bb2ac63a 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.h @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/lrucache_map.h> #include <atomic> +#include <mutex> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index 2b52e9e167f..e69dd36d6f5 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT bits.cpp clock.cpp crc.cpp + document_runnable.cpp doom.cpp foregroundtaskexecutor.cpp growablebytebuffer.cpp @@ -12,10 +13,10 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT jsonexception.cpp jsonstream.cpp jsonwriter.cpp + malloc_mmap_guard.cpp process_memory_stats.cpp programoptions.cpp programoptions_testutils.cpp - document_runnable.cpp rusage.cpp sequencedtaskexecutor.cpp sequencedtaskexecutorobserver.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index 4d08e14375c..1e23ba15785 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp @@ -95,7 +95,7 @@ AdaptiveSequencedExecutor::maybe_block_self(std::unique_lock<std::mutex> &lock) while (_self.state == Self::State::BLOCKED) { _self.cond.wait(lock); } - while ((_self.state == Self::State::OPEN) && (_self.pending_tasks >= _cfg.max_pending)) { + while ((_self.state == Self::State::OPEN) && _cfg.is_above_max_pending(_self.pending_tasks)) { _self.state = Self::State::BLOCKED; while (_self.state == Self::State::BLOCKED) { _self.cond.wait(lock); @@ -228,7 +228,8 @@ AdaptiveSequencedExecutor::worker_main() } AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, - size_t max_waiting, size_t max_pending) + size_t max_waiting, size_t max_pending, + bool is_max_pending_hard) : ISequencedTaskExecutor(num_strands), _thread_tools(std::make_unique<ThreadTools>(*this)), _mutex(), @@ -238,7 +239,7 @@ AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t _self(), _stats(), _idleTracker(steady_clock::now()), - _cfg(num_threads, max_waiting, max_pending) + _cfg(num_threads, max_waiting, max_pending, is_max_pending_hard) { _stats.queueSize.add(_self.pending_tasks); _thread_tools->start(num_threads); diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index ccf6ab977f3..d6244564fbd 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -51,14 +51,22 @@ private: size_t max_waiting; size_t max_pending; size_t wakeup_limit; + bool is_max_pending_hard; void set_max_pending(size_t max_pending_in) { max_pending = std::max(1uL, max_pending_in); wakeup_limit = std::max(1uL, size_t(max_pending * 0.9)); assert(wakeup_limit > 0); assert(wakeup_limit <= max_pending); } - Config(size_t num_threads_in, size_t max_waiting_in, size_t max_pending_in) - : num_threads(num_threads_in), max_waiting(max_waiting_in), max_pending(1000), wakeup_limit(900) + bool is_above_max_pending(size_t pending) { + return (pending >= max_pending) && is_max_pending_hard; + } + Config(size_t num_threads_in, size_t max_waiting_in, size_t max_pending_in, bool is_max_pending_hard_in) + : num_threads(num_threads_in), + max_waiting(max_waiting_in), + max_pending(1000), + wakeup_limit(900), + is_max_pending_hard(is_max_pending_hard_in) { assert(num_threads > 0); set_max_pending(max_pending_in); @@ -143,7 +151,8 @@ private: void worker_main(); public: AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, - size_t max_waiting, size_t max_pending); + size_t max_waiting, size_t max_pending, + bool is_max_pending_hard); ~AdaptiveSequencedExecutor() override; ExecutorId getExecutorId(uint64_t component) const override; void executeTask(ExecutorId id, Task::UP task) override; diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp index 54d90a0f310..d7534514f41 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -15,28 +15,47 @@ Runnable::Runnable() Runnable::~Runnable() { std::lock_guard monitorGuard(_stateLock); - assert(_state == NOT_RUNNING); + assert(getState() == NOT_RUNNING); } bool Runnable::start(FastOS_ThreadPool& pool) { std::unique_lock guard(_stateLock); - _stateCond.wait(guard, [&](){ return (_state != STOPPING);}); + _stateCond.wait(guard, [&](){ return (getState() != STOPPING);}); - if (_state != NOT_RUNNING) return false; - _state = STARTING; + if (getState() != NOT_RUNNING) return false; + set_state(STARTING); if (pool.NewThread(this) == nullptr) { throw vespalib::IllegalStateException("Failed starting a new thread", VESPA_STRLOC); } return true; } +void Runnable::set_state(State new_state) noexcept +{ + _state.store(new_state, std::memory_order_relaxed); +} + +bool Runnable::stopping() const noexcept +{ + State s(getState()); + return (s == STOPPING) || (s == RUNNING && GetThread()->GetBreakFlag()); +} + +bool Runnable::running() const noexcept +{ + State s(getState()); + // Must check break-flag too, as threadpool will use that to close + // down. + return (s == STARTING || (s == RUNNING && !GetThread()->GetBreakFlag())); +} + bool Runnable::stop() { std::lock_guard monitor(_stateLock); - if (_state == STOPPING || _state == NOT_RUNNING) return false; + if (getState() == STOPPING || getState() == NOT_RUNNING) return false; GetThread()->SetBreakFlag(); - _state = STOPPING; + set_state(STOPPING); return onStop(); } @@ -48,8 +67,8 @@ bool Runnable::onStop() bool Runnable::join() const { std::unique_lock guard(_stateLock); - assert ((_state != STARTING) && (_state != RUNNING)); - _stateCond.wait(guard, [&](){ return (_state == NOT_RUNNING);}); + assert ((getState() != STARTING) && (getState() != RUNNING)); + _stateCond.wait(guard, [&](){ return (getState() == NOT_RUNNING);}); return true; } @@ -57,21 +76,21 @@ void Runnable::Run(FastOS_ThreadInterface*, void*) { { std::lock_guard guard(_stateLock); - // Dont set state if its alreadyt at stopping. (And let run() be + // Don't set state if its already at stopping. (And let run() be // called even though about to stop for consistency) - if (_state == STARTING) { - _state = RUNNING; + if (getState() == STARTING) { + set_state(RUNNING); } } // By not catching exceptions, they should abort whole application. - // We should thus not need to have a catch all to set state to not + // We should thus not need to have a catch-all to set state to not // running. run(); { std::lock_guard guard(_stateLock); - _state = NOT_RUNNING; + set_state(NOT_RUNNING); _stateCond.notify_all(); } } diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.h b/staging_vespalib/src/vespa/vespalib/util/document_runnable.h index cf2befcc8d5..5ca344ea7ef 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.h +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.h @@ -19,6 +19,7 @@ #pragma once #include <vespa/fastos/thread.h> +#include <atomic> namespace document { @@ -29,16 +30,17 @@ public: private: mutable std::mutex _stateLock; mutable std::condition_variable _stateCond; - State _state; + std::atomic<State> _state; void Run(FastOS_ThreadInterface*, void*) override; + void set_state(State new_state) noexcept; // _stateLock must be held public: /** * Create a runnable. * @param pool If set, runnable will be started in constructor. */ Runnable(); - ~Runnable(); + ~Runnable() override; /** * Start this runnable. @@ -71,26 +73,21 @@ public: */ virtual void run() = 0; - /** Get the current state of this runnable. */ - State getState() const { return _state; } + /** + * Get the current state of this runnable. + * Thread safe (but relaxed) read; may be stale if done outside _stateLock. + */ + [[nodiscard]] State getState() const noexcept { + return _state.load(std::memory_order_relaxed); + } /** Check if system is in the process of stopping. */ - bool stopping() const - { - State s(getState()); - return (s == STOPPING) || (s == RUNNING && GetThread()->GetBreakFlag()); - } + [[nodiscard]] bool stopping() const noexcept; /** * Checks if runnable is running or not. (Started is considered running) */ - bool running() const - { - State s(getState()); - // Must check breakflag too, as threadpool will use that to close - // down. - return (s == STARTING || (s == RUNNING && !GetThread()->GetBreakFlag())); - } + [[nodiscard]] bool running() const noexcept; }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp index c54f182891c..b31d72da3b1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp @@ -12,9 +12,16 @@ ISequencedTaskExecutor::ISequencedTaskExecutor(uint32_t numExecutors) ISequencedTaskExecutor::~ISequencedTaskExecutor() = default; +void +ISequencedTaskExecutor::executeTasks(TaskList tasks) { + for (auto & task : tasks) { + executeTask(task.first, std::move(task.second)); + } +} + ISequencedTaskExecutor::ExecutorId -ISequencedTaskExecutor::getExecutorIdFromName(vespalib::stringref componentId) const { - vespalib::hash<vespalib::stringref> hashfun; +ISequencedTaskExecutor::getExecutorIdFromName(stringref componentId) const { + hash<stringref> hashfun; return getExecutorId(hashfun(componentId)); } diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h index 3fe6fb5d678..ff90556e3e4 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h @@ -14,7 +14,7 @@ namespace vespalib { * Interface class to run multiple tasks in parallel, but tasks with same * id has to be run in sequence. */ -class ISequencedTaskExecutor : public vespalib::IWakeup +class ISequencedTaskExecutor : public IWakeup { public: class ExecutorId { @@ -28,6 +28,7 @@ public: private: uint32_t _id; }; + using TaskList = std::vector<std::pair<ExecutorId, Executor::Task::UP>>; ISequencedTaskExecutor(uint32_t numExecutors); virtual ~ISequencedTaskExecutor(); @@ -40,7 +41,7 @@ public: virtual ExecutorId getExecutorId(uint64_t componentId) const = 0; uint32_t getNumExecutors() const { return _numExecutors; } - ExecutorId getExecutorIdFromName(vespalib::stringref componentId) const; + ExecutorId getExecutorIdFromName(stringref componentId) const; /** * Returns an executor id that is NOT equal to the given executor id, @@ -58,7 +59,13 @@ public: * @param id which internal executor to use * @param task unique pointer to the task to be executed */ - virtual void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) = 0; + virtual void executeTask(ExecutorId id, Executor::Task::UP task) = 0; + /** + * Schedule a list of tasks to run after all previously scheduled tasks with + * same id. Default is to just iterate and execute one by one, but implementations + * that can schedule all in one go more efficiently can implement this one. + */ + virtual void executeTasks(TaskList tasks); /** * Call this one to ensure you get the attention of the workers. */ @@ -74,7 +81,7 @@ public: */ template <class FunctionType> void executeLambda(ExecutorId id, FunctionType &&function) { - executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); } /** * Wait for all scheduled tasks to complete. @@ -83,7 +90,7 @@ public: virtual void setTaskLimit(uint32_t taskLimit) = 0; - virtual vespalib::ExecutorStats getStats() = 0; + virtual ExecutorStats getStats() = 0; /** * Wrap lambda function into a task and schedule it to be run. @@ -96,7 +103,7 @@ public: template <class FunctionType> void execute(uint64_t componentId, FunctionType &&function) { ExecutorId id = getExecutorId(componentId); - executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); } /** @@ -109,7 +116,7 @@ public: */ template <class FunctionType> void execute(ExecutorId id, FunctionType &&function) { - executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); } private: diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp new file mode 100644 index 00000000000..67181dfd16f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "malloc_mmap_guard.h" +#include <vespa/vespalib/util/size_literals.h> +#ifdef __linux__ +#include <malloc.h> +#endif +#include <limits> +#include <cassert> + +namespace vespalib { + +MallocMmapGuard::MallocMmapGuard(size_t mmapLimit) : + _threadId(std::this_thread::get_id()) +{ +#ifdef __linux__ + int limit = mmapLimit <= std::numeric_limits<int>::max() ? mmapLimit : std::numeric_limits<int>::max(); + mallopt(M_MMAP_THRESHOLD, limit); +#else + (void) mmapLimit; +#endif +} + +MallocMmapGuard::~MallocMmapGuard() +{ + assert(_threadId == std::this_thread::get_id()); +#ifdef __linux__ + mallopt(M_MMAP_THRESHOLD, 1_Gi); +#endif +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h new file mode 100644 index 00000000000..03e6d38c03c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <thread> + +namespace vespalib { + +/** + * Provides a hint to malloc implementation that all allocations in the scope of this guard + * will use mmap directly for allocation larger than the given limit. + * NB !! Note that guards can not be nested. Intention is to use around third party libraries where + * you do not control allocation yourself. + * The effect is implementation dependent. vespamalloc applies this only for the calling thread. + **/ +class MallocMmapGuard +{ +public: + MallocMmapGuard(size_t mmapLimit); + MallocMmapGuard(const MallocMmapGuard &) = delete; + MallocMmapGuard & operator=(const MallocMmapGuard &) = delete; + MallocMmapGuard(MallocMmapGuard &&) = delete; + MallocMmapGuard & operator=(MallocMmapGuard &&) = delete; + ~MallocMmapGuard(); +private: + std::thread::id _threadId; +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index a74010df944..f7e8e087727 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -3,6 +3,7 @@ #include "process_memory_stats.h" #include <vespa/vespalib/stllike/asciistream.h> #include <algorithm> +#include <vector> #include <vespa/log/log.h> diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index 76b0235301b..59ffad88d09 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -3,6 +3,7 @@ #include "sequencedtaskexecutor.h" #include "adaptive_sequenced_executor.h" #include "singleexecutor.h" +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/stllike/hashtable.h> @@ -46,30 +47,34 @@ SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads) { std::unique_ptr<ISequencedTaskExecutor> SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit) { - return create(func, threads, taskLimit, OptimizeFor::LATENCY); + return create(func, threads, taskLimit, true, OptimizeFor::LATENCY); } std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, OptimizeFor optimize) { - return create(func, threads, taskLimit, optimize, 0); +SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, bool is_task_limit_hard, OptimizeFor optimize) { + return create(func, threads, taskLimit, is_task_limit_hard, optimize, 0); } std::unique_ptr<ISequencedTaskExecutor> SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, - OptimizeFor optimize, uint32_t kindOfWatermark) + bool is_task_limit_hard, OptimizeFor optimize, uint32_t kindOfWatermark) { if (optimize == OptimizeFor::ADAPTIVE) { size_t num_strands = std::min(taskLimit, threads*32); - return std::make_unique<AdaptiveSequencedExecutor>(num_strands, threads, kindOfWatermark, taskLimit); + return std::make_unique<AdaptiveSequencedExecutor>(num_strands, threads, kindOfWatermark, taskLimit, is_task_limit_hard); } else { auto executors = std::vector<std::unique_ptr<SyncableThreadExecutor>>(); executors.reserve(threads); for (uint32_t id = 0; id < threads; ++id) { if (optimize == OptimizeFor::THROUGHPUT) { uint32_t watermark = (kindOfWatermark == 0) ? taskLimit / 10 : kindOfWatermark; - executors.push_back(std::make_unique<SingleExecutor>(func, taskLimit, watermark, 100ms)); + executors.push_back(std::make_unique<SingleExecutor>(func, taskLimit, is_task_limit_hard, watermark, 100ms)); } else { - executors.push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit, func)); + if (is_task_limit_hard) { + executors.push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit, func)); + } else { + executors.push_back(std::make_unique<ThreadStackExecutor>(1, stackSize, func)); + } } } return std::unique_ptr<ISequencedTaskExecutor>(new SequencedTaskExecutor(std::move(executors))); diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index 91304a6a2e3..a4b1b82aacf 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h @@ -35,10 +35,10 @@ public: static std::unique_ptr<ISequencedTaskExecutor> create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit); static std::unique_ptr<ISequencedTaskExecutor> - create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, OptimizeFor optimize); + create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, bool is_task_limit_hard, OptimizeFor optimize); static std::unique_ptr<ISequencedTaskExecutor> create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, - OptimizeFor optimize, uint32_t kindOfWatermark); + bool is_task_limit_hard, OptimizeFor optimize, uint32_t kindOfWatermark); /** * For testing only */ diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp index 5ae8e96b606..d81b8ec1db6 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp @@ -17,7 +17,7 @@ SequencedTaskExecutorObserver::SequencedTaskExecutorObserver(ISequencedTaskExecu SequencedTaskExecutorObserver::~SequencedTaskExecutorObserver() = default; void -SequencedTaskExecutorObserver::executeTask(ExecutorId id, vespalib::Executor::Task::UP task) +SequencedTaskExecutorObserver::executeTask(ExecutorId id, Executor::Task::UP task) { ++_executeCnt; { @@ -28,6 +28,19 @@ SequencedTaskExecutorObserver::executeTask(ExecutorId id, vespalib::Executor::Ta } void +SequencedTaskExecutorObserver::executeTasks(TaskList tasks) +{ + _executeCnt += tasks.size(); + { + std::lock_guard<std::mutex> guard(_mutex); + for (const auto & task : tasks) { + _executeHistory.emplace_back(task.first.getId()); + } + } + _executor.executeTasks(std::move(tasks)); +} + +void SequencedTaskExecutorObserver::sync_all() { ++_syncCnt; @@ -45,7 +58,7 @@ void SequencedTaskExecutorObserver::setTaskLimit(uint32_t taskLimit) { _executor.setTaskLimit(taskLimit); } -vespalib::ExecutorStats SequencedTaskExecutorObserver::getStats() { +ExecutorStats SequencedTaskExecutorObserver::getStats() { return _executor.getStats(); } diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h index 7e2bf968952..1d54283c393 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h @@ -24,10 +24,11 @@ public: ~SequencedTaskExecutorObserver() override; ExecutorId getExecutorId(uint64_t componentId) const override; - void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; + void executeTask(ExecutorId id, Executor::Task::UP task) override; + void executeTasks(TaskList tasks) override; void sync_all() override; void setTaskLimit(uint32_t taskLimit) override; - vespalib::ExecutorStats getStats() override; + ExecutorStats getStats() override; uint32_t getExecuteCnt() const { return _executeCnt; } uint32_t getSyncCnt() const { return _syncCnt; } diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp index a99bce0a705..21ed90c3d22 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -7,12 +7,12 @@ namespace vespalib { SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit) - : SingleExecutor(func, taskLimit, taskLimit/10, 100ms) + : SingleExecutor(func, taskLimit, true, taskLimit/10, 100ms) { } -SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t watermark, duration reactionTime) - : _watermarkRatio(watermark < taskLimit ? double(watermark) / taskLimit : 1.0), - _taskLimit(vespalib::roundUp2inN(taskLimit)), +SingleExecutor::SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime) + : _watermarkRatio(watermark < reservedQueueSize ? double(watermark) / reservedQueueSize : 1.0), + _taskLimit(vespalib::roundUp2inN(reservedQueueSize)), _wantedTaskLimit(_taskLimit.load()), _rp(0), _tasks(std::make_unique<Task::UP[]>(_taskLimit)), @@ -30,9 +30,13 @@ SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t wat _wp(0), _watermark(_taskLimit.load()*_watermarkRatio), _reactionTime(reactionTime), - _closed(false) + _closed(false), + _overflow() { - assert(taskLimit >= watermark); + assert(reservedQueueSize >= watermark); + if ( ! isQueueSizeHard) { + _overflow = std::make_unique<ArrayQueue<Task::UP>>(); + } _thread.start(); } @@ -62,10 +66,12 @@ SingleExecutor::execute(Task::UP task) { if (_closed) { return task; } - wait_for_room(guard); - wp = _wp.load(std::memory_order_relaxed); - _tasks[index(wp)] = std::move(task); - _wp.store(wp + 1, std::memory_order_release); + task = wait_for_room_or_put_in_overflow_Q(guard, std::move(task)); + if (task) { + wp = move_to_main_q(guard, std::move(task)); + } else { + wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(guard); + } } if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) { _consumerCondition.notify_one(); @@ -73,6 +79,24 @@ SingleExecutor::execute(Task::UP task) { return task; } +uint64_t +SingleExecutor::numTasks() { + if (_overflow) { + Lock guard(_mutex); + return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); + } else { + return num_tasks_in_main_q(); + } +} + +uint64_t +SingleExecutor::move_to_main_q(Lock &, Task::UP task) { + uint64_t wp = _wp.load(std::memory_order_relaxed); + _tasks[index(wp)] = std::move(task); + _wp.store(wp + 1, std::memory_order_release); + return wp; +} + void SingleExecutor::setTaskLimit(uint32_t taskLimit) { _wantedTaskLimit = vespalib::roundUp2inN(taskLimit); @@ -81,7 +105,7 @@ SingleExecutor::setTaskLimit(uint32_t taskLimit) { void SingleExecutor::drain(Lock & lock) { uint64_t wp = _wp.load(std::memory_order_relaxed); - while (numTasks() > 0) { + while (numTasks(lock) > 0) { _consumerCondition.notify_one(); sleepProducer(lock, 100us, wp); } @@ -97,7 +121,7 @@ SingleExecutor::wakeup() { SingleExecutor & SingleExecutor::sync() { Lock lock(_mutex); - uint64_t wp = _wp.load(std::memory_order_relaxed); + uint64_t wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); while (wp > _rp.load(std::memory_order_acquire)) { _consumerCondition.notify_one(); sleepProducer(lock, 100us, wp); @@ -119,7 +143,7 @@ SingleExecutor::run() { _producerCondition.notify_all(); _wakeupConsumerAt.store(_wp.load(std::memory_order_relaxed) + get_watermark(), std::memory_order_relaxed); Lock lock(_mutex); - if (numTasks() <= 0) { + if (numTasks(lock) <= 0) { steady_time now = steady_clock::now(); _threadIdleTracker.set_idle(now); _consumerCondition.wait_until(lock, now + _reactionTime); @@ -134,6 +158,22 @@ void SingleExecutor::drain_tasks() { while (numTasks() > 0) { run_tasks_till(_wp.load(std::memory_order_acquire)); + move_overflow_to_main_q(); + } +} + +void +SingleExecutor::move_overflow_to_main_q() +{ + if ( ! _overflow) return; + Lock guard(_mutex); + move_overflow_to_main_q(guard); +} +void +SingleExecutor::move_overflow_to_main_q(Lock & guard) { + while ( !_overflow->empty() && num_tasks_in_main_q() < _taskLimit.load(std::memory_order_relaxed)) { + move_to_main_q(guard, std::move(_overflow->front())); + _overflow->pop(); } } @@ -151,26 +191,42 @@ SingleExecutor::run_tasks_till(uint64_t available) { } } -void -SingleExecutor::wait_for_room(Lock & lock) { +Executor::Task::UP +SingleExecutor::wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task) { uint64_t wp = _wp.load(std::memory_order_relaxed); uint64_t taskLimit = _taskLimit.load(std::memory_order_relaxed); if (taskLimit != _wantedTaskLimit.load(std::memory_order_relaxed)) { - drain(lock); + drain(guard); _tasks = std::make_unique<Task::UP[]>(_wantedTaskLimit); _taskLimit = _wantedTaskLimit.load(); _watermark = _taskLimit * _watermarkRatio; } - _queueSize.add(numTasks()); - while (numTasks() >= _taskLimit.load(std::memory_order_relaxed)) { - sleepProducer(lock, _reactionTime, wp - get_watermark()); + uint64_t numTaskInQ = numTasks(guard); + _queueSize.add(numTaskInQ); + if (numTaskInQ >= _taskLimit.load(std::memory_order_relaxed)) { + if (_overflow) { + _overflow->push(std::move(task)); + } else { + while (numTasks(guard) >= _taskLimit.load(std::memory_order_relaxed)) { + sleepProducer(guard, _reactionTime, wp - get_watermark()); + } + } + } else { + if (_overflow && !_overflow->empty()) { + _overflow->push(std::move(task)); + } + } + if (_overflow && !_overflow->empty()) { + assert(!task); + move_overflow_to_main_q(guard); } + return task; } ExecutorStats SingleExecutor::getStats() { Lock lock(_mutex); - uint64_t accepted = _wp.load(std::memory_order_relaxed); + uint64_t accepted = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); steady_time now = steady_clock::now(); _idleTracker.was_idle(_threadIdleTracker.reset(now)); ExecutorStats stats(_queueSize, (accepted - _lastAccepted), 0, _wakeupCount); diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h index e76e3f17a41..dd755a76302 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/threadexecutor.h> #include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/executor_idle_tracking.h> #include <thread> #include <atomic> @@ -19,8 +20,8 @@ namespace vespalib { */ class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable { public: - SingleExecutor(init_fun_t func, uint32_t taskLimit); - SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t watermark, duration reactionTime); + SingleExecutor(init_fun_t func, uint32_t reservedQueueSize); + SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime); ~SingleExecutor() override; Task::UP execute(Task::UP task) override; void setTaskLimit(uint32_t taskLimit) override; @@ -32,6 +33,7 @@ public: duration get_reaction_time() const { return _reactionTime; } ExecutorStats getStats() override; SingleExecutor & shutdown() override; + bool isBlocking() const { return !_overflow; } private: using Lock = std::unique_lock<std::mutex>; void drain(Lock & lock); @@ -39,12 +41,22 @@ private: void drain_tasks(); void sleepProducer(Lock & guard, duration maxWaitTime, uint64_t wakeupAt); void run_tasks_till(uint64_t available); - void wait_for_room(Lock & guard); + Task::UP wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task); + uint64_t move_to_main_q(Lock & guard, Task::UP task); + void move_overflow_to_main_q(); + void move_overflow_to_main_q(Lock & guard); uint64_t index(uint64_t counter) const { return counter & (_taskLimit.load(std::memory_order_relaxed) - 1); } - uint64_t numTasks() const { + uint64_t numTasks(); + uint64_t numTasks(Lock & guard) const { + return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); + } + uint64_t num_tasks_in_overflow_q(Lock &) const { + return _overflow ? _overflow->size() : 0; + } + uint64_t num_tasks_in_main_q() const { return _wp.load(std::memory_order_relaxed) - _rp.load(std::memory_order_acquire); } const double _watermarkRatio; @@ -67,6 +79,7 @@ private: std::atomic<uint32_t> _watermark; const duration _reactionTime; bool _closed; + std::unique_ptr<ArrayQueue<Task::UP>> _overflow; }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp index fe0a47315e0..bdc09da127b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <cassert> +#include <vector> namespace vespalib::xml { diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index d1a56b8ac8a..5560a633f66 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -95,6 +95,11 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { } @Override + public Optional<String> cloud() { + return Optional.ofNullable(System.getenv("VESPA_CLOUD")); + } + + @Override public Optional<Boolean> useVespaVersionInRequest() { return getInstallVariable("use_vespa_version_in_request", Boolean::parseBoolean); } @@ -121,12 +126,6 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } - @Override - public String zooKeeperSnapshotMethod() { - String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); - return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; - } - static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index 74eff01984f..fe3e8782973 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -256,7 +256,7 @@ public class StandaloneContainerApplication implements Application { // such that the above and below code to finalize the container can be // replaced by root.finalize(); - initializeContainer(deployState.getDeployLogger(), container, spec); + initializeContainer(deployState, container, spec); root.freezeModelTopology(); return new Pair<>(root, container); @@ -287,12 +287,12 @@ public class StandaloneContainerApplication implements Application { return builder.build(); } - private static void initializeContainer(DeployLogger deployLogger, Container container, Element spec) { + private static void initializeContainer(DeployState deployState, Container container, Element spec) { HostResource host = container.getRoot().hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)); container.setHostResource(host); - container.initService(deployLogger); + container.initService(deployState); } private static Element getJDiscInServices(Element element) { diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java index d9db169ec54..5555b3b4077 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java @@ -4,6 +4,7 @@ package com.yahoo.container.standalone; import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.SubscriberClosedException; import com.yahoo.container.di.config.Subscriber; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.vespa.config.ConfigKey; @@ -31,6 +32,7 @@ public class StandaloneSubscriberFactory implements SubscriberFactory { private final Set<ConfigKey<ConfigInstance>> configKeys; private long generation = -1L; + private volatile boolean shutdown = false; StandaloneSubscriber(Set<ConfigKey<ConfigInstance>> configKeys) { this.configKeys = configKeys; @@ -41,9 +43,7 @@ public class StandaloneSubscriberFactory implements SubscriberFactory { return generation == 0; } - @Override - public void close() { - } + @Override public void close() { shutdown = true; } @Override public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() { @@ -57,15 +57,17 @@ public class StandaloneSubscriberFactory implements SubscriberFactory { return ret; } + @SuppressWarnings("removal") // TODO Vespa 8: remove @Override public long waitNextGeneration(boolean isInitializing) { generation++; if (generation != 0) { try { - while (!Thread.interrupted()) { - Thread.sleep(10000); + while (!shutdown && !Thread.interrupted()) { + Thread.sleep(100); } + if (shutdown) throw new SubscriberClosedException(); } catch (InterruptedException e) { throw new ConfigInterruptedException(e); } diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java index 39d8ad19f29..08b700dbe3c 100644 --- a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java +++ b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java @@ -15,6 +15,7 @@ import java.util.Iterator; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) @Deprecated public class CounterGroup extends Group { private final boolean resetCounter; diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java index e8f26e625d4..ea12c1e4c89 100644 --- a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java +++ b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java @@ -13,6 +13,7 @@ import java.util.HashMap; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) @Deprecated public class ValueGroup extends Group { // A map for names of subevents and Value instances diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java index 323a4fd23c6..c1899aa7e94 100644 --- a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertFalse; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") +@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove public class CounterGroupTestCase { private volatile boolean gotRecord = false; diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java index f9c3ee79ad7..f57fff8c312 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") +@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove public class ValueGroupTestCase { private volatile boolean gotRecord = false; diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java index 91a02d90a6e..b7f0d9ca8c4 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java @@ -21,7 +21,7 @@ import org.junit.Test; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") +@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove public class ValueTestCase { private static final double delta = 0.0000000001; diff --git a/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp b/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp index 23a0df81bab..cd70aecd1bb 100644 --- a/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp +++ b/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp @@ -2,7 +2,6 @@ #include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/config.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index 7a19e84791d..91fdf5aa602 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -4,14 +4,13 @@ #include <vespa/storage/common/content_bucket_db_options.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/config-stor-distribution.h> -#include <vespa/config-load-type.h> #include <vespa/config-fleetcontroller.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/time.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/configgetter.hpp> #include <thread> #include <sstream> diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index bee7650aebd..501f9a18c47 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST bucketdbmetricupdatertest.cpp bucketgctimecalculatortest.cpp bucketstateoperationtest.cpp + distributor_bucket_space_repo_test.cpp distributor_bucket_space_test.cpp distributor_host_info_reporter_test.cpp distributor_message_sender_stub.cpp diff --git a/storage/src/tests/distributor/distributor_bucket_space_repo_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_repo_test.cpp new file mode 100644 index 00000000000..151dcff3d10 --- /dev/null +++ b/storage/src/tests/distributor/distributor_bucket_space_repo_test.cpp @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <memory> + +namespace storage::distributor { + +using document::FixedBucketSpaces; +using namespace ::testing; + +struct DistributorBucketSpaceRepoTest : Test { + DistributorBucketSpaceRepo _repo; + + DistributorBucketSpaceRepoTest() : _repo(123) {} +}; + +namespace { + +lib::ClusterStateBundle bundle_with_global_merges() { + auto global_state = std::make_shared<lib::ClusterState>("distributor:1 storage:2"); + auto default_state = std::make_shared<lib::ClusterState>("distributor:1 storage:2 .1.s:m"); + return lib::ClusterStateBundle(*global_state, {{FixedBucketSpaces::default_space(), default_state}, + {FixedBucketSpaces::global_space(), global_state}}); +} + +lib::ClusterStateBundle bundle_without_global_merges() { + auto global_state = std::make_shared<lib::ClusterState>("distributor:1 storage:2"); + auto default_state = std::make_shared<lib::ClusterState>("distributor:1 storage:2"); + return lib::ClusterStateBundle(*global_state, {{FixedBucketSpaces::default_space(), default_state}, + {FixedBucketSpaces::global_space(), global_state}}); +} + +} + +TEST_F(DistributorBucketSpaceRepoTest, bucket_spaces_are_initially_not_tagged_as_merge_inhibited) { + EXPECT_FALSE(_repo.get(FixedBucketSpaces::default_space()).merges_inhibited()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::global_space()).merges_inhibited()); +} + +TEST_F(DistributorBucketSpaceRepoTest, enabled_bundle_with_pending_global_merges_tags_default_space_as_merge_inhibited) { + _repo.enable_cluster_state_bundle(bundle_with_global_merges()); + EXPECT_TRUE(_repo.get(FixedBucketSpaces::default_space()).merges_inhibited()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::global_space()).merges_inhibited()); +} + +TEST_F(DistributorBucketSpaceRepoTest, enabled_bundle_without_pending_global_merges_unsets_merge_inhibition) { + _repo.enable_cluster_state_bundle(bundle_with_global_merges()); + _repo.enable_cluster_state_bundle(bundle_without_global_merges()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::default_space()).merges_inhibited()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::global_space()).merges_inhibited()); +} + +TEST_F(DistributorBucketSpaceRepoTest, pending_bundle_with_pending_global_merges_tags_default_space_as_merge_inhibited) { + _repo.enable_cluster_state_bundle(bundle_without_global_merges()); + _repo.set_pending_cluster_state_bundle(bundle_with_global_merges()); + EXPECT_TRUE(_repo.get(FixedBucketSpaces::default_space()).merges_inhibited()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::global_space()).merges_inhibited()); +} + +TEST_F(DistributorBucketSpaceRepoTest, pending_bundle_without_pending_global_unsets_merge_inhibition) { + _repo.enable_cluster_state_bundle(bundle_with_global_merges()); + _repo.set_pending_cluster_state_bundle(bundle_without_global_merges()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::default_space()).merges_inhibited()); + EXPECT_FALSE(_repo.get(FixedBucketSpaces::global_space()).merges_inhibited()); +} + +} diff --git a/storage/src/tests/distributor/distributor_stripe_test.cpp b/storage/src/tests/distributor/distributor_stripe_test.cpp index 8c2ebc983fa..709f2e6cdc5 100644 --- a/storage/src/tests/distributor/distributor_stripe_test.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test.cpp @@ -629,6 +629,19 @@ TEST_F(DistributorStripeTest, max_clock_skew_config_is_propagated_to_distributor EXPECT_EQ(getConfig().getMaxClusterClockSkew(), std::chrono::seconds(5)); } +TEST_F(DistributorStripeTest, inhibit_default_merge_if_global_merges_pending_config_is_propagated) +{ + setup_stripe(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + ConfigBuilder builder; + builder.inhibitDefaultMergesWhenGlobalMergesPending = true; + configure_stripe(builder); + EXPECT_TRUE(getConfig().inhibit_default_merges_when_global_merges_pending()); + + builder.inhibitDefaultMergesWhenGlobalMergesPending = false; + configure_stripe(builder); + EXPECT_FALSE(getConfig().inhibit_default_merges_when_global_merges_pending()); +} + namespace { auto makeDummyRemoveCommand() { diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index b02395717e0..53773a55826 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/distributor/distributor_stripe_test_util.h> -#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/top_level_distributor.h> @@ -13,6 +12,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/config/helper/configgetter.h> using std::shared_ptr; using config::ConfigGetter; diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index d481370b2c1..94f913a3325 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -175,6 +175,8 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { bool _includeSchedulingPriority {false}; bool _merge_operations_disabled {false}; bool _prioritize_global_bucket_merges {true}; + bool _config_enable_default_space_merge_inhibition {false}; + bool _merges_inhibited_in_bucket_space {false}; CheckerParams(); ~CheckerParams(); @@ -222,6 +224,14 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { _bucket_space = bucket_space; return *this; } + CheckerParams& config_enable_default_space_merge_inhibition(bool enabled) noexcept { + _config_enable_default_space_merge_inhibition = enabled; + return *this; + } + CheckerParams& merges_inhibited_in_bucket_space(bool inhibited) noexcept { + _merges_inhibited_in_bucket_space = inhibited; + return *this; + } }; template <typename CheckerImpl> @@ -236,10 +246,12 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { vespa::config::content::core::StorDistributormanagerConfigBuilder config; config.mergeOperationsDisabled = params._merge_operations_disabled; config.prioritizeGlobalBucketMerges = params._prioritize_global_bucket_merges; + config.inhibitDefaultMergesWhenGlobalMergesPending = params._config_enable_default_space_merge_inhibition; configure_stripe(config); if (!params._pending_cluster_state.empty()) { simulate_set_pending_cluster_state(params._pending_cluster_state); } + getBucketSpaceRepo().get(params._bucket_space).set_merges_inhibited(params._merges_inhibited_in_bucket_space); NodeMaintenanceStatsTracker statsTracker; StateChecker::Context c(node_context(), operation_context(), @@ -818,6 +830,32 @@ TEST_F(StateCheckersTest, no_merge_operation_generated_if_merges_explicitly_conf .merge_operations_disabled(true)); } +TEST_F(StateCheckersTest, no_merge_operation_generated_if_merges_inhibited_in_default_bucket_space_and_config_allowed) { + // Technically, the state checker doesn't look at global vs. non-global but instead defers + // to the distributor bucket space repo to set the inhibition flag on the correct bucket space. + // This particular logic is tested at a higher repo-level. + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") // Would normally generate a merge op + .bucketInfo("0=1,2=2") + .config_enable_default_space_merge_inhibition(true) + .merges_inhibited_in_bucket_space(true) + .clusterState("distributor:1 storage:3")); +} + +TEST_F(StateCheckersTest, merge_operation_still_generated_if_merges_inhibited_in_default_bucket_space_but_config_disallowed) { + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Moving bucket to ideal node 1]" + "[Synchronizing buckets with different checksums " + "node(idx=0,crc=0x1,docs=1/1,bytes=1/1,trusted=false,active=false,ready=false), " + "node(idx=2,crc=0x2,docs=2/2,bytes=2/2,trusted=false,active=false,ready=false)]") + .bucketInfo("0=1,2=2") + .config_enable_default_space_merge_inhibition(false) + .merges_inhibited_in_bucket_space(true) + .clusterState("distributor:1 storage:3")); +} + std::string StateCheckersTest::testDeleteExtraCopies( const std::string& bucketInfo, uint32_t redundancy, diff --git a/storage/src/tests/distributor/top_level_distributor_test.cpp b/storage/src/tests/distributor/top_level_distributor_test.cpp index a0477e352d1..2c19652b4c2 100644 --- a/storage/src/tests/distributor/top_level_distributor_test.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test.cpp @@ -253,7 +253,7 @@ TEST_F(TopLevelDistributorTest, tick_aggregates_status_requests_from_all_stripes FakeClock clock; ThreadPoolImpl pool(clock); int ticksBeforeWait = 1; - framework::Thread::UP tp(pool.startThread(thread, "statustest", 5ms, 5s, ticksBeforeWait)); + framework::Thread::UP tp(pool.startThread(thread, "statustest", 5ms, 5s, ticksBeforeWait, std::nullopt)); while (true) { std::this_thread::sleep_for(1ms); diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index 2a61141865a..5dcc7f5c6ad 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -31,8 +31,8 @@ TopLevelDistributorTestUtil::~TopLevelDistributorTestUtil() = default; void TopLevelDistributorTestUtil::create_links() { - _node.reset(new TestDistributorApp(_config.getConfigId())); - _thread_pool = framework::TickingThreadPool::createDefault("distributor"); + _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); + _thread_pool = framework::TickingThreadPool::createDefault("distributor", 100ms); _stripe_pool = DistributorStripePool::make_non_threaded_pool_for_testing(); _distributor.reset(new TopLevelDistributor( _node->getComponentRegister(), @@ -43,7 +43,7 @@ TopLevelDistributorTestUtil::create_links() _num_distributor_stripes, _host_info, &_message_sender)); - _component.reset(new storage::DistributorComponent(_node->getComponentRegister(), "distrtestutil")); + _component = std::make_unique<storage::DistributorComponent>(_node->getComponentRegister(), "distrtestutil"); }; void diff --git a/storage/src/tests/distributor/visitoroperationtest.cpp b/storage/src/tests/distributor/visitoroperationtest.cpp index ea99186a434..6c597b620dd 100644 --- a/storage/src/tests/distributor/visitoroperationtest.cpp +++ b/storage/src/tests/distributor/visitoroperationtest.cpp @@ -252,6 +252,20 @@ TEST_F(VisitorOperationTest, no_bucket) { runEmptyVisitor(msg)); } +TEST_F(VisitorOperationTest, none_fieldset_is_rejected) { + enable_cluster_state("distributor:1 storage:1"); + auto msg = std::make_shared<api::CreateVisitorCommand>( + makeBucketSpace(), "dumpvisitor", "instance", ""); + msg->addBucketToBeVisited(document::BucketId(16, 1)); + msg->addBucketToBeVisited(nullId); + msg->setFieldSet("[none]"); + + EXPECT_EQ("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ILLEGAL_PARAMETERS, Field set '[none]' is not supported " + "for external visitor operations. Use '[id]' to return documents with no fields set.)", + runEmptyVisitor(msg)); +} + TEST_F(VisitorOperationTest, only_super_bucket_and_progress_allowed) { enable_cluster_state("distributor:1 storage:1"); diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 2e136977026..97bfa41aece 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -2,7 +2,6 @@ #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> -#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <tests/common/teststorageapp.h> @@ -10,6 +9,7 @@ #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/sync_crypto_socket.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index fb8120210c1..7b165e11b66 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST persistencethread_splittest.cpp processalltest.cpp provider_error_wrapper_test.cpp - shared_operation_throttler_test.cpp splitbitdetectortest.cpp testandsettest.cpp gtest_runner.cpp diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 1552a955221..e0538fb7ca7 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -72,8 +72,14 @@ public: /** * Set a mask for operations to fail with _result */ - void setFailureMask(uint32_t mask) { _failureMask = mask; } - uint32_t getFailureMask() const { return _failureMask; } + void setFailureMask(uint32_t mask) { + Guard guard(_lock); + _failureMask = mask; + } + uint32_t getFailureMask() const { + Guard guard(_lock); + return _failureMask; + } /** * Get a string representation of all the operations performed on the diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 6288f86993d..e9d399d357f 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -21,11 +21,9 @@ using namespace ::testing; namespace storage { /* - * Class for testing merge handler taking async_apply_bucket_diff as - * parameter for the test. + * Class for testing merge handler. */ -struct MergeHandlerTest : PersistenceTestUtils, - public testing::WithParamInterface<bool> { +struct MergeHandlerTest : PersistenceTestUtils { uint32_t _location; // Location used for all merge tests document::Bucket _bucket; // Bucket used for all merge tests uint64_t _maxTimestamp; @@ -172,11 +170,11 @@ struct MergeHandlerTest : PersistenceTestUtils, MergeHandler createHandler(size_t maxChunkSize = 0x400000) { return MergeHandler(getEnv(), getPersistenceProvider(), - getEnv()._component.cluster_context(), getEnv()._component.getClock(), *_sequenceTaskExecutor, maxChunkSize, 64, GetParam()); + getEnv()._component.cluster_context(), getEnv()._component.getClock(), *_sequenceTaskExecutor, maxChunkSize, 64); } MergeHandler createHandler(spi::PersistenceProvider & spi) { return MergeHandler(getEnv(), spi, - getEnv()._component.cluster_context(), getEnv()._component.getClock(), *_sequenceTaskExecutor, 4190208, 64, GetParam()); + getEnv()._component.cluster_context(), getEnv()._component.getClock(), *_sequenceTaskExecutor, 4190208, 64); } std::shared_ptr<api::StorageMessage> get_queued_reply() { @@ -232,7 +230,7 @@ MergeHandlerTest::setUpChain(ChainPos pos) { // Test a regular merge bucket command fetching data, including // puts, removes, unrevertable removes & duplicates. -TEST_P(MergeHandlerTest, merge_bucket_command) { +TEST_F(MergeHandlerTest, merge_bucket_command) { MergeHandler handler = createHandler(); LOG(debug, "Handle a merge bucket command"); @@ -293,11 +291,11 @@ MergeHandlerTest::testGetBucketDiffChain(bool midChain) EXPECT_EQ(17, diff.size()); } -TEST_P(MergeHandlerTest, get_bucket_diff_mid_chain) { +TEST_F(MergeHandlerTest, get_bucket_diff_mid_chain) { testGetBucketDiffChain(true); } -TEST_P(MergeHandlerTest, get_bucket_diff_end_of_chain) { +TEST_F(MergeHandlerTest, get_bucket_diff_end_of_chain) { testGetBucketDiffChain(false); } @@ -344,17 +342,17 @@ MergeHandlerTest::testApplyBucketDiffChain(bool midChain) EXPECT_EQ(0, diff.size()); } -TEST_P(MergeHandlerTest, apply_bucket_diff_mid_chain) { +TEST_F(MergeHandlerTest, apply_bucket_diff_mid_chain) { testApplyBucketDiffChain(true); } -TEST_P(MergeHandlerTest, apply_bucket_diff_end_of_chain) { +TEST_F(MergeHandlerTest, apply_bucket_diff_end_of_chain) { testApplyBucketDiffChain(false); } // Test that a simplistic merge with one thing to actually merge, // sends correct commands and finish. -TEST_P(MergeHandlerTest, master_message_flow) { +TEST_F(MergeHandlerTest, master_message_flow) { MergeHandler handler = createHandler(); LOG(debug, "Handle a merge bucket command"); @@ -448,7 +446,7 @@ getFilledDataSize(const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) } -TEST_P(MergeHandlerTest, chunked_apply_bucket_diff) { +TEST_F(MergeHandlerTest, chunked_apply_bucket_diff) { uint32_t docSize = 1024; uint32_t docCount = 10; uint32_t maxChunkSize = docSize * 3; @@ -512,7 +510,7 @@ TEST_P(MergeHandlerTest, chunked_apply_bucket_diff) { EXPECT_TRUE(reply->getResult().success()); } -TEST_P(MergeHandlerTest, chunk_limit_partially_filled_diff) { +TEST_F(MergeHandlerTest, chunk_limit_partially_filled_diff) { setUpChain(FRONT); uint32_t docSize = 1024; @@ -548,7 +546,7 @@ TEST_P(MergeHandlerTest, chunk_limit_partially_filled_diff) { EXPECT_LE(getFilledDataSize(fwdDiffCmd->getDiff()), maxChunkSize); } -TEST_P(MergeHandlerTest, max_timestamp) { +TEST_F(MergeHandlerTest, max_timestamp) { doPut(1234, spi::Timestamp(_maxTimestamp + 10), 1024, 1024); MergeHandler handler = createHandler(); @@ -656,7 +654,7 @@ MergeHandlerTest::createDummyGetBucketDiff(int timestampOffset, uint16_t hasMask return getBucketDiffCmd; } -TEST_P(MergeHandlerTest, spi_flush_guard) { +TEST_F(MergeHandlerTest, spi_flush_guard) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler = createHandler(providerWrapper); @@ -671,16 +669,14 @@ TEST_P(MergeHandlerTest, spi_flush_guard) { try { auto cmd = createDummyApplyDiff(6000); handler.handleApplyBucketDiff(*cmd, createTracker(cmd, _bucket)); - if (GetParam()) { - convert_delayed_error_to_exception(handler); - } + convert_delayed_error_to_exception(handler); FAIL() << "No exception thrown on failing in-place remove"; } catch (const std::runtime_error& e) { EXPECT_TRUE(std::string(e.what()).find("Failed remove") != std::string::npos); } } -TEST_P(MergeHandlerTest, bucket_not_found_in_db) { +TEST_F(MergeHandlerTest, bucket_not_found_in_db) { MergeHandler handler = createHandler(); // Send merge for unknown bucket auto cmd = std::make_shared<api::MergeBucketCommand>(makeDocumentBucket(document::BucketId(16, 6789)), _nodes, _maxTimestamp); @@ -688,7 +684,7 @@ TEST_P(MergeHandlerTest, bucket_not_found_in_db) { EXPECT_TRUE(tracker->getResult().isBucketDisappearance()); } -TEST_P(MergeHandlerTest, merge_progress_safe_guard) { +TEST_F(MergeHandlerTest, merge_progress_safe_guard) { MergeHandler handler = createHandler(); auto cmd = std::make_shared<api::MergeBucketCommand>(_bucket, _nodes, _maxTimestamp); handler.handleMergeBucket(*cmd, createTracker(cmd, _bucket)); @@ -711,7 +707,7 @@ TEST_P(MergeHandlerTest, merge_progress_safe_guard) { EXPECT_EQ(mergeReply->getResult().getResult(), api::ReturnCode::INTERNAL_FAILURE); } -TEST_P(MergeHandlerTest, safe_guard_not_invoked_when_has_mask_changes) { +TEST_F(MergeHandlerTest, safe_guard_not_invoked_when_has_mask_changes) { MergeHandler handler = createHandler(); _nodes.clear(); _nodes.emplace_back(0, false); @@ -743,7 +739,7 @@ TEST_P(MergeHandlerTest, safe_guard_not_invoked_when_has_mask_changes) { EXPECT_EQ(0x5, applyBucketDiffCmd2->getDiff()[0]._entry._hasMask); } -TEST_P(MergeHandlerTest, entry_removed_after_get_bucket_diff) { +TEST_F(MergeHandlerTest, entry_removed_after_get_bucket_diff) { MergeHandler handler = createHandler(); std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff; { @@ -799,9 +795,7 @@ MergeHandlerTest::doTestSPIException(MergeHandler& handler, providerWrapper.setFailureMask(failureMask); try { invoker.invoke(*this, handler, *_context); - if (GetParam()) { - convert_delayed_error_to_exception(handler); - } + convert_delayed_error_to_exception(handler); if (failureMask != 0) { return (std::string("No exception was thrown during handler " "invocation. Expected exception containing '") @@ -870,7 +864,7 @@ MergeHandlerTest::HandleMergeBucketInvoker::invoke( handler.handleMergeBucket(*cmd, test.createTracker(cmd, test._bucket)); } -TEST_P(MergeHandlerTest, merge_bucket_spi_failures) { +TEST_F(MergeHandlerTest, merge_bucket_spi_failures) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler = createHandler(providerWrapper); providerWrapper.setResult( @@ -901,7 +895,7 @@ MergeHandlerTest::HandleGetBucketDiffInvoker::invoke( handler.handleGetBucketDiff(*cmd, test.createTracker(cmd, test._bucket)); } -TEST_P(MergeHandlerTest, get_bucket_diff_spi_failures) { +TEST_F(MergeHandlerTest, get_bucket_diff_spi_failures) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler = createHandler(providerWrapper); providerWrapper.setResult(spi::Result(spi::Result::ErrorType::PERMANENT_ERROR, "who you gonna call?")); @@ -933,7 +927,7 @@ MergeHandlerTest::HandleApplyBucketDiffInvoker::invoke( handler.handleApplyBucketDiff(*cmd, test.createTracker(cmd, test._bucket)); } -TEST_P(MergeHandlerTest, apply_bucket_diff_spi_failures) { +TEST_F(MergeHandlerTest, apply_bucket_diff_spi_failures) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler = createHandler(providerWrapper); providerWrapper.setResult( @@ -998,7 +992,7 @@ MergeHandlerTest::HandleGetBucketDiffReplyInvoker::afterInvoke( api::ReturnCode::INTERNAL_FAILURE); } -TEST_P(MergeHandlerTest, get_bucket_diff_reply_spi_failures) { +TEST_F(MergeHandlerTest, get_bucket_diff_reply_spi_failures) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler = createHandler(providerWrapper); providerWrapper.setResult( @@ -1073,9 +1067,7 @@ MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::invoke( test.fillDummyApplyDiff(reply->getDiff()); _stub.clear(); handler.handleApplyBucketDiffReply(*reply, _stub, test.createTracker(reply, test._bucket)); - if (test.GetParam()) { - convert_delayed_error_to_exception(test, handler); - } + convert_delayed_error_to_exception(test, handler); } std::string @@ -1099,7 +1091,7 @@ MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::afterInvoke( } } -TEST_P(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { +TEST_F(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); HandleApplyBucketDiffReplyInvoker invoker; for (int i = 0; i < 2; ++i) { @@ -1126,7 +1118,7 @@ TEST_P(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { } } -TEST_P(MergeHandlerTest, remove_from_diff) { +TEST_F(MergeHandlerTest, remove_from_diff) { framework::defaultimplementation::FakeClock clock; MergeStatus status(clock, 0, 0); @@ -1192,7 +1184,7 @@ TEST_P(MergeHandlerTest, remove_from_diff) { } } -TEST_P(MergeHandlerTest, remove_put_on_existing_timestamp) { +TEST_F(MergeHandlerTest, remove_put_on_existing_timestamp) { setUpChain(BACK); document::TestDocMan docMan; @@ -1216,15 +1208,10 @@ TEST_P(MergeHandlerTest, remove_put_on_existing_timestamp) { auto tracker = handler.handleApplyBucketDiff(*applyBucketDiffCmd, createTracker(applyBucketDiffCmd, _bucket)); - if (GetParam()) { - ASSERT_FALSE(tracker); - handler.drain_async_writes(); - auto applyBucketDiffReply = std::dynamic_pointer_cast<api::ApplyBucketDiffReply>(get_queued_reply()); - ASSERT_TRUE(applyBucketDiffReply.get()); - } else { - auto applyBucketDiffReply = std::dynamic_pointer_cast<api::ApplyBucketDiffReply>(std::move(*tracker).stealReplySP()); - ASSERT_TRUE(applyBucketDiffReply.get()); - } + ASSERT_FALSE(tracker); + handler.drain_async_writes(); + auto applyBucketDiffReply = std::dynamic_pointer_cast<api::ApplyBucketDiffReply>(get_queued_reply()); + ASSERT_TRUE(applyBucketDiffReply.get()); tracker.reset(); auto cmd = std::make_shared<api::MergeBucketCommand>(_bucket, _nodes, _maxTimestamp); @@ -1314,7 +1301,7 @@ std::ostream &operator<<(std::ostream &os, const GetBucketDiffCommand::Entry &en } -TEST_P(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) +TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) { using NodeList = decltype(_nodes); // Redundancy is 2 and source only nodes 3 and 4 have doc1 and doc2 @@ -1446,14 +1433,10 @@ TEST_P(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) handler.handleApplyBucketDiffReply(*reply, messageKeeper(), createTracker(reply, _bucket)); LOG(debug, "handled fourth ApplyBucketDiffReply"); } - if (GetParam()) { - handler.drain_async_writes(); - } + handler.drain_async_writes(); ASSERT_EQ(6u, messageKeeper()._msgs.size()); ASSERT_EQ(api::MessageType::MERGEBUCKET_REPLY, messageKeeper()._msgs[5]->getType()); LOG(debug, "got mergebucket reply"); } -VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(AsyncApplyBucketDiffParams, MergeHandlerTest, testing::Values(false, true)); - } // storage diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index c163f6de024..c35be2789da 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -127,7 +127,7 @@ VESPA_THREAD_STACK_TAG(test_executor) void PersistenceTestUtils::setupExecutor(uint32_t numThreads) { - _sequenceTaskExecutor = vespalib::SequencedTaskExecutor::create(test_executor, numThreads, 1000, vespalib::Executor::OptimizeFor::ADAPTIVE); + _sequenceTaskExecutor = vespalib::SequencedTaskExecutor::create(test_executor, numThreads, 1000, true, vespalib::Executor::OptimizeFor::ADAPTIVE); } StorBucketDatabase::WrappedEntry diff --git a/storage/src/tests/persistence/shared_operation_throttler_test.cpp b/storage/src/tests/persistence/shared_operation_throttler_test.cpp deleted file mode 100644 index 0ad380937c7..00000000000 --- a/storage/src/tests/persistence/shared_operation_throttler_test.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/persistence/shared_operation_throttler.h> -#include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/barrier.h> -#include <chrono> -#include <thread> - -using namespace ::testing; - -namespace storage { - -using ThrottleToken = SharedOperationThrottler::Token; - -TEST(SharedOperationThrottlerTest, unlimited_throttler_does_not_throttle) { - // We technically can't test that the unlimited throttler _never_ throttles, but at - // least check that it doesn't throttle _twice_, and then induce from this ;) - auto throttler = SharedOperationThrottler::make_unlimited_throttler(); - auto token1 = throttler->try_acquire_one(); - EXPECT_TRUE(token1.valid()); - auto token2 = throttler->blocking_acquire_one(); - EXPECT_TRUE(token2.valid()); - // Window size should be zero (i.e. unlimited) for unlimited throttler - EXPECT_EQ(throttler->current_window_size(), 0); -} - -TEST(SharedOperationThrottlerTest, dynamic_throttler_respects_initial_window_size) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - auto token1 = throttler->try_acquire_one(); - EXPECT_TRUE(token1.valid()); - auto token2 = throttler->try_acquire_one(); - EXPECT_FALSE(token2.valid()); - - EXPECT_EQ(throttler->current_window_size(), 1); -} - -TEST(SharedOperationThrottlerTest, blocking_acquire_returns_immediately_if_slot_available) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - auto token = throttler->blocking_acquire_one(); - EXPECT_TRUE(token.valid()); - token.reset(); - token = throttler->blocking_acquire_one(600s); // Should never block. - EXPECT_TRUE(token.valid()); -} - -TEST(SharedOperationThrottlerTest, blocking_call_woken_up_if_throttle_slot_available) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - vespalib::Barrier barrier(2); - std::thread t([&] { - auto token = throttler->try_acquire_one(); - assert(token.valid()); - barrier.await(); - while (throttler->waiting_threads() != 1) { - std::this_thread::sleep_for(100us); - } - // Implicit token release at thread scope exit - }); - barrier.await(); - auto token = throttler->blocking_acquire_one(); - EXPECT_TRUE(token.valid()); - t.join(); -} - -TEST(SharedOperationThrottlerTest, time_bounded_blocking_acquire_waits_for_timeout) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - auto window_filling_token = throttler->try_acquire_one(); - auto before = std::chrono::steady_clock::now(); - // Will block for at least 1ms. Since no window slot will be available by that time, - // an invalid token should be returned. - auto token = throttler->blocking_acquire_one(1ms); - auto after = std::chrono::steady_clock::now(); - EXPECT_TRUE((after - before) >= 1ms); - EXPECT_FALSE(token.valid()); -} - -TEST(SharedOperationThrottlerTest, default_constructed_token_is_invalid) { - ThrottleToken token; - EXPECT_FALSE(token.valid()); - token.reset(); // no-op - EXPECT_FALSE(token.valid()); -} - -TEST(SharedOperationThrottlerTest, token_destruction_frees_up_throttle_window_slot) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - { - auto token = throttler->try_acquire_one(); - EXPECT_TRUE(token.valid()); - } - auto token = throttler->try_acquire_one(); - EXPECT_TRUE(token.valid()); -} - -TEST(SharedOperationThrottlerTest, token_can_be_moved_and_reset) { - auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); - auto token1 = throttler->try_acquire_one(); - auto token2 = std::move(token1); // move ctor - EXPECT_TRUE(token2.valid()); - EXPECT_FALSE(token1.valid()); - ThrottleToken token3; - token3 = std::move(token2); // move assignment op - EXPECT_TRUE(token3.valid()); - EXPECT_FALSE(token2.valid()); - - // Trying to fetch new token should not succeed due to active token and win size of 1 - token1 = throttler->try_acquire_one(); - EXPECT_FALSE(token1.valid()); - // Resetting the token should free up the slot in the window - token3.reset(); - token1 = throttler->try_acquire_one(); - EXPECT_TRUE(token1.valid()); -} - -// TODO ideally we'd test that the dynamic throttler has a window size that is actually -// dynamic, but the backing DynamicThrottlePolicy implementation is a black box so -// it's not trivial to know how to do this reliably. - -} diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 0f844ab6b4f..89b769078cc 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1,18 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/document_runnable.h> -#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/messagebus/dynamicthrottlepolicy.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/storageserver/mergethrottler.h> -#include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> -#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/vespalib/util/exceptions.h> #include <unordered_set> #include <memory> #include <iterator> @@ -159,6 +160,8 @@ struct MergeThrottlerTest : Test { assert(merge); return merge; } + + [[nodiscard]] uint32_t throttler_max_merges_pending(uint16_t throttler_index) const noexcept; }; MergeThrottlerTest::MergeThrottlerTest() = default; @@ -212,13 +215,10 @@ bool checkChain(const StorageMessage::SP& msg, Iterator first, Iterator end) { - const MergeBucketCommand& cmd = - dynamic_cast<const MergeBucketCommand&>(*msg); - + auto& cmd = dynamic_cast<const MergeBucketCommand&>(*msg); if (cmd.getChain().size() != static_cast<size_t>(std::distance(first, end))) { return false; } - return std::equal(cmd.getChain().begin(), cmd.getChain().end(), first); } @@ -247,11 +247,17 @@ void waitUntilMergeQueueIs(MergeThrottler& throttler, size_t sz, int timeout) } +uint32_t +MergeThrottlerTest::throttler_max_merges_pending(uint16_t throttler_index) const noexcept +{ + return static_cast<uint32_t>(_throttlers[throttler_index]->getThrottlePolicy().getMaxWindowSize()); +} + // Extremely simple test that just checks that (min|max)_merges_per_node // under the stor-server config gets propagated to all the nodes TEST_F(MergeThrottlerTest, merges_config) { for (int i = 0; i < _storageNodeCount; ++i) { - EXPECT_EQ(25, _throttlers[i]->getThrottlePolicy().getMaxPendingCount()); + EXPECT_EQ(25, throttler_max_merges_pending(i)); EXPECT_EQ(20, _throttlers[i]->getMaxQueueSize()); } } @@ -636,7 +642,7 @@ TEST_F(MergeThrottlerTest, resend_handling) { TEST_F(MergeThrottlerTest, priority_queuing) { // Fill up all active merges - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); std::vector<MergeBucketCommand::Node> nodes({{0}, {1}, {2}}); ASSERT_GE(maxPending, 4u); for (size_t i = 0; i < maxPending; ++i) { @@ -691,7 +697,7 @@ TEST_F(MergeThrottlerTest, priority_queuing) { // in the queue for a merge that is already known. TEST_F(MergeThrottlerTest, command_in_queue_duplicate_of_known_merge) { // Fill up all active merges and 1 queued one - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); for (size_t i = 0; i < maxPending + 1; ++i) { std::vector<MergeBucketCommand::Node> nodes({{0}, {uint16_t(2 + i)}, {uint16_t(5 + i)}}); @@ -786,7 +792,7 @@ TEST_F(MergeThrottlerTest, invalid_receiver_node) { // order. TEST_F(MergeThrottlerTest, forward_queued_merge) { // Fill up all active merges and then 3 queued ones - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); for (size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes({{0}, {uint16_t(2 + i)}, {uint16_t(5 + i)}}); @@ -846,7 +852,7 @@ TEST_F(MergeThrottlerTest, execute_queued_merge) { DummyStorageLink& bottomLink(*_bottomLinks[1]); // Fill up all active merges and then 3 queued ones - size_t maxPending = throttler.getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(1); ASSERT_LT(maxPending, 100); for (size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes({{1}, {uint16_t(5 + i)}, {uint16_t(7 + i)}}); @@ -914,7 +920,7 @@ TEST_F(MergeThrottlerTest, execute_queued_merge) { TEST_F(MergeThrottlerTest, flush) { // Fill up all active merges and then 3 queued ones - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); for (size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes({{0}, {1}, {2}}); @@ -972,14 +978,11 @@ TEST_F(MergeThrottlerTest, unseen_merge_with_node_in_chain) { // Second, test that we get rejected before queueing up. This is to // avoid a hypothetical deadlock scenario. // Fill up all active merges - { - - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - for (size_t i = 0; i < maxPending; ++i) { - auto fillCmd = std::make_shared<MergeBucketCommand>( - makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); - _topLinks[0]->sendDown(fillCmd); - } + size_t maxPending = throttler_max_merges_pending(0); + for (size_t i = 0; i < maxPending; ++i) { + auto fillCmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); + _topLinks[0]->sendDown(fillCmd); } _topLinks[0]->sendDown(cmd); @@ -993,7 +996,7 @@ TEST_F(MergeThrottlerTest, unseen_merge_with_node_in_chain) { TEST_F(MergeThrottlerTest, merge_with_newer_cluster_state_flushes_outdated_queued){ // Fill up all active merges and then 3 queued ones with the same // system state - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); std::vector<api::StorageMessage::Id> ids; for (size_t i = 0; i < maxPending + 3; ++i) { @@ -1035,7 +1038,7 @@ TEST_F(MergeThrottlerTest, merge_with_newer_cluster_state_flushes_outdated_queue TEST_F(MergeThrottlerTest, updated_cluster_state_flushes_outdated_queued) { // State is version 1. Send down several merges with state version 2. - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); std::vector<api::StorageMessage::Id> ids; for (size_t i = 0; i < maxPending + 3; ++i) { @@ -1074,7 +1077,7 @@ TEST_F(MergeThrottlerTest, updated_cluster_state_flushes_outdated_queued) { // TODO remove functionality and test TEST_F(MergeThrottlerTest, legacy_42_merges_do_not_trigger_flush) { // Fill up all active merges and then 1 queued one - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); ASSERT_LT(maxPending, 100); for (size_t i = 0; i < maxPending + 1; ++i) { std::vector<MergeBucketCommand::Node> nodes({{0}, {1}, {2}}); @@ -1156,7 +1159,7 @@ TEST_F(MergeThrottlerTest, unknown_merge_with_self_in_chain) { } TEST_F(MergeThrottlerTest, busy_returned_on_full_queue_for_merges_sent_from_distributors) { - size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxPending = throttler_max_merges_pending(0); size_t maxQueue = _throttlers[0]->getMaxQueueSize(); ASSERT_EQ(20, maxQueue); ASSERT_LT(maxPending, 100); @@ -1205,7 +1208,7 @@ MergeThrottlerTest::receive_chained_merge_with_full_queue(bool disable_queue_lim { // Note: uses node with index 1 to not be the first node in chain _throttlers[1]->set_disable_queue_limits_for_chained_merges(disable_queue_limits); - size_t max_pending = _throttlers[1]->getThrottlePolicy().getMaxPendingCount(); + size_t max_pending = throttler_max_merges_pending(1); size_t max_enqueued = _throttlers[1]->getMaxQueueSize(); for (size_t i = 0; i < max_pending + max_enqueued; ++i) { std::vector<MergeBucketCommand::Node> nodes({{1}, {2}, {3}}); @@ -1452,7 +1455,7 @@ TEST_F(MergeThrottlerTest, source_only_merges_are_not_affected_by_backpressure) } void MergeThrottlerTest::fill_throttler_queue_with_n_commands(uint16_t throttler_index, size_t queued_count) { - size_t max_pending = _throttlers[throttler_index]->getThrottlePolicy().getMaxPendingCount(); + size_t max_pending = throttler_max_merges_pending(throttler_index); for (size_t i = 0; i < max_pending + queued_count; ++i) { _topLinks[throttler_index]->sendDown(MergeBuilder(document::BucketId(16, i)) .nodes(throttler_index, throttler_index + 1) diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index 742b5eeb671..f909a939efb 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -100,7 +100,7 @@ struct BTreeBucketDatabase::ReplicaValueTraits { template class bucketdb::GenericBTreeBucketDatabase<BTreeBucketDatabase::ReplicaValueTraits>; BTreeBucketDatabase::BTreeBucketDatabase() - : _impl(std::make_unique<ImplType>(make_default_array_store_config<ReplicaValueTraits::DataStoreType>())) + : _impl(std::make_unique<ImplType>(make_default_array_store_config<ReplicaValueTraits::DataStoreType>(), std::shared_ptr<vespalib::alloc::MemoryAllocator>())) { } diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 9edac8d79ff..73ddeeb888b 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -18,7 +18,6 @@ #include <vespa/storageapi/message/stat.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> #include <chrono> #include <thread> diff --git a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp b/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp index 9f9bbf4992e..a8c6bf6529c 100644 --- a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp +++ b/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp @@ -2,9 +2,8 @@ #include "global_bucket_space_distribution_converter.h" #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/config.h> #include <vespa/config/print/asciiconfigwriter.h> -#include <vespa/config/print/asciiconfigreader.h> +#include <vespa/config/print/asciiconfigreader.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vdslib/distribution/distribution_config_util.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 8a40899165f..b4c23725493 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -51,6 +51,7 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component) _enable_revert(true), _implicitly_clear_priority_on_schedule(false), _use_unordered_merge_chaining(false), + _inhibit_default_merges_when_global_merges_pending(false), _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED) { } @@ -173,6 +174,7 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist _enable_revert = config.enableRevert; _implicitly_clear_priority_on_schedule = config.implicitlyClearBucketPriorityOnSchedule; _use_unordered_merge_chaining = config.useUnorderedMergeChaining; + _inhibit_default_merges_when_global_merges_pending = config.inhibitDefaultMergesWhenGlobalMergesPending; _minimumReplicaCountingMode = config.minimumReplicaCountingMode; diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index ea1aca17116..b26f115827e 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -273,6 +273,12 @@ public: [[nodiscard]] bool use_unordered_merge_chaining() const noexcept { return _use_unordered_merge_chaining; } + void set_inhibit_default_merges_when_global_merges_pending(bool inhibit) noexcept { + _inhibit_default_merges_when_global_merges_pending = inhibit; + } + [[nodiscard]] bool inhibit_default_merges_when_global_merges_pending() const noexcept { + return _inhibit_default_merges_when_global_merges_pending; + } uint32_t num_distributor_stripes() const noexcept { return _num_distributor_stripes; } @@ -331,6 +337,7 @@ private: bool _enable_revert; bool _implicitly_clear_priority_on_schedule; bool _use_unordered_merge_chaining; + bool _inhibit_default_merges_when_global_merges_pending; DistrConfig::MinimumReplicaCountingMode _minimumReplicaCountingMode; diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index dba36d1b73d..1d2d4babf74 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -5,9 +5,9 @@ namespace=vespa.config.content.core maxpendingidealstateoperations int default=100 ## The total size of unique documents in a bucket before we split it due to -## being too big. By default this is now 32 MB. -splitsize int default=33544432 - +## being too big. By default this is now 16 MB. Should be kept in sync with stor-filestor.def:bucket_merge_chunk_size. +splitsize int default=16772216 + ## The maximum amount of entries in a file before we should attempt to split it. ## A meta data entry in a slotfile currently uses 40 bytes. It is probably ## good to have the split size, such that all meta data entries are normally @@ -293,3 +293,9 @@ implicitly_clear_bucket_priority_on_schedule bool default=true ## involved in a given merge have previously reported (as part of bucket info fetching) ## that they support the unordered merge feature. use_unordered_merge_chaining bool default=true + +## If true, inhibits _all_ merges to buckets in the default bucket space if the current +## cluster state bundle indicates that global merges are pending in the cluster, i.e. +## one or more nodes is in maintenance mode in the default bucket space but marked up in +## the global bucket space. +inhibit_default_merges_when_global_merges_pending bool default=false diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index a368c2e5b6f..00be2929229 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -33,9 +33,19 @@ node_reliability int default=1 restart ## A merge operation will be chained through all nodes involved in the ## merge, only actually starting the operation when every node has ## allowed it to pass through. +## NOTE: these config values are _not_ used if merge_throttling_policy.type +## is configured to DYNAMIC (see below). max_merges_per_node int default=16 max_merge_queue_size int default=100 +## Chooses the throttling policy used to control the active merge window size +## of the MergeThrottler component. +merge_throttling_policy.type enum { STATIC, DYNAMIC } default=STATIC +## Only used if merge_throttling_policy.type == DYNAMIC: +merge_throttling_policy.min_window_size int default=16 +merge_throttling_policy.max_window_size int default=128 +merge_throttling_policy.window_size_increment double default=2.0 + ## If the persistence provider indicates that it has exhausted one or more ## of its internal resources during a mutating operation, new merges will ## be bounced for this duration. Not allowing further merges helps take diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index 7293f9f7acc..5969ccad4cb 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -28,6 +28,7 @@ DistributorBucketSpace::DistributorBucketSpace(uint16_t node_index) _distribution(), _node_index(node_index), _distribution_bits(1u), + _merges_inhibited(false), _pending_cluster_state(), _available_nodes(), _ownerships(), diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index 32da78f9972..3dfd1e1ce30 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -37,6 +37,7 @@ class DistributorBucketSpace { std::shared_ptr<const lib::Distribution> _distribution; uint16_t _node_index; uint16_t _distribution_bits; + bool _merges_inhibited; std::shared_ptr<const lib::ClusterState> _pending_cluster_state; std::vector<bool> _available_nodes; mutable vespalib::hash_map<document::BucketId, BucketOwnershipFlags, document::BucketId::hash> _ownerships; @@ -85,6 +86,13 @@ public: bool has_pending_cluster_state() const noexcept { return static_cast<bool>(_pending_cluster_state); } const lib::ClusterState& get_pending_cluster_state() const noexcept { return *_pending_cluster_state; } + void set_merges_inhibited(bool inhibited) noexcept { + _merges_inhibited = inhibited; + } + [[nodiscard]] bool merges_inhibited() const noexcept { + return _merges_inhibited; + } + /** * Returns true if this distributor owns the given bucket in the * given cluster and current distribution config. diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index 09468b55430..c88abaf8373 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -3,6 +3,7 @@ #include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" #include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/vdslib/state/clusterstate.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <cassert> @@ -10,14 +11,15 @@ LOG_SETUP(".distributor.distributor_bucket_space_repo"); using document::BucketSpace; +using document::FixedBucketSpaces; namespace storage::distributor { DistributorBucketSpaceRepo::DistributorBucketSpaceRepo(uint16_t node_index) : _map() { - add(document::FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>(node_index)); - add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>(node_index)); + add(FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>(node_index)); + add(FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>(node_index)); } DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() = default; @@ -44,12 +46,54 @@ DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) const return *itr->second; } +namespace { + +bool content_node_is_up(const lib::ClusterState& state, uint16_t index) noexcept { + return (state.getNodeState(lib::Node(lib::NodeType::STORAGE, index)).getState() == lib::State::UP); +} + +bool content_node_is_in_maintenance(const lib::ClusterState& state, uint16_t index) noexcept { + return (state.getNodeState(lib::Node(lib::NodeType::STORAGE, index)).getState() == lib::State::MAINTENANCE); +} + +// Prioritized global bucket merging is taking place if at least one content node is +// marked as Up in the global bucket space state, but Maintenance in the default +// bucket space state. +bool bundle_implies_global_merging_active(const lib::ClusterStateBundle& bundle) noexcept { + auto& default_cs = bundle.getDerivedClusterState(FixedBucketSpaces::default_space()); + auto& global_cs = bundle.getDerivedClusterState(FixedBucketSpaces::global_space()); + if (default_cs.get() == global_cs.get()) { + return false; + } + uint16_t node_count = global_cs->getNodeCount(lib::NodeType::STORAGE); + for (uint16_t i = 0; i < node_count; ++i) { + if (content_node_is_up(*global_cs, i) && content_node_is_in_maintenance(*default_cs, i)) { + return true; + } + } + return false; +} + +} + +void +DistributorBucketSpaceRepo::enable_cluster_state_bundle(const lib::ClusterStateBundle& cluster_state_bundle) +{ + for (auto& entry : _map) { + entry.second->setClusterState(cluster_state_bundle.getDerivedClusterState(entry.first)); + } + get(FixedBucketSpaces::default_space()).set_merges_inhibited( + bundle_implies_global_merging_active(cluster_state_bundle)); +} + void DistributorBucketSpaceRepo::set_pending_cluster_state_bundle(const lib::ClusterStateBundle& cluster_state_bundle) { for (auto& entry : _map) { entry.second->set_pending_cluster_state(cluster_state_bundle.getDerivedClusterState(entry.first)); } + get(FixedBucketSpaces::default_space()).set_merges_inhibited( + bundle_implies_global_merging_active(cluster_state_bundle)); } void diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h index ab1e235ae35..d77a9f37fb0 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -33,6 +33,7 @@ public: BucketSpaceMap::const_iterator begin() const { return _map.begin(); } BucketSpaceMap::const_iterator end() const { return _map.end(); } void add(document::BucketSpace bucketSpace, std::unique_ptr<DistributorBucketSpace> distributorBucketSpace); + void enable_cluster_state_bundle(const lib::ClusterStateBundle& cluster_state_bundle); void set_pending_cluster_state_bundle(const lib::ClusterStateBundle& cluster_state_bundle); void clear_pending_cluster_state_bundle(); }; diff --git a/storage/src/vespa/storage/distributor/distributor_component.h b/storage/src/vespa/storage/distributor/distributor_component.h index 34fd08140e0..282f06981d3 100644 --- a/storage/src/vespa/storage/distributor/distributor_component.h +++ b/storage/src/vespa/storage/distributor/distributor_component.h @@ -23,7 +23,7 @@ class DistributorComponent : public storage::DistributorComponent, public DistributorOperationContext { private: DistributorInterface& _distributor; - BucketSpaceStateMap _bucket_space_states; + BucketSpaceStateMap _bucket_space_states; public: diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index bcba976f2c3..1173a19d729 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -481,9 +481,7 @@ void DistributorStripe::propagateClusterStates() { for (auto* repo : {_bucketSpaceRepo.get(), _readOnlyBucketSpaceRepo.get()}) { - for (auto& iter : *repo) { - iter.second->setClusterState(_clusterStateBundle.getDerivedClusterState(iter.first)); - } + repo->enable_cluster_state_bundle(_clusterStateBundle); } } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index 809b4dd0e41..49b062d6cc1 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -13,7 +13,6 @@ #include "stripe_access_guard.h" #include "stripe_bucket_db_updater.h" #include "tickable_stripe.h" -#include <vespa/config/config.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp index 42411d53d52..5abaad6ef9f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitoroperation.h" +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/storage/common/reindexing_constants.h> #include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/storage/distributor/top_level_distributor.h> @@ -347,6 +348,17 @@ VisitorOperation::verifyOperationSentToCorrectDistributor() verifyDistributorOwnsBucket(_superBucket.bid); } +void +VisitorOperation::verify_fieldset_makes_sense_for_visiting() +{ + if (_msg->getFieldSet() == document::NoFields::NAME) { + throw VisitorVerificationException( + api::ReturnCode::ILLEGAL_PARAMETERS, + "Field set '[none]' is not supported for external visitor operations. " + "Use '[id]' to return documents with no fields set."); + } +} + bool VisitorOperation::verifyCreateVisitorCommand(DistributorStripeMessageSender& sender) { @@ -354,6 +366,7 @@ VisitorOperation::verifyCreateVisitorCommand(DistributorStripeMessageSender& sen verifyOperationContainsBuckets(); verifyOperationHasSuperbucketAndProgress(); verifyOperationSentToCorrectDistributor(); + verify_fieldset_makes_sense_for_visiting(); // TODO wrap and test if (is_read_for_write() && (_msg->getMaxBucketsPerVisitor() != 1)) { throw VisitorVerificationException( diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index c38c463a313..ce10ea87c11 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -106,6 +106,7 @@ private: void verifyOperationContainsBuckets(); void verifyOperationHasSuperbucketAndProgress(); void verifyOperationSentToCorrectDistributor(); + void verify_fieldset_makes_sense_for_visiting(); bool verifyCreateVisitorCommand(DistributorStripeMessageSender& sender); bool pickBucketsToVisit(const std::vector<BucketDatabase::Entry>& buckets); bool expandBucketContaining(); diff --git a/storage/src/vespa/storage/distributor/statechecker.cpp b/storage/src/vespa/storage/distributor/statechecker.cpp index d1241f0bf66..e5a1822d455 100644 --- a/storage/src/vespa/storage/distributor/statechecker.cpp +++ b/storage/src/vespa/storage/distributor/statechecker.cpp @@ -63,9 +63,9 @@ StateChecker::Result::createStoredResult( StateChecker::Context::Context(const DistributorNodeContext& node_ctx_in, const DistributorStripeOperationContext& op_ctx_in, - const DistributorBucketSpace &distributorBucketSpace, + const DistributorBucketSpace& distributorBucketSpace, NodeMaintenanceStatsTracker& statsTracker, - const document::Bucket &bucket_) + const document::Bucket& bucket_) : bucket(bucket_), siblingBucket(op_ctx_in.get_sibling(bucket.getBucketId())), systemState(distributorBucketSpace.getClusterState()), @@ -77,7 +77,8 @@ StateChecker::Context::Context(const DistributorNodeContext& node_ctx_in, node_ctx(node_ctx_in), op_ctx(op_ctx_in), db(distributorBucketSpace.getBucketDatabase()), - stats(statsTracker) + stats(statsTracker), + merges_inhibited_in_bucket_space(distributorBucketSpace.merges_inhibited()) { idealState = distributorBucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nonretired_or_maintenance_nodes(); unorderedIdealState.insert(idealState.begin(), idealState.end()); diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index 3f938238ce7..b72898208e6 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -82,6 +82,7 @@ public: const DistributorStripeOperationContext& op_ctx; const BucketDatabase& db; NodeMaintenanceStatsTracker& stats; + const bool merges_inhibited_in_bucket_space; const BucketDatabase::Entry& getSiblingEntry() const { return siblingEntry; @@ -97,7 +98,7 @@ public: class ResultImpl { public: - virtual ~ResultImpl() {} + virtual ~ResultImpl() = default; virtual IdealStateOperation::UP createOperation() = 0; virtual MaintenancePriority getPriority() const = 0; virtual MaintenanceOperation::Type getType() const = 0; diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index a2069219d5d..71b2da1359a 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -656,6 +656,10 @@ public: addNode(entry->getNodeRef(i).getNode()); } } + MergeNodes(MergeNodes && rhs) noexcept = default; + MergeNodes & operator =(MergeNodes && rhs) noexcept = delete; + MergeNodes(const MergeNodes & rhs) = delete; + MergeNodes & operator =(const MergeNodes & rhs) = delete; ~MergeNodes(); @@ -828,12 +832,20 @@ allCopiesAreInvalid(const StateChecker::Context& c) return true; } +bool +merging_effectively_disabled_for_state_checker(const StateChecker::Context& c) noexcept +{ + return (c.distributorConfig.merge_operations_disabled() + || (c.distributorConfig.inhibit_default_merges_when_global_merges_pending() + && c.merges_inhibited_in_bucket_space)); +} + } StateChecker::Result SynchronizeAndMoveStateChecker::check(StateChecker::Context& c) { - if (c.distributorConfig.merge_operations_disabled()) { + if (merging_effectively_disabled_for_state_checker(c)) { return Result::noMaintenanceNeeded(); } if (isInconsistentlySplit(c)) { diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.h b/storage/src/vespa/storage/distributor/top_level_distributor.h index 64e431c9d4e..95bff77fe40 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.h +++ b/storage/src/vespa/storage/distributor/top_level_distributor.h @@ -17,7 +17,6 @@ #include "statusreporterdelegate.h" #include "stripe_bucket_db_updater.h" // TODO this is temporary #include "stripe_host_info_notifier.h" -#include <vespa/config/config.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> @@ -107,8 +106,8 @@ public: bool handleStatusRequest(const DelegatedStatusRequest& request) const override; - virtual framework::ThreadWaitInfo doCriticalTick(framework::ThreadIndex) override; - virtual framework::ThreadWaitInfo doNonCriticalTick(framework::ThreadIndex) override; + framework::ThreadWaitInfo doCriticalTick(framework::ThreadIndex) override; + framework::ThreadWaitInfo doNonCriticalTick(framework::ThreadIndex) override; // Called by DistributorStripe threads when they want to notify the cluster controller of changed stats. // Thread safe. diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 45dd44d0245..a9b7a727a5b 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -7,6 +7,8 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> #include <functional> #include <vespa/log/log.h> @@ -21,17 +23,17 @@ StatusWebServer::StatusWebServer( : _reporterMap(reporterMap), _port(0), _httpServer(), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _component(std::make_unique<framework::Component>(componentRegister, "Status")) { - _configFetcher.subscribe<vespa::config::content::core::StorStatusConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorStatusConfig>(configUri.getConfigId(), this); + _configFetcher->start(); } StatusWebServer::~StatusWebServer() { // Avoid getting config during shutdown - _configFetcher.close(); + _configFetcher->close(); if (_httpServer) { LOG(debug, "Shutting down status web server on port %u", _httpServer->getListenPort()); diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h index f4bf2534036..429f5249441 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h @@ -12,12 +12,15 @@ #include <vespa/storage/config/config-stor-status.h> #include <vespa/storageframework/generic/thread/runnable.h> -#include <vespa/config/config.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/vespalib/portal/portal.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <list> +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { namespace framework { @@ -28,6 +31,7 @@ namespace framework { class HttpUrlPath; class Component; } + class StatusWebServer : private config::IFetcherCallback<vespa::config::content::core::StorStatusConfig> { class WebServer : public vespalib::Portal::GetHandler { @@ -64,7 +68,7 @@ class StatusWebServer : private config::IFetcherCallback<vespa::config::content: framework::StatusReporterMap& _reporterMap; uint16_t _port; std::unique_ptr<WebServer> _httpServer; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; std::unique_ptr<framework::Component> _component; public: diff --git a/storage/src/vespa/storage/persistence/CMakeLists.txt b/storage/src/vespa/storage/persistence/CMakeLists.txt index 5e068236026..c737d2bed28 100644 --- a/storage/src/vespa/storage/persistence/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(storage_spersistence OBJECT persistenceutil.cpp processallhandler.cpp provider_error_wrapper.cpp - shared_operation_throttler.cpp simplemessagehandler.cpp splitbitdetector.cpp splitjoinhandler.cpp diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp index 73ccc7f6085..69d35253aa6 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp @@ -9,7 +9,7 @@ namespace storage { ApplyBucketDiffEntryComplete::ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, document::DocumentId doc_id, - SharedOperationThrottler::Token throttle_token, + ThrottleToken throttle_token, const char *op, const framework::Clock& clock, metrics::DoubleAverageMetric& latency_metric) diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h index 8478cab4c17..a78fbe38ae5 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h @@ -22,14 +22,14 @@ class ApplyBucketDiffEntryComplete : public spi::OperationComplete const spi::ResultHandler* _result_handler; std::shared_ptr<ApplyBucketDiffState> _state; document::DocumentId _doc_id; - SharedOperationThrottler::Token _throttle_token; + ThrottleToken _throttle_token; const char* _op; framework::MilliSecTimer _start_time; metrics::DoubleAverageMetric& _latency_metric; public: ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, document::DocumentId doc_id, - SharedOperationThrottler::Token throttle_token, + ThrottleToken throttle_token, const char *op, const framework::Clock& clock, metrics::DoubleAverageMetric& latency_metric); ~ApplyBucketDiffEntryComplete() override; diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 3d24ee87879..d80eb140eec 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -19,6 +19,7 @@ #include <vespa/log/log.h> LOG_SETUP(".storage.persistence.asynchandler"); +using vespalib::CpuUsage; using vespalib::make_string_short::fmt; namespace storage { @@ -113,7 +114,7 @@ class UnrevertableRemoveEntryProcessor : public BucketProcessor::EntryProcessor public: using DocumentIdsAndTimeStamps = std::vector<std::pair<spi::Timestamp, spi::DocumentId>>; UnrevertableRemoveEntryProcessor(DocumentIdsAndTimeStamps & to_remove) - : _to_remove(to_remove) + : _to_remove(to_remove) {} void process(spi::DocEntry& entry) override { @@ -410,9 +411,13 @@ AsyncHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, MessageTrack spi::Bucket bucket(cmd.getBucket()); UnrevertableRemoveEntryProcessor::DocumentIdsAndTimeStamps to_remove; UnrevertableRemoveEntryProcessor processor(to_remove); - BucketProcessor::iterateAll(_spi, bucket, cmd.getDocumentSelection(), - std::make_shared<document::DocIdOnly>(), - processor, spi::NEWEST_DOCUMENT_ONLY,tracker->context()); + + { + auto usage = vespalib::CpuUsage::use(CpuUsage::Category::READ); + BucketProcessor::iterateAll(_spi, bucket, cmd.getDocumentSelection(), + std::make_shared<document::DocIdOnly>(), + processor, spi::NEWEST_DOCUMENT_ONLY, tracker->context()); + } auto task = makeResultTask([&cmd, tracker = std::move(tracker), removed = to_remove.size()](spi::Result::UP response) { tracker->checkForError(*response); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index 6f740ce2c28..66dc7126058 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -78,7 +78,7 @@ public: struct LockedMessage { std::shared_ptr<BucketLockInterface> lock; std::shared_ptr<api::StorageMessage> msg; - SharedOperationThrottler::Token throttle_token; + ThrottleToken throttle_token; LockedMessage() noexcept = default; LockedMessage(std::shared_ptr<BucketLockInterface> lock_, @@ -89,7 +89,7 @@ public: {} LockedMessage(std::shared_ptr<BucketLockInterface> lock_, std::shared_ptr<api::StorageMessage> msg_, - SharedOperationThrottler::Token token) noexcept + ThrottleToken token) noexcept : lock(std::move(lock_)), msg(std::move(msg_)), throttle_token(std::move(token)) @@ -274,7 +274,7 @@ public: virtual ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const = 0; - virtual SharedOperationThrottler& operation_throttler() const noexcept = 0; + virtual vespalib::SharedOperationThrottler& operation_throttler() const noexcept = 0; }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 2ccbc7a85ef..f7d4f750884 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -40,14 +40,14 @@ uint32_t per_stripe_merge_limit(uint32_t num_threads, uint32_t num_stripes) noex FileStorHandlerImpl::FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg) - : FileStorHandlerImpl(1, 1, sender, metrics, compReg, SharedOperationThrottler::make_unlimited_throttler()) + : FileStorHandlerImpl(1, 1, sender, metrics, compReg, vespalib::SharedOperationThrottler::make_unlimited_throttler()) { } FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg, - std::unique_ptr<SharedOperationThrottler> operation_throttler) + std::unique_ptr<vespalib::SharedOperationThrottler> operation_throttler) : _component(compReg, "filestorhandlerimpl"), _state(FileStorHandler::AVAILABLE), _metrics(nullptr), @@ -920,7 +920,7 @@ FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) { std::unique_lock guard(*_lock); - SharedOperationThrottler::Token throttle_token; + ThrottleToken throttle_token; // Try to grab a message+lock, immediately retrying once after a wait // if none can be found and then exiting if the same is the case on the // second attempt. This is key to allowing the run loop to register @@ -997,7 +997,7 @@ FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard) FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter, - SharedOperationThrottler::Token throttle_token) + ThrottleToken throttle_token) { std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop(_metrics->averageQueueWaitingTime))); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index c4b85ac596c..698f52359f5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -149,7 +149,7 @@ public: // with its locking requirements. FileStorHandler::LockedMessage getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter, - SharedOperationThrottler::Token throttle_token); + ThrottleToken throttle_token); using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>; const FileStorHandlerImpl &_owner; MessageSender &_messageSender; @@ -191,7 +191,7 @@ public: FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg); FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender&, FileStorMetrics&, - ServiceLayerComponentRegister&, std::unique_ptr<SharedOperationThrottler>); + ServiceLayerComponentRegister&, std::unique_ptr<vespalib::SharedOperationThrottler>); ~FileStorHandlerImpl() override; void setGetNextMessageTimeout(vespalib::duration timeout) override { _getNextMessageTimeout = timeout; } @@ -243,7 +243,7 @@ public: ResumeGuard pause() override; void abortQueuedOperations(const AbortBucketOperationsCommand& cmd) override; - SharedOperationThrottler& operation_throttler() const noexcept override { + vespalib::SharedOperationThrottler& operation_throttler() const noexcept override { return *_operation_throttler; } @@ -257,7 +257,7 @@ private: ServiceLayerComponent _component; std::atomic<DiskState> _state; FileStorDiskMetrics * _metrics; - std::unique_ptr<SharedOperationThrottler> _operation_throttler; + std::unique_ptr<vespalib::SharedOperationThrottler> _operation_throttler; std::vector<Stripe> _stripes; MessageSender& _messageSender; const document::BucketIdFactory& _bucketIdFactory; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index f5b9da0e1f5..d5097f4f8e4 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -6,33 +6,36 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/doneinitializehandler.h> -#include <vespa/vdslib/state/cluster_state_bundle.h> -#include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/messagebucket.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> -#include <vespa/storage/persistence/persistencethread.h> #include <vespa/storage/persistence/persistencehandler.h> +#include <vespa/storage/persistence/persistencethread.h> #include <vespa/storage/persistence/provider_error_wrapper.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <algorithm> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> #include <thread> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.filestor.manager"); -using std::shared_ptr; using document::BucketSpace; -using vespalib::make_string_short::fmt; +using std::shared_ptr; using vespa::config::content::StorFilestorConfig; +using vespalib::CpuUsage; +using vespalib::make_string_short::fmt; namespace { @@ -71,7 +74,7 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _persistenceHandlers(), _threads(), _bucketOwnershipNotifier(std::make_unique<BucketOwnershipNotifier>(_component, *this)), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _use_async_message_handling_on_schedule(false), _metrics(std::make_unique<FileStorMetrics>()), _filestorHandler(), @@ -81,8 +84,8 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _host_info_reporter(_component.getStateUpdater()), _resource_usage_listener_registration(provider.register_resource_usage_listener(_host_info_reporter)) { - _configFetcher.subscribe(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe(configUri.getConfigId(), this); + _configFetcher->start(); _component.registerMetric(*_metrics); _component.registerStatusPage(*this); _component.getStateUpdater().addStateListener(*this); @@ -143,16 +146,31 @@ selectSequencer(StorFilestorConfig::ResponseSequencerType sequencerType) { } } -std::unique_ptr<SharedOperationThrottler> +vespalib::SharedOperationThrottler::DynamicThrottleParams +dynamic_throttle_params_from_config(const StorFilestorConfig& config, size_t num_threads) +{ + const auto& cfg_params = config.asyncOperationThrottler; + auto win_size_incr = std::max(static_cast<size_t>(std::max(cfg_params.windowSizeIncrement, 1)), num_threads); + + vespalib::SharedOperationThrottler::DynamicThrottleParams params; + params.window_size_increment = win_size_incr; + params.min_window_size = win_size_incr; + params.window_size_decrement_factor = cfg_params.windowSizeDecrementFactor; + params.window_size_backoff = cfg_params.windowSizeBackoff; + return params; +} + +std::unique_ptr<vespalib::SharedOperationThrottler> make_operation_throttler_from_config(const StorFilestorConfig& config, size_t num_threads) { - const bool use_dynamic_throttling = (config.asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC); + // TODO only use struct config field instead once config model is updated + const bool use_dynamic_throttling = ((config.asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC) || + (config.asyncOperationThrottler.type == StorFilestorConfig::AsyncOperationThrottler::Type::DYNAMIC)); if (use_dynamic_throttling) { - auto config_win_size_incr = std::max(config.asyncOperationDynamicThrottlingWindowIncrement, 1); - auto win_size_increment = std::max(static_cast<size_t>(config_win_size_incr), num_threads); - return SharedOperationThrottler::make_dynamic_throttler(win_size_increment); + auto dyn_params = dynamic_throttle_params_from_config(config, num_threads); + return vespalib::SharedOperationThrottler::make_dynamic_throttler(dyn_params); } else { - return SharedOperationThrottler::make_unlimited_throttler(); + return vespalib::SharedOperationThrottler::make_unlimited_throttler(); } } @@ -217,8 +235,9 @@ FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) _filestorHandler = std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, *this, *_metrics, _compReg, std::move(operation_throttler)); uint32_t numResponseThreads = computeNumResponseThreads(_config->numResponseThreads); - _sequencedExecutor = vespalib::SequencedTaskExecutor::create(response_executor, numResponseThreads, 10000, - selectSequencer(_config->responseSequencerType)); + _sequencedExecutor = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(response_executor, CpuUsage::Category::WRITE), + numResponseThreads, 10000, + true, selectSequencer(_config->responseSequencerType)); assert(_sequencedExecutor); LOG(spam, "Setting up the disk"); for (uint32_t i = 0; i < numThreads; i++) { @@ -227,10 +246,9 @@ FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) } _bucketExecutorRegistration = _provider->register_executor(std::make_shared<BucketExecutorWrapper>(*this)); } else { - std::lock_guard guard(_lock); - for (auto& handler : _persistenceHandlers) { - handler->configure(*config); - } + assert(_filestorHandler); + auto updated_dyn_throttle_params = dynamic_throttle_params_from_config(*config, _threads.size()); + _filestorHandler->operation_throttler().reconfigure_dynamic_throttling(updated_dyn_throttle_params); } } @@ -834,7 +852,7 @@ void FileStorManager::onClose() _bucketExecutorRegistration.reset(); _resource_usage_listener_registration.reset(); // Avoid getting config during shutdown - _configFetcher.close(); + _configFetcher->close(); LOG(debug, "Closed _configFetcher."); _filestorHandler->close(); LOG(debug, "Closed _filestorHandler."); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index b7450de13d8..83f1826c498 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -25,10 +25,12 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/ifetchercallback.h> -#include <vespa/config/config.h> +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace vespalib { class IDestructorCallback; } namespace storage { @@ -68,7 +70,7 @@ class FileStorManager : public StorageLinkQueued, std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; std::unique_ptr<vespa::config::content::StorFilestorConfig> _config; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; bool _use_async_message_handling_on_schedule; std::shared_ptr<FileStorMetrics> _metrics; std::unique_ptr<FileStorHandler> _filestorHandler; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index 6cb76c32997..80fd79fe780 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -212,7 +212,7 @@ FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, const std::str averageQueueWaitingTime("averagequeuewait.sum", {}, "Average time an operation spends in input queue.", this), queueSize("queuesize", {}, "Size of input message queue.", this), pendingMerges("pendingmerge", {}, "Number of buckets currently being merged.", this), - throttle_window_size("throttlewindowsize", {}, "Current size of async operation throttler window size", this), + throttle_window_size("throttle_window_size", {}, "Current size of async operation throttler window size", this), waitingForLockHitRate("waitingforlockrate", {}, "Amount of times a filestor thread has needed to wait for " "lock to take next message in queue.", this), diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp index 3ad34a94726..040ace55c52 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp @@ -4,6 +4,8 @@ #include "filestormanager.h" #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> #include <algorithm> #include <vespa/log/log.h> @@ -47,7 +49,7 @@ ModifiedBucketChecker::ModifiedBucketChecker( _provider(provider), _component(), _thread(), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _monitor(), _stateLock(), _bucketSpaces(), @@ -56,8 +58,8 @@ ModifiedBucketChecker::ModifiedBucketChecker( _maxPendingChunkSize(100), _singleThreadMode(false) { - _configFetcher.subscribe<vespa::config::content::core::StorServerConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorServerConfig>(configUri.getConfigId(), this); + _configFetcher->start(); std::ostringstream threadName; threadName << "Modified bucket checker " << static_cast<void*>(this); @@ -97,7 +99,9 @@ ModifiedBucketChecker::onClose() if (_singleThreadMode) { return; } - assert(_thread); + if (!_thread) { + return; // Aborted startup; onOpen() was never called so there's nothing to close. + } LOG(debug, "Interrupting modified bucket checker thread"); _thread->interrupt(); _cond.notify_one(); @@ -193,7 +197,7 @@ ModifiedBucketChecker::tick() // Do two phases of locking, as we want tick() to both fetch modified // buckets and send the first chunk for these in a single call. However, // we want getModifiedBuckets() to called outside the lock. - bool shouldRequestFromProvider = false; + bool shouldRequestFromProvider; { std::lock_guard guard(_stateLock); if (!currentChunkFinished()) { diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 6a8b9e5c008..8c0b4084428 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -9,8 +9,12 @@ #include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/types.h> #include <vespa/document/bucket/bucketidlist.h> -#include <vespa/config/config.h> +#include <vespa/config/helper/ifetchercallback.h> +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { namespace spi { struct PersistenceProvider; } @@ -80,18 +84,18 @@ private: void pop_back() { _buckets.pop_back(); } }; - spi::PersistenceProvider& _provider; - ServiceLayerComponent::UP _component; - framework::Thread::UP _thread; - config::ConfigFetcher _configFetcher; - std::mutex _monitor; - std::condition_variable _cond; - std::mutex _stateLock; - CyclicBucketSpaceIterator::UP _bucketSpaces; - BucketIdListResult _rechecksNotStarted; - size_t _pendingRequests; - size_t _maxPendingChunkSize; - bool _singleThreadMode; // For unit testing only + spi::PersistenceProvider & _provider; + ServiceLayerComponent::UP _component; + framework::Thread::UP _thread; + std::unique_ptr<config::ConfigFetcher> _configFetcher; + std::mutex _monitor; + std::condition_variable _cond; + std::mutex _stateLock; + CyclicBucketSpaceIterator::UP _bucketSpaces; + BucketIdListResult _rechecksNotStarted; + size_t _pendingRequests; + size_t _maxPendingChunkSize; + bool _singleThreadMode; // For unit testing only }; } // ns storage diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 7dcf4bcbee2..4a5362f3d8d 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -27,8 +27,7 @@ MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, const ClusterContext& cluster_context, const framework::Clock & clock, vespalib::ISequencedTaskExecutor& executor, uint32_t maxChunkSize, - uint32_t commonMergeChainOptimalizationMinimumSize, - bool async_apply_bucket_diff) + uint32_t commonMergeChainOptimalizationMinimumSize) : _clock(clock), _cluster_context(cluster_context), _env(env), @@ -37,7 +36,6 @@ MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, _monitored_ref_count(std::make_unique<MonitoredRefCount>()), _maxChunkSize(maxChunkSize), _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize), - _async_apply_bucket_diff(async_apply_bucket_diff), _executor(executor) { } @@ -1281,9 +1279,6 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra if (applyDiffHasLocallyNeededData(cmd.getDiff(), index)) { async_results = ApplyBucketDiffState::create(*this, _env._metrics.merge_handler_metrics, _clock, bucket, RetainGuard(*_monitored_ref_count)); applyDiffLocally(bucket, cmd.getDiff(), index, tracker->context(), async_results); - if (!_async_apply_bucket_diff.load(std::memory_order_relaxed)) { - check_apply_diff_sync(std::move(async_results)); - } } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u).", bucket.toString().c_str(), _env._nodeIndex, index); @@ -1388,9 +1383,6 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, Messa if (applyDiffHasLocallyNeededData(diff, index)) { async_results = ApplyBucketDiffState::create(*this, _env._metrics.merge_handler_metrics, _clock, bucket, RetainGuard(*_monitored_ref_count)); applyDiffLocally(bucket, diff, index, s->context, async_results); - if (!_async_apply_bucket_diff.load(std::memory_order_relaxed)) { - check_apply_diff_sync(std::move(async_results)); - } } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u)", bucket.toString().c_str(), @@ -1481,12 +1473,6 @@ MergeHandler::drain_async_writes() } void -MergeHandler::configure(bool async_apply_bucket_diff) noexcept -{ - _async_apply_bucket_diff.store(async_apply_bucket_diff, std::memory_order_release); -} - -void MergeHandler::schedule_delayed_delete(std::unique_ptr<ApplyBucketDiffState> state) const { auto bucket_id = state->get_bucket().getBucketId(); diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 1007f35c241..ee6eed63eb5 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -22,7 +22,10 @@ #include <vespa/vespalib/util/monitored_refcount.h> #include <atomic> -namespace vespalib { class ISequencedTaskExecutor; } +namespace vespalib { +class ISequencedTaskExecutor; +class SharedOperationThrottler; +} namespace storage { @@ -34,7 +37,6 @@ namespace spi { class PersistenceUtil; class ApplyBucketDiffState; class MergeStatus; -class SharedOperationThrottler; class MergeHandler : public Types, public MergeBucketInfoSyncer { @@ -50,8 +52,7 @@ public: const ClusterContext& cluster_context, const framework::Clock & clock, vespalib::ISequencedTaskExecutor& executor, uint32_t maxChunkSize = 4190208, - uint32_t commonMergeChainOptimalizationMinimumSize = 64, - bool async_apply_bucket_diff = false); + uint32_t commonMergeChainOptimalizationMinimumSize = 64); ~MergeHandler() override; @@ -79,7 +80,6 @@ public: MessageTrackerUP handleApplyBucketDiff(api::ApplyBucketDiffCommand&, MessageTrackerUP) const; void handleApplyBucketDiffReply(api::ApplyBucketDiffReply&, MessageSender&, MessageTrackerUP) const; void drain_async_writes(); - void configure(bool async_apply_bucket_diff) noexcept; private: using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; @@ -87,11 +87,10 @@ private: const ClusterContext &_cluster_context; PersistenceUtil &_env; spi::PersistenceProvider &_spi; - SharedOperationThrottler& _operation_throttler; + vespalib::SharedOperationThrottler& _operation_throttler; std::unique_ptr<vespalib::MonitoredRefCount> _monitored_ref_count; const uint32_t _maxChunkSize; const uint32_t _commonMergeChainOptimalizationMinimumSize; - std::atomic<bool> _async_apply_bucket_diff; vespalib::ISequencedTaskExecutor& _executor; MessageTrackerUP handleGetBucketDiffStage2(api::GetBucketDiffCommand&, MessageTrackerUP) const; diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index c0c95ffd7af..41df1e8a075 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -5,6 +5,8 @@ #include <vespa/log/log.h> LOG_SETUP(".persistence.persistencehandler"); +using vespalib::CpuUsage; + namespace storage { PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequencedExecutor, @@ -19,8 +21,7 @@ PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequen _processAllHandler(_env, provider), _mergeHandler(_env, provider, component.cluster_context(), _clock, sequencedExecutor, cfg.bucketMergeChunkSize, - cfg.commonMergeChainOptimalizationMinimumSize, - cfg.asyncApplyBucketDiff), + cfg.commonMergeChainOptimalizationMinimumSize), _asyncHandler(_env, provider, bucketOwnershipNotifier, sequencedExecutor, component.getBucketIdFactory()), _splitJoinHandler(_env, provider, bucketOwnershipNotifier, cfg.enableMultibitSplitOptimalization), _simpleHandler(_env, provider) @@ -54,7 +55,10 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr OperationSyncPhaseTrackingGuard sync_guard(*tracker); switch (msg.getType().getId()) { case api::MessageType::GET_ID: + { + auto usage = vespalib::CpuUsage::use(CpuUsage::Category::READ); return _simpleHandler.handleGet(static_cast<api::GetCommand&>(msg), std::move(tracker)); + } case api::MessageType::PUT_ID: return _asyncHandler.handlePut(static_cast<api::PutCommand&>(msg), std::move(tracker)); case api::MessageType::REMOVE_ID: @@ -87,9 +91,15 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr case api::MessageType::INTERNAL_ID: switch(static_cast<api::InternalCommand&>(msg).getType()) { case GetIterCommand::ID: + { + auto usage = vespalib::CpuUsage::use(CpuUsage::Category::READ); return _simpleHandler.handleGetIter(static_cast<GetIterCommand&>(msg), std::move(tracker)); + } case CreateIteratorCommand::ID: + { + auto usage = vespalib::CpuUsage::use(CpuUsage::Category::READ); return _simpleHandler.handleCreateIterator(static_cast<CreateIteratorCommand&>(msg), std::move(tracker)); + } case ReadBucketList::ID: return _simpleHandler.handleReadBucketList(static_cast<ReadBucketList&>(msg), std::move(tracker)); case ReadBucketInfo::ID: @@ -171,10 +181,4 @@ PersistenceHandler::processLockedMessage(FileStorHandler::LockedMessage lock) co } } -void -PersistenceHandler::configure(vespa::config::content::StorFilestorConfig& config) noexcept -{ - _mergeHandler.configure(config.asyncApplyBucketDiff); -} - } diff --git a/storage/src/vespa/storage/persistence/persistencehandler.h b/storage/src/vespa/storage/persistence/persistencehandler.h index c60fb05e56e..a92c2dc78ca 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.h +++ b/storage/src/vespa/storage/persistence/persistencehandler.h @@ -35,7 +35,6 @@ public: const AsyncHandler & asyncHandler() const { return _asyncHandler; } const SplitJoinHandler & splitjoinHandler() const { return _splitJoinHandler; } const SimpleMessageHandler & simpleMessageHandler() const { return _simpleHandler; } - void configure(vespa::config::content::StorFilestorConfig& config) noexcept; private: // Message handling functions MessageTracker::UP handleCommandSplitByType(api::StorageCommand&, MessageTracker::UP tracker) const; diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 499e9807cbf..b89c60d4720 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -16,7 +16,7 @@ PersistenceThread::PersistenceThread(PersistenceHandler & persistenceHandler, Fi _stripeId(stripeId), _thread() { - _thread = component.startThread(*this, 60s, 1s); + _thread = component.startThread(*this, 60s, 1s, 1, vespalib::CpuUsage::Category::WRITE); } PersistenceThread::~PersistenceThread() diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 65eab99b8fb..2781cc61b83 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -32,7 +32,7 @@ MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, MessageSender & replySender, FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg, - SharedOperationThrottler::Token throttle_token) + ThrottleToken throttle_token) : MessageTracker(timer, env, replySender, true, std::move(bucketLock), std::move(msg), std::move(throttle_token)) {} MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, @@ -41,7 +41,7 @@ MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, bool updateBucketInfo, FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg, - SharedOperationThrottler::Token throttle_token) + ThrottleToken throttle_token) : _sendReply(true), _updateBucketInfo(updateBucketInfo && hasBucketInfo(msg->getType().getId())), _bucketLock(std::move(bucketLock)), @@ -60,7 +60,7 @@ MessageTracker::createForTesting(const framework::MilliSecTimer & timer, Persist FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg) { return MessageTracker::UP(new MessageTracker(timer, env, replySender, false, std::move(bucketLock), - std::move(msg), SharedOperationThrottler::Token())); + std::move(msg), ThrottleToken())); } void diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 588cbef2170..4130a276239 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -31,7 +31,7 @@ public: MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, - SharedOperationThrottler::Token throttle_token); + ThrottleToken throttle_token); ~MessageTracker(); @@ -93,7 +93,7 @@ public: private: MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, - SharedOperationThrottler::Token throttle_token); + ThrottleToken throttle_token); [[nodiscard]] bool count_result_as_failure() const noexcept; @@ -101,7 +101,7 @@ private: bool _updateBucketInfo; FileStorHandler::BucketLockInterface::SP _bucketLock; std::shared_ptr<api::StorageMessage> _msg; - SharedOperationThrottler::Token _throttle_token; + ThrottleToken _throttle_token; spi::Context _context; const PersistenceUtil &_env; MessageSender &_replySender; diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp b/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp deleted file mode 100644 index 7b05decb851..00000000000 --- a/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "shared_operation_throttler.h" -#include <vespa/messagebus/dynamicthrottlepolicy.h> -#include <vespa/storage/common/dummy_mbus_messages.h> -#include <condition_variable> -#include <cassert> -#include <mutex> - -namespace storage { - -namespace { - -class NoLimitsOperationThrottler final : public SharedOperationThrottler { -public: - ~NoLimitsOperationThrottler() override = default; - Token blocking_acquire_one() noexcept override { - return Token(this, TokenCtorTag{}); - } - Token blocking_acquire_one(vespalib::duration) noexcept override { - return Token(this, TokenCtorTag{}); - } - Token try_acquire_one() noexcept override { - return Token(this, TokenCtorTag{}); - } - uint32_t current_window_size() const noexcept override { return 0; } - uint32_t waiting_threads() const noexcept override { return 0; } -private: - void release_one() noexcept override { /* no-op */ } -}; - -class DynamicOperationThrottler final : public SharedOperationThrottler { - mutable std::mutex _mutex; - std::condition_variable _cond; - mbus::DynamicThrottlePolicy _throttle_policy; - uint32_t _pending_ops; - uint32_t _waiting_threads; -public: - explicit DynamicOperationThrottler(uint32_t min_size_and_window_increment); - ~DynamicOperationThrottler() override; - - Token blocking_acquire_one() noexcept override; - Token blocking_acquire_one(vespalib::duration timeout) noexcept override; - Token try_acquire_one() noexcept override; - uint32_t current_window_size() const noexcept override; - uint32_t waiting_threads() const noexcept override; -private: - void release_one() noexcept override; - // Non-const since actually checking the send window of a dynamic throttler might change - // it if enough time has passed. - [[nodiscard]] bool has_spare_capacity_in_active_window() noexcept; - void add_one_to_active_window_size(); - void subtract_one_from_active_window_size(); -}; - -DynamicOperationThrottler::DynamicOperationThrottler(uint32_t min_size_and_window_increment) - : _mutex(), - _cond(), - _throttle_policy(static_cast<double>(min_size_and_window_increment)), - _pending_ops(0), - _waiting_threads(0) -{ -} - -DynamicOperationThrottler::~DynamicOperationThrottler() = default; - -bool -DynamicOperationThrottler::has_spare_capacity_in_active_window() noexcept -{ - DummyMbusRequest dummy_request; - return _throttle_policy.canSend(dummy_request, _pending_ops); -} - -void -DynamicOperationThrottler::add_one_to_active_window_size() -{ - DummyMbusRequest dummy_request; - _throttle_policy.processMessage(dummy_request); - ++_pending_ops; -} - -void -DynamicOperationThrottler::subtract_one_from_active_window_size() -{ - DummyMbusReply dummy_reply; - _throttle_policy.processReply(dummy_reply); - assert(_pending_ops > 0); - --_pending_ops; -} - -DynamicOperationThrottler::Token -DynamicOperationThrottler::blocking_acquire_one() noexcept -{ - std::unique_lock lock(_mutex); - if (!has_spare_capacity_in_active_window()) { - ++_waiting_threads; - _cond.wait(lock, [&] { - return has_spare_capacity_in_active_window(); - }); - --_waiting_threads; - } - add_one_to_active_window_size(); - return Token(this, TokenCtorTag{}); -} - -DynamicOperationThrottler::Token -DynamicOperationThrottler::blocking_acquire_one(vespalib::duration timeout) noexcept -{ - std::unique_lock lock(_mutex); - if (!has_spare_capacity_in_active_window()) { - ++_waiting_threads; - const bool accepted = _cond.wait_for(lock, timeout, [&] { - return has_spare_capacity_in_active_window(); - }); - --_waiting_threads; - if (!accepted) { - return Token(); - } - } - add_one_to_active_window_size(); - return Token(this, TokenCtorTag{}); -} - -DynamicOperationThrottler::Token -DynamicOperationThrottler::try_acquire_one() noexcept -{ - std::unique_lock lock(_mutex); - if (!has_spare_capacity_in_active_window()) { - return Token(); - } - add_one_to_active_window_size(); - return Token(this, TokenCtorTag{}); -} - -void -DynamicOperationThrottler::release_one() noexcept -{ - std::unique_lock lock(_mutex); - subtract_one_from_active_window_size(); - // Only wake up a waiting thread if doing so would possibly result in success. - if ((_waiting_threads > 0) && has_spare_capacity_in_active_window()) { - lock.unlock(); - _cond.notify_one(); - } -} - -uint32_t -DynamicOperationThrottler::current_window_size() const noexcept -{ - std::unique_lock lock(_mutex); - return _throttle_policy.getMaxPendingCount(); // Actually returns current window size -} - -uint32_t -DynamicOperationThrottler::waiting_threads() const noexcept -{ - std::unique_lock lock(_mutex); - return _waiting_threads; -} - -} - -std::unique_ptr<SharedOperationThrottler> -SharedOperationThrottler::make_unlimited_throttler() -{ - return std::make_unique<NoLimitsOperationThrottler>(); -} - -std::unique_ptr<SharedOperationThrottler> -SharedOperationThrottler::make_dynamic_throttler(uint32_t min_size_and_window_increment) -{ - return std::make_unique<DynamicOperationThrottler>(min_size_and_window_increment); -} - -DynamicOperationThrottler::Token::~Token() -{ - if (_throttler) { - _throttler->release_one(); - } -} - -void -DynamicOperationThrottler::Token::reset() noexcept -{ - if (_throttler) { - _throttler->release_one(); - _throttler = nullptr; - } -} - -DynamicOperationThrottler::Token& -DynamicOperationThrottler::Token::operator=(Token&& rhs) noexcept -{ - reset(); - _throttler = rhs._throttler; - rhs._throttler = nullptr; - return *this; -} - -} diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.h b/storage/src/vespa/storage/persistence/shared_operation_throttler.h index 4ee8d017c05..b829f077bcb 100644 --- a/storage/src/vespa/storage/persistence/shared_operation_throttler.h +++ b/storage/src/vespa/storage/persistence/shared_operation_throttler.h @@ -1,72 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/time.h> -#include <memory> -#include <optional> +#include <vespa/vespalib/util/shared_operation_throttler.h> namespace storage { -/** - * Operation throttler that is intended to provide global throttling of - * async operations across all persistence stripe threads. A throttler - * wraps a logical max pending window size of in-flight operations. Depending - * on the throttler implementation, the window size may expand and shrink - * dynamically. Exactly how and when this happens is unspecified. - * - * Offers both polling and (timed, non-timed) blocking calls for acquiring - * a throttle token. If the returned token is valid, the caller may proceed - * to invoke the asynchronous operation. - * - * The window slot taken up by a valid throttle token is implicitly freed up - * when the token is destroyed. - * - * All operations on the throttler are thread safe. - */ -class SharedOperationThrottler { -protected: - struct TokenCtorTag {}; // Make available to subclasses for token construction. -public: - class Token { - SharedOperationThrottler* _throttler; - public: - constexpr Token(SharedOperationThrottler* throttler, TokenCtorTag) noexcept : _throttler(throttler) {} - constexpr Token() noexcept : _throttler(nullptr) {} - constexpr Token(Token&& rhs) noexcept - : _throttler(rhs._throttler) - { - rhs._throttler = nullptr; - } - Token& operator=(Token&& rhs) noexcept; - ~Token(); - - Token(const Token&) = delete; - Token& operator=(const Token&) = delete; - - [[nodiscard]] constexpr bool valid() const noexcept { return (_throttler != nullptr); } - void reset() noexcept; - }; - - virtual ~SharedOperationThrottler() = default; - - // All methods are thread safe - [[nodiscard]] virtual Token blocking_acquire_one() noexcept = 0; - [[nodiscard]] virtual Token blocking_acquire_one(vespalib::duration timeout) noexcept = 0; - [[nodiscard]] virtual Token try_acquire_one() noexcept = 0; - - // May return 0, in which case the window size is unlimited. - [[nodiscard]] virtual uint32_t current_window_size() const noexcept = 0; - - // Exposed for unit testing only. - [[nodiscard]] virtual uint32_t waiting_threads() const noexcept = 0; - - // Creates a throttler that does exactly zero throttling (but also has zero overhead and locking) - static std::unique_ptr<SharedOperationThrottler> make_unlimited_throttler(); - // Creates a throttler that uses a MessageBus DynamicThrottlePolicy under the hood - static std::unique_ptr<SharedOperationThrottler> make_dynamic_throttler(uint32_t min_size_and_window_increment); -private: - // Exclusively called from a valid Token. Thread safe. - virtual void release_one() noexcept = 0; -}; +using ThrottleToken = vespalib::SharedOperationThrottler::Token; } diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index ea14a1b7492..5241a1a88dd 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> #include <sstream> @@ -27,7 +28,7 @@ Bouncer::Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & co _baselineNodeState("s:i"), _derivedNodeStates(), _clusterState(&lib::State::UP), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _metrics(std::make_unique<BouncerMetrics>()) { _component.getStateUpdater().addStateListener(*this); @@ -36,8 +37,8 @@ Bouncer::Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & co // exception allowing program to continue if missing/faulty config. try{ if (!configUri.empty()) { - _configFetcher.subscribe<vespa::config::content::core::StorBouncerConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorBouncerConfig>(configUri.getConfigId(), this); + _configFetcher->start(); } else { LOG(info, "No config id specified. Using defaults rather than " "config"); @@ -66,7 +67,7 @@ Bouncer::print(std::ostream& out, bool verbose, void Bouncer::onClose() { - _configFetcher.close(); + _configFetcher->close(); _component.getStateUpdater().removeStateListener(*this); } diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index e6ddfe7ad49..50a08387b39 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -11,7 +11,7 @@ #pragma once -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/vdslib/state/nodestate.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/storagecomponent.h> @@ -19,7 +19,10 @@ #include <vespa/storage/config/config-stor-bouncer.h> #include <unordered_map> -namespace config { class ConfigUri; } +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { @@ -36,7 +39,7 @@ class Bouncer : public StorageLink, using BucketSpaceNodeStateMapping = std::unordered_map<document::BucketSpace, lib::NodeState, document::BucketSpace::hash>; BucketSpaceNodeStateMapping _derivedNodeStates; const lib::State* _clusterState; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; std::unique_ptr<BouncerMetrics> _metrics; public: @@ -47,36 +50,20 @@ public: const std::string& indent) const override; void configure(std::unique_ptr<vespa::config::content::core::StorBouncerConfig> config) override; - const BouncerMetrics& metrics() const noexcept; private: - void validateConfig( - const vespa::config::content::core::StorBouncerConfig&) const; - + void validateConfig(const vespa::config::content::core::StorBouncerConfig&) const; void onClose() override; - - void abortCommandForUnavailableNode(api::StorageMessage&, - const lib::State&); - - void rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, - int maxClockSkewInSeconds); - + void abortCommandForUnavailableNode(api::StorageMessage&, const lib::State&); + void rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, int maxClockSkewInSeconds); void abortCommandDueToClusterDown(api::StorageMessage&); - - void rejectDueToInsufficientPriority(api::StorageMessage&, - api::StorageMessage::Priority); - + void rejectDueToInsufficientPriority(api::StorageMessage&, api::StorageMessage::Priority); void reject_due_to_too_few_bucket_bits(api::StorageMessage&); - bool clusterIsUp() const; - bool isDistributor() const; - bool isExternalLoad(const api::MessageType&) const noexcept; - bool isExternalWriteOperation(const api::MessageType&) const noexcept; - bool priorityRejectionIsEnabled(int configuredPriority) const noexcept { return (configuredPriority != -1); } @@ -86,12 +73,9 @@ private: * update commands), return that timestamp. Otherwise, return 0. */ uint64_t extractMutationTimestampIfAny(const api::StorageMessage& msg); - bool onDown(const std::shared_ptr<api::StorageMessage>&) override; - void handleNewState() override; const lib::NodeState &getDerivedNodeState(document::BucketSpace bucketSpace) const; - void append_node_identity(std::ostream& target_stream) const; }; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index ee58710e585..9d7dd95d922 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -11,6 +11,9 @@ #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/metrics/metrictimer.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> + #include <vespa/log/bufferedlogger.h> @@ -24,7 +27,7 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( : StorageLink("Changed bucket ownership handler"), _component(compReg, "changedbucketownershiphandler"), _metrics(), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _stateLock(), _currentState(), // Not set yet, so ownership will not be valid _currentOwnership(std::make_shared<OwnershipState>( @@ -33,8 +36,8 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( _abortMutatingIdealStateOps(false), _abortMutatingExternalLoadOps(false) { - _configFetcher.subscribe<vespa::config::content::PersistenceConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::PersistenceConfig>(configUri.getConfigId(), this); + _configFetcher->start(); _component.registerMetric(_metrics); } diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index ddf8516ed6e..e753d96871e 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -3,8 +3,8 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/storage/common/storagelink.h> -#include <vespa/config/config.h> #include <vespa/config-persistence.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/persistence/messages.h> #include <vespa/metrics/valuemetric.h> @@ -14,6 +14,10 @@ #include <vector> #include <unordered_map> +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { namespace lib { @@ -112,7 +116,7 @@ public: private: ServiceLayerComponent _component; Metrics _metrics; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; mutable std::mutex _stateLock; std::shared_ptr<const lib::ClusterStateBundle> _currentState; OwnershipState::CSP _currentOwnership; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 9b67830b3dc..237dc76d783 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -19,6 +19,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/config/helper/configfetcher.hpp> #include <string_view> #include <vespa/log/bufferedlogger.h> diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index c899cee3e14..80117b32030 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -21,15 +21,18 @@ #include <vespa/storageapi/mbusprot/storagereply.h> #include <vespa/messagebus/imessagehandler.h> #include <vespa/messagebus/ireplyhandler.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/config/subscription/configuri.h> +#include <vespa/config-bucketspaces.h> #include <map> #include <queue> #include <atomic> #include <mutex> -#include <vespa/config-bucketspaces.h> +namespace config { + class ConfigFetcher; +} namespace mbus { class RPCMessageBus; class SourceSession; diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index b7016220531..6fd8cded08e 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -26,9 +26,7 @@ DistributorNode::DistributorNode( : StorageNode(configUri, context, generationFetcher, std::make_unique<HostInfo>(), !communicationManager ? NORMAL : SINGLE_THREADED_TEST_MODE), - // TODO STRIPE: Change waitTime default to 100ms when legacy mode is removed. - _threadPool(framework::TickingThreadPool::createDefault("distributor", - (num_distributor_stripes > 0) ? 100ms : 5ms)), + _threadPool(framework::TickingThreadPool::createDefault("distributor", 100ms, 1, 5s)), _stripe_pool(std::make_unique<distributor::DistributorStripePool>()), _context(context), _timestamp_mutex(), @@ -72,9 +70,6 @@ DistributorNode::handleConfigChange(vespa::config::content::core::StorDistributo { framework::TickingLockGuard guard(_threadPool->freezeAllTicks()); _context.getComponentRegister().setDistributorConfig(c); - _threadPool->updateParametersAllThreads(std::chrono::milliseconds(c.ticksWaitTimeMs), - std::chrono::milliseconds(c.maxProcessTimeMs), - c.ticksBeforeWait); } void diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 2a30acb1a74..28a76413149 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -5,17 +5,20 @@ #include <vespa/storage/common/dummy_mbus_messages.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/messagebus/message.h> +#include <vespa/messagebus/dynamicthrottlepolicy.h> #include <vespa/messagebus/error.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configfetcher.hpp> +#include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> -#include <algorithm> #include <cassert> #include <vespa/log/log.h> LOG_SETUP(".mergethrottler"); +using vespa::config::content::core::StorServerConfig; + namespace storage { namespace { @@ -126,7 +129,7 @@ MergeThrottler::MergeNodeSequence::MergeNodeSequence(const api::MergeBucketComma } uint16_t -MergeThrottler::MergeNodeSequence::getNextNodeInChain() const +MergeThrottler::MergeNodeSequence::getNextNodeInChain() const noexcept { assert(_cmd.getChain().size() < _sortedNodes.size()); if (_use_unordered_forwarding) { @@ -144,7 +147,7 @@ MergeThrottler::MergeNodeSequence::getNextNodeInChain() const } bool -MergeThrottler::MergeNodeSequence::isChainCompleted() const +MergeThrottler::MergeNodeSequence::isChainCompleted() const noexcept { if (_cmd.getChain().size() != _sortedNodes.size()) return false; @@ -179,23 +182,25 @@ MergeThrottler::MergeThrottler( _merges(), _queue(), _maxQueueSize(1024), - _throttlePolicy(std::make_unique<mbus::StaticThrottlePolicy>()), + _throttlePolicy(std::make_unique<mbus::DynamicThrottlePolicy>()), _queueSequence(0), _messageLock(), _stateLock(), - _configFetcher(configUri.getContext()), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _metrics(std::make_unique<Metrics>()), _component(compReg, "mergethrottler"), _thread(), - _rendezvous(RENDEZVOUS_NONE), + _rendezvous(RendezvousState::NONE), _throttle_until_time(), _backpressure_duration(std::chrono::seconds(30)), + _use_dynamic_throttling(false), _disable_queue_limits_for_chained_merges(false), _closing(false) { - _throttlePolicy->setMaxPendingCount(20); - _configFetcher.subscribe<vespa::config::content::core::StorServerConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _throttlePolicy->setMinWindowSize(20); + _throttlePolicy->setMaxWindowSize(20); + _configFetcher->subscribe<StorServerConfig>(configUri.getConfigId(), this); + _configFetcher->start(); _component.registerStatusPage(*this); _component.registerMetric(*_metrics); } @@ -204,7 +209,8 @@ void MergeThrottler::configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) { std::lock_guard lock(_stateLock); - + _use_dynamic_throttling = (newConfig->mergeThrottlingPolicy.type + == StorServerConfig::MergeThrottlingPolicy::Type::DYNAMIC); if (newConfig->maxMergesPerNode < 1) { throw config::InvalidConfigException("Cannot have a max merge count of less than 1"); } @@ -214,12 +220,22 @@ MergeThrottler::configure(std::unique_ptr<vespa::config::content::core::StorServ if (newConfig->resourceExhaustionMergeBackPressureDurationSecs < 0.0) { throw config::InvalidConfigException("Merge back-pressure duration cannot be less than 0"); } - if (static_cast<double>(newConfig->maxMergesPerNode) - != _throttlePolicy->getMaxPendingCount()) - { - LOG(debug, "Setting new max pending count from max_merges_per_node: %d", - newConfig->maxMergesPerNode); - _throttlePolicy->setMaxPendingCount(newConfig->maxMergesPerNode); + if (_use_dynamic_throttling) { + auto min_win_sz = std::max(newConfig->mergeThrottlingPolicy.minWindowSize, 1); + auto max_win_sz = std::max(newConfig->mergeThrottlingPolicy.maxWindowSize, 1); + if (min_win_sz > max_win_sz) { + min_win_sz = max_win_sz; + } + auto win_sz_increment = std::max(1.0, newConfig->mergeThrottlingPolicy.windowSizeIncrement); + _throttlePolicy->setMinWindowSize(min_win_sz); + _throttlePolicy->setMaxWindowSize(max_win_sz); + _throttlePolicy->setWindowSizeIncrement(win_sz_increment); + LOG(debug, "Using dynamic throttling window min/max [%d, %d], win size increment %.2g", + min_win_sz, max_win_sz, win_sz_increment); + } else { + // Use legacy config values when static throttling is enabled. + _throttlePolicy->setMinWindowSize(newConfig->maxMergesPerNode); + _throttlePolicy->setMaxWindowSize(newConfig->maxMergesPerNode); } LOG(debug, "Setting new max queue size to %d", newConfig->maxMergeQueueSize); @@ -256,7 +272,7 @@ void MergeThrottler::onClose() { // Avoid getting config on shutdown - _configFetcher.close(); + _configFetcher->close(); { std::lock_guard guard(_messageLock); // Note: used to prevent taking locks in different order if onFlush @@ -573,16 +589,16 @@ MergeThrottler::processQueuedMerges(MessageGuard& msgGuard) void MergeThrottler::handleRendezvous(std::unique_lock<std::mutex> & guard, std::condition_variable & cond) { - if (_rendezvous != RENDEZVOUS_NONE) { + if (_rendezvous != RendezvousState::NONE) { LOG(spam, "rendezvous requested by external thread; establishing"); - assert(_rendezvous == RENDEZVOUS_REQUESTED); - _rendezvous = RENDEZVOUS_ESTABLISHED; + assert(_rendezvous == RendezvousState::REQUESTED); + _rendezvous = RendezvousState::ESTABLISHED; cond.notify_all(); - while (_rendezvous != RENDEZVOUS_RELEASED) { + while (_rendezvous != RendezvousState::RELEASED) { cond.wait(guard); } LOG(spam, "external thread rendezvous released"); - _rendezvous = RENDEZVOUS_NONE; + _rendezvous = RendezvousState::NONE; cond.notify_all(); } } @@ -604,7 +620,7 @@ MergeThrottler::run(framework::ThreadHandle& thread) while (_messagesDown.empty() && _messagesUp.empty() && !thread.interrupted() - && _rendezvous == RENDEZVOUS_NONE) + && _rendezvous == RendezvousState::NONE) { _messageCond.wait_for(msgLock, 1000ms); thread.registerTick(framework::WAIT_CYCLE); @@ -683,10 +699,20 @@ bool MergeThrottler::backpressure_mode_active() const { return backpressure_mode_active_no_lock(); } -bool MergeThrottler::allow_merge_despite_full_window(const api::MergeBucketCommand& cmd) noexcept { +bool MergeThrottler::allow_merge_despite_full_window(const api::MergeBucketCommand& cmd) const noexcept { // We cannot let forwarded unordered merges fall into the queue, as that might lead to a deadlock. // See comment in may_allow_into_queue() for rationale. - return (cmd.use_unordered_forwarding() && !cmd.from_distributor()); + if (!cmd.use_unordered_forwarding() || cmd.from_distributor()) { + return false; + } + // We'll only get here if we're dealing with an unordered merge that has been forwarded + // from another content node. In other words, it's a merge we want to handle immediately + // instead of deferring in the queue for later processing. We already know that the merge + // window is full, so we must either allow it in regardless or bounce it back. The latter + // makes the most sense when dynamic throttling is enabled, as NACKed replies count + // _against_ incrementing the throttling window, thereby implicitly helping to reduce the + // merge pressure generated by other nodes. + return !_use_dynamic_throttling; } bool MergeThrottler::may_allow_into_queue(const api::MergeBucketCommand& cmd) const noexcept { @@ -1155,10 +1181,10 @@ void MergeThrottler::rendezvousWithWorkerThread(std::unique_lock<std::mutex> & guard, std::condition_variable & cond) { LOG(spam, "establishing rendezvous with worker thread"); - assert(_rendezvous == RENDEZVOUS_NONE); - _rendezvous = RENDEZVOUS_REQUESTED; + assert(_rendezvous == RendezvousState::NONE); + _rendezvous = RendezvousState::REQUESTED; cond.notify_all(); - while (_rendezvous != RENDEZVOUS_ESTABLISHED) { + while (_rendezvous != RendezvousState::ESTABLISHED) { cond.wait(guard); } LOG(spam, "rendezvous established with worker thread"); @@ -1167,9 +1193,9 @@ MergeThrottler::rendezvousWithWorkerThread(std::unique_lock<std::mutex> & guard, void MergeThrottler::releaseWorkerThreadRendezvous(std::unique_lock<std::mutex> & guard, std::condition_variable & cond) { - _rendezvous = RENDEZVOUS_RELEASED; + _rendezvous = RendezvousState::RELEASED; cond.notify_all(); - while (_rendezvous != RENDEZVOUS_NONE) { + while (_rendezvous != RendezvousState::NONE) { cond.wait(guard); } } @@ -1288,57 +1314,62 @@ MergeThrottler::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const { std::lock_guard lock(_stateLock); - { - out << "<p>Max pending: " + if (_use_dynamic_throttling) { + out << "<p>Dynamic throttle policy; window size min/max: [" + << _throttlePolicy->getMinWindowSize() << ", " + << _throttlePolicy->getMaxWindowSize() + << "], current window size: " + << _throttlePolicy->getMaxPendingCount() + << "</p>\n"; + } else { + out << "<p>Static throttle policy; max pending: " << _throttlePolicy->getMaxPendingCount() << "</p>\n"; - out << "<p>Please see node metrics for performance numbers</p>\n"; - out << "<h3>Active merges (" - << _merges.size() - << ")</h3>\n"; - if (!_merges.empty()) { - out << "<ul>\n"; - for (auto& m : _merges) { - out << "<li>" << m.second.getMergeCmdString(); - if (m.second.isExecutingLocally()) { - out << " <strong>("; - if (m.second.isInCycle()) { - out << "cycled - "; - } else if (m.second.isCycleBroken()) { - out << "broken cycle (another node in the chain likely went down) - "; - } - out << "executing on this node)</strong>"; - } else if (m.second.isUnwinding()) { - out << " <strong>(was executed here, now unwinding)</strong>"; - } - if (m.second.isAborted()) { - out << " <strong>aborted</strong>"; + } + out << "<p>Please see node metrics for performance numbers</p>\n"; + out << "<h3>Active merges (" + << _merges.size() + << ")</h3>\n"; + if (!_merges.empty()) { + out << "<ul>\n"; + for (auto& m : _merges) { + out << "<li>" << m.second.getMergeCmdString(); + if (m.second.isExecutingLocally()) { + out << " <strong>("; + if (m.second.isInCycle()) { + out << "cycled - "; + } else if (m.second.isCycleBroken()) { + out << "broken cycle (another node in the chain likely went down) - "; } - out << "</li>\n"; + out << "executing on this node)</strong>"; + } else if (m.second.isUnwinding()) { + out << " <strong>(was executed here, now unwinding)</strong>"; } - out << "</ul>\n"; - } else { - out << "<p>None</p>\n"; + if (m.second.isAborted()) { + out << " <strong>aborted</strong>"; + } + out << "</li>\n"; } + out << "</ul>\n"; + } else { + out << "<p>None</p>\n"; } - { - out << "<h3>Queued merges (in priority order) (" - << _queue.size() - << ")</h3>\n"; - if (!_queue.empty()) { - out << "<ol>\n"; - for (auto& qm : _queue) { - // The queue always owns its messages, thus this is safe - out << "<li>Pri " - << static_cast<unsigned int>(qm._msg->getPriority()) - << ": " << *qm._msg; - out << "</li>\n"; - } - out << "</ol>\n"; - } else { - out << "<p>None</p>\n"; + out << "<h3>Queued merges (in priority order) (" + << _queue.size() + << ")</h3>\n"; + if (!_queue.empty()) { + out << "<ol>\n"; + for (auto& qm : _queue) { + // The queue always owns its messages, thus this is safe + out << "<li>Pri " + << static_cast<unsigned int>(qm._msg->getPriority()) + << ": " << *qm._msg; + out << "</li>\n"; } + out << "</ol>\n"; + } else { + out << "<p>None</p>\n"; } } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index c115d36ad89..e501e0edd39 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -15,15 +15,20 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/document/bucket/bucket.h> #include <vespa/vespalib/util/document_runnable.h> -#include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/metrics/metricset.h> #include <vespa/metrics/summetric.h> #include <vespa/metrics/countmetric.h> #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/metrictimer.h> -#include <vespa/config/config.h> +#include <vespa/config/helper/ifetchercallback.h> + #include <chrono> +namespace mbus { class DynamicThrottlePolicy; } +namespace config { + class ConfigFetcher; + class ConfigUri; +} namespace storage { class AbortBucketOperationsCommand; @@ -47,7 +52,7 @@ public: metrics::LongCountMetric rejected; metrics::LongCountMetric other; - MergeFailureMetrics(metrics::MetricSet* owner); + explicit MergeFailureMetrics(metrics::MetricSet* owner); ~MergeFailureMetrics() override; }; @@ -69,8 +74,8 @@ public: MergeOperationMetrics chaining; MergeOperationMetrics local; - Metrics(metrics::MetricSet* owner = 0); - ~Metrics(); + explicit Metrics(metrics::MetricSet* owner = nullptr); + ~Metrics() override; }; private: @@ -114,14 +119,14 @@ private: bool _aborted; ChainedMergeState(); - ChainedMergeState(const api::StorageMessage::SP& cmd, bool executing = false); + explicit ChainedMergeState(const api::StorageMessage::SP& cmd, bool executing = false); ~ChainedMergeState(); // Use default copy-constructor/assignment operator - bool isExecutingLocally() const { return _executingLocally; } - void setExecutingLocally(bool execLocally) { _executingLocally = execLocally; } + bool isExecutingLocally() const noexcept { return _executingLocally; } + void setExecutingLocally(bool execLocally) noexcept { _executingLocally = execLocally; } - const api::StorageMessage::SP& getMergeCmd() const { return _cmd; } + const api::StorageMessage::SP& getMergeCmd() const noexcept { return _cmd; } void setMergeCmd(const api::StorageMessage::SP& cmd) { _cmd = cmd; if (cmd.get()) { @@ -129,45 +134,45 @@ private: } } - bool isInCycle() const { return _inCycle; } - void setInCycle(bool inCycle) { _inCycle = inCycle; } + bool isInCycle() const noexcept { return _inCycle; } + void setInCycle(bool inCycle) noexcept { _inCycle = inCycle; } - bool isUnwinding() const { return _unwinding; } - void setUnwinding(bool unwinding) { _unwinding = unwinding; } + bool isUnwinding() const noexcept { return _unwinding; } + void setUnwinding(bool unwinding) noexcept { _unwinding = unwinding; } - bool isCycleBroken() const { return _cycleBroken; } - void setCycleBroken(bool cycleBroken) { _cycleBroken = cycleBroken; } + bool isCycleBroken() const noexcept { return _cycleBroken; } + void setCycleBroken(bool cycleBroken) noexcept { _cycleBroken = cycleBroken; } - bool isAborted() const { return _aborted; } - void setAborted(bool aborted) { _aborted = aborted; } + bool isAborted() const noexcept { return _aborted; } + void setAborted(bool aborted) noexcept { _aborted = aborted; } - const std::string& getMergeCmdString() const { return _cmdString; } + const std::string& getMergeCmdString() const noexcept { return _cmdString; } }; - typedef std::map<document::Bucket, ChainedMergeState> ActiveMergeMap; + using ActiveMergeMap = std::map<document::Bucket, ChainedMergeState>; // Use a set rather than a priority_queue, since we want to be // able to iterate over the collection during status rendering - typedef std::set< + using MergePriorityQueue = std::set< StablePriorityOrderingWrapper<api::StorageMessage::SP> - > MergePriorityQueue; + >; - enum RendezvousState { - RENDEZVOUS_NONE, - RENDEZVOUS_REQUESTED, - RENDEZVOUS_ESTABLISHED, - RENDEZVOUS_RELEASED + enum class RendezvousState { + NONE, + REQUESTED, + ESTABLISHED, + RELEASED }; ActiveMergeMap _merges; MergePriorityQueue _queue; size_t _maxQueueSize; - mbus::StaticThrottlePolicy::UP _throttlePolicy; + std::unique_ptr<mbus::DynamicThrottlePolicy> _throttlePolicy; uint64_t _queueSequence; // TODO: move into a stable priority queue class mutable std::mutex _messageLock; std::condition_variable _messageCond; mutable std::mutex _stateLock; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; // Messages pending to be processed by the worker thread std::vector<api::StorageMessage::SP> _messagesDown; std::vector<api::StorageMessage::SP> _messagesUp; @@ -177,6 +182,7 @@ private: RendezvousState _rendezvous; mutable std::chrono::steady_clock::time_point _throttle_until_time; std::chrono::steady_clock::duration _backpressure_duration; + bool _use_dynamic_throttling; bool _disable_queue_limits_for_chained_merges; bool _closing; public: @@ -213,8 +219,8 @@ public: // For unit testing only const MergePriorityQueue& getMergeQueue() const { return _queue; } // For unit testing only - const mbus::StaticThrottlePolicy& getThrottlePolicy() const { return *_throttlePolicy; } - mbus::StaticThrottlePolicy& getThrottlePolicy() { return *_throttlePolicy; } + const mbus::DynamicThrottlePolicy& getThrottlePolicy() const { return *_throttlePolicy; } + mbus::DynamicThrottlePolicy& getThrottlePolicy() { return *_throttlePolicy; } void set_disable_queue_limits_for_chained_merges(bool disable_limits) noexcept; // For unit testing only std::mutex& getStateLock() { return _stateLock; } @@ -237,26 +243,26 @@ private: MergeNodeSequence(const api::MergeBucketCommand& cmd, uint16_t thisIndex); - const std::vector<api::MergeBucketCommand::Node>& getSortedNodes() const { + [[nodiscard]] const std::vector<api::MergeBucketCommand::Node>& getSortedNodes() const noexcept { return _sortedNodes; } - bool isIndexUnknown() const { + [[nodiscard]] bool isIndexUnknown() const noexcept { return (_sortedIndex == UINT16_MAX); } /** * This node is the merge executor if it's the first element in the * _unsorted_ node sequence. */ - bool isMergeExecutor() const { + [[nodiscard]] bool isMergeExecutor() const noexcept { return (_cmd.getNodes()[0].index == _thisIndex); } - uint16_t getExecutorNodeIndex() const{ + [[nodiscard]] uint16_t getExecutorNodeIndex() const noexcept { return _cmd.getNodes()[0].index; } - const std::vector<api::MergeBucketCommand::Node>& unordered_nodes() const noexcept { + [[nodiscard]] const std::vector<api::MergeBucketCommand::Node>& unordered_nodes() const noexcept { return _cmd.getNodes(); } - [[nodiscard]] bool isLastNode() const { + [[nodiscard]] bool isLastNode() const noexcept { if (!_use_unordered_forwarding) { return (_sortedIndex == _sortedNodes.size() - 1); } else { @@ -267,13 +273,13 @@ private: /** * Gets node to forward to in strictly increasing order. */ - uint16_t getNextNodeInChain() const; + [[nodiscard]] uint16_t getNextNodeInChain() const noexcept; /** * Returns true iff the chain vector (which is implicitly sorted) * pairwise compares equally to the vector of sorted node indices */ - bool isChainCompleted() const; + [[nodiscard]] bool isChainCompleted() const noexcept; }; /** @@ -359,7 +365,7 @@ private: [[nodiscard]] bool merge_has_this_node_as_source_only_node(const api::MergeBucketCommand& cmd) const; [[nodiscard]] bool backpressure_mode_active_no_lock() const; void backpressure_bounce_all_queued_merges(MessageGuard& guard); - [[nodiscard]] static bool allow_merge_despite_full_window(const api::MergeBucketCommand& cmd) noexcept; + [[nodiscard]] bool allow_merge_despite_full_window(const api::MergeBucketCommand& cmd) const noexcept; [[nodiscard]] bool may_allow_into_queue(const api::MergeBucketCommand& cmd) const noexcept; void sendReply(const api::MergeBucketCommand& cmd, diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index 244813775a1..03322cb55fd 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -2,6 +2,8 @@ #include "opslogger.h" #include <vespa/storageapi/message/persistence.h> +#include <vespa/config/helper/configfetcher.hpp> +#include <vespa/config/subscription/configuri.h> #include <sstream> #include <vespa/log/log.h> @@ -16,10 +18,10 @@ OpsLogger::OpsLogger(StorageComponentRegister& compReg, _fileName(), _targetFile(nullptr), _component(compReg, "opslogger"), - _configFetcher(configUri.getContext()) + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) { - _configFetcher.subscribe<vespa::config::content::core::StorOpsloggerConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorOpsloggerConfig>(configUri.getConfigId(), this); + _configFetcher->start(); } OpsLogger::~OpsLogger() @@ -35,8 +37,8 @@ OpsLogger::~OpsLogger() void OpsLogger::onClose() { - // Avoid getting config during shutdown - _configFetcher.close(); + // Avoid getting config during shutdown + _configFetcher->close(); } void diff --git a/storage/src/vespa/storage/storageserver/opslogger.h b/storage/src/vespa/storage/storageserver/opslogger.h index cf39f51279d..039cb72969e 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.h +++ b/storage/src/vespa/storage/storageserver/opslogger.h @@ -13,7 +13,12 @@ #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/message/state.h> #include <vespa/storage/config/config-stor-opslogger.h> -#include <vespa/config/config.h> +#include <vespa/config/helper/ifetchercallback.h> + +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { @@ -40,7 +45,7 @@ private: FILE * _targetFile; framework::Component _component; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; }; } diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.cpp b/storage/src/vespa/storage/storageserver/priorityconverter.cpp index 1c11edea796..49297f216ca 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.cpp +++ b/storage/src/vespa/storage/storageserver/priorityconverter.cpp @@ -1,16 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "priorityconverter.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configfetcher.hpp> + namespace storage { PriorityConverter::PriorityConverter(const config::ConfigUri & configUri) - : _configFetcher(configUri.getContext()) + : _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) { - _configFetcher.subscribe<vespa::config::content::core::StorPrioritymappingConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorPrioritymappingConfig>(configUri.getConfigId(), this); + _configFetcher->start(); } PriorityConverter::~PriorityConverter() = default; diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.h b/storage/src/vespa/storage/storageserver/priorityconverter.h index 0dcd66645cb..b1f774e4ff2 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.h +++ b/storage/src/vespa/storage/storageserver/priorityconverter.h @@ -3,13 +3,16 @@ #pragma once #include <vespa/storage/config/config-stor-prioritymapping.h> -#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/documentapi/messagebus/priority.h> #include <atomic> #include <array> #include <mutex> -namespace config {class ConfigUri; } +namespace config { + class ConfigUri; + class ConfigFetcher; +} namespace storage { @@ -42,7 +45,7 @@ private: std::map<uint8_t, documentapi::Priority::Value> _reverseMapping; mutable std::mutex _mutex; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; }; } // storage diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index 8f36e1d8748..f0189f943ab 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -13,7 +13,6 @@ #include "storagenode.h" #include <vespa/storage/visiting/visitormessagesessionfactory.h> #include <vespa/storage/common/visitorfactory.h> -#include <vespa/config/config.h> namespace storage { diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index c19fca8c58c..34db3b102e7 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -14,6 +14,7 @@ #include <vespa/storage/common/storage_chain_builder.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> +#include <vespa/config/helper/configfetcher.hpp> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index c49737af78b..2e13c0e24c0 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -14,7 +14,6 @@ #include <vespa/config-stor-distribution.h> #include <vespa/config-upgrading.h> -#include <vespa/config/helper/configfetcher.h> #include <vespa/config/helper/ifetchercallback.h> #include <vespa/config/subscription/configuri.h> #include <vespa/document/config/config-documenttypes.h> @@ -27,6 +26,7 @@ #include <mutex> namespace document { class DocumentTypeRepo; } +namespace config { class ConfigFetcher; } namespace storage { @@ -35,7 +35,6 @@ class CommunicationManager; class FileStorManager; class HostInfo; class IStorageChainBuilder; -class MemoryStatusViewer; class NodeIdentity; class StateManager; class StateReporter; diff --git a/storage/src/vespa/storage/tools/getidealstate.cpp b/storage/src/vespa/storage/tools/getidealstate.cpp index 5b17fa63b5a..8b120924aaa 100644 --- a/storage/src/vespa/storage/tools/getidealstate.cpp +++ b/storage/src/vespa/storage/tools/getidealstate.cpp @@ -4,9 +4,9 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/programoptions.h> -#include <vespa/config/config.h> #include <vespa/config/print/ostreamconfigwriter.h> #include <vespa/config-stor-distribution.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/configgetter.hpp> #include <iostream> #include <sstream> diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index b66285f5048..e8a217fc718 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -24,35 +24,23 @@ using document::BucketSpace; namespace storage { Visitor::HitCounter::HitCounter() - : _firstPassHits(0), - _firstPassBytes(0), - _secondPassHits(0), - _secondPassBytes(0) + : _doc_hits(0), + _doc_bytes(0) { } void Visitor::HitCounter::addHit(const document::DocumentId& , uint32_t size) { - bool firstPass = false; - - if (firstPass) { - _firstPassHits++; - _firstPassBytes += size; - } else { - _secondPassHits++; - _secondPassBytes += size; - } - + _doc_hits++; + _doc_bytes += size; } void -Visitor::HitCounter::updateVisitorStatistics(vdslib::VisitorStatistics& statistics) +Visitor::HitCounter::updateVisitorStatistics(vdslib::VisitorStatistics& statistics) const { - statistics.setDocumentsReturned(statistics.getDocumentsReturned() + _firstPassHits); - statistics.setBytesReturned(statistics.getBytesReturned() + _firstPassBytes); - statistics.setSecondPassDocumentsReturned(statistics.getSecondPassDocumentsReturned() + _secondPassHits); - statistics.setSecondPassBytesReturned(statistics.getSecondPassBytesReturned() + _secondPassBytes); + statistics.setDocumentsReturned(statistics.getDocumentsReturned() + _doc_hits); + statistics.setBytesReturned(statistics.getBytesReturned() + _doc_bytes); } Visitor::VisitorTarget::MessageMeta::MessageMeta( diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 8857a54e8df..11192021577 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -89,24 +89,11 @@ public: class HitCounter { public: HitCounter(); - void addHit(const document::DocumentId& hit, uint32_t size); - - void updateVisitorStatistics(vdslib::VisitorStatistics& statistics); - - uint32_t getFirstPassHits() const { return _firstPassHits; } - - uint64_t getFirstPassBytes() const { return _firstPassBytes; } - - uint32_t getSecondPassHits() const { return _secondPassHits; } - - uint64_t getSecondPassBytes() const { return _secondPassBytes; } - + void updateVisitorStatistics(vdslib::VisitorStatistics& statistics) const; private: - uint32_t _firstPassHits; - uint64_t _firstPassBytes; - uint32_t _secondPassHits; - uint64_t _secondPassBytes; + uint32_t _doc_hits; + uint64_t _doc_bytes; }; enum VisitorState diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 8450fab5dcb..eb5c927873c 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -8,8 +8,10 @@ #include "recoveryvisitor.h" #include "reindexing_visitor.h" #include <vespa/storage/common/statusmessages.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/config/helper/configfetcher.hpp> #include <cassert> #include <vespa/log/log.h> @@ -30,8 +32,8 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _visitorLock(), _visitorCond(), _visitorCounter(0), - _configFetcher(configUri.getContext()), - _metrics(new VisitorMetrics), + _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), + _metrics(std::make_shared<VisitorMetrics>()), _maxFixedConcurrentVisitors(1), _maxVariableConcurrentVisitors(0), _maxVisitorQueueSize(1024), @@ -46,10 +48,10 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _enforceQueueUse(false), _visitorFactories(externalFactories) { - _configFetcher.subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this); - _configFetcher.start(); + _configFetcher->subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this); + _configFetcher->start(); _component.registerMetric(*_metrics); - _thread = _component.startThread(*this, 30s, 1s); + _thread = _component.startThread(*this, 30s, 1s, 1, vespalib::CpuUsage::Category::READ); _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); _visitorFactories["dumpvisitor"] = std::make_shared<DumpVisitorSingleFactory>(); _visitorFactories["dumpvisitorsingle"] = std::make_shared<DumpVisitorSingleFactory>(); @@ -81,7 +83,7 @@ void VisitorManager::onClose() { // Avoid getting config during shutdown - _configFetcher.close(); + _configFetcher->close(); { std::lock_guard sync(_visitorLock); for (CommandQueue<api::CreateVisitorCommand>::iterator it diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 32237de7f36..888c0873e9c 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -29,12 +29,15 @@ #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> -#include <vespa/config/config.h> +#include <vespa/config/helper/ifetchercallback.h> #include <vespa/vespalib/util/document_runnable.h> -namespace storage { +namespace config { + class ConfigUri; + class ConfigFetcher; +} -namespace api { class BucketTimeInterval; } +namespace storage { class RequestStatusPageReply; @@ -63,7 +66,7 @@ private: mutable std::mutex _visitorLock; std::condition_variable _visitorCond; uint64_t _visitorCounter; - config::ConfigFetcher _configFetcher; + std::unique_ptr<config::ConfigFetcher> _configFetcher; std::shared_ptr<VisitorMetrics> _metrics; uint32_t _maxFixedConcurrentVisitors; uint32_t _maxVariableConcurrentVisitors; diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 16b4abf5b34..ef01a684901 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -96,7 +96,7 @@ VisitorThread::VisitorThread(uint32_t threadIndex, _messageSessionFactory(messageSessionFac), _visitorFactories(visitorFactories) { - _thread = _component.startThread(*this, 30s, 1s); + _thread = _component.startThread(*this, 30s, 1s, 1, vespalib::CpuUsage::Category::READ); _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index cc7363af7bc..fa400b565b2 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -538,6 +538,7 @@ ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& ms buf.putLong(msg.getVisitorStatistics().getBytesVisited()); buf.putLong(msg.getVisitorStatistics().getDocumentsReturned()); buf.putLong(msg.getVisitorStatistics().getBytesReturned()); + // TODO remove second pass concept on Vespa 8 buf.putLong(msg.getVisitorStatistics().getSecondPassDocumentsReturned()); buf.putLong(msg.getVisitorStatistics().getSecondPassBytesReturned()); } @@ -554,6 +555,7 @@ ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) vs.setBytesVisited(SH::getLong(buf)); vs.setDocumentsReturned(SH::getLong(buf)); vs.setBytesReturned(SH::getLong(buf)); + // TODO remove second pass concept on Vespa 8 vs.setSecondPassDocumentsReturned(SH::getLong(buf)); vs.setSecondPassBytesReturned(SH::getLong(buf)); msg->setVisitorStatistics(vs); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 8425294cbbd..91b5999e34c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -1240,8 +1240,8 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorReply& proto_stats->set_bytes_visited(stats.getBytesVisited()); proto_stats->set_documents_returned(stats.getDocumentsReturned()); proto_stats->set_bytes_returned(stats.getBytesReturned()); - proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); - proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); + proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); // TODO remove on Vespa 8 + proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); // TODO remove on Vespa 8 }); } @@ -1287,8 +1287,8 @@ api::StorageReply::UP ProtocolSerialization7::onDecodeCreateVisitorReply(const S vs.setBytesVisited(proto_stats.bytes_visited()); vs.setDocumentsReturned(proto_stats.documents_returned()); vs.setBytesReturned(proto_stats.bytes_returned()); - vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); - vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); + vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); // TODO remove on Vespa 8 + vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); // TODO remove on Vespa 8 reply->setVisitorStatistics(vs); return reply; }); diff --git a/storageframework/src/tests/thread/taskthreadtest.cpp b/storageframework/src/tests/thread/taskthreadtest.cpp index b7d7de3239e..32bc6e8a8fb 100644 --- a/storageframework/src/tests/thread/taskthreadtest.cpp +++ b/storageframework/src/tests/thread/taskthreadtest.cpp @@ -31,7 +31,7 @@ struct MyThread : public TaskThread<Task> { TEST(TaskThreadTest, test_normal_usage) { - TickingThreadPool::UP pool(TickingThreadPool::createDefault("testApp")); + TickingThreadPool::UP pool(TickingThreadPool::createDefault("testApp", 100ms)); MyThread t(*pool); t.addTask(Task("a", 6)); diff --git a/storageframework/src/tests/thread/tickingthreadtest.cpp b/storageframework/src/tests/thread/tickingthreadtest.cpp index 4d6ce76a676..5fe8f25ae72 100644 --- a/storageframework/src/tests/thread/tickingthreadtest.cpp +++ b/storageframework/src/tests/thread/tickingthreadtest.cpp @@ -4,7 +4,6 @@ #include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h> #include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/stringfmt.h> #include <thread> @@ -86,7 +85,7 @@ MyApp::MyApp(int threadCount, bool doCritOverlapTest) : _critOverlapCounter(0), _doCritOverlapTest(doCritOverlapTest), _critOverlap(false), - _threadPool(TickingThreadPool::createDefault("testApp")) + _threadPool(TickingThreadPool::createDefault("testApp", 100ms)) { for (int i=0; i<threadCount; ++i) { _threadPool->addThread(*this); @@ -100,8 +99,7 @@ MyApp::~MyApp() = default; TEST(TickingThreadTest, test_ticks_before_wait_basic) { - TestComponentRegister testReg( - ComponentRegisterImpl::UP(new ComponentRegisterImpl)); + TestComponentRegister testReg(std::make_unique<ComponentRegisterImpl>()); int threadCount = 1; MyApp app(threadCount); app.start(testReg.getThreadPoolImpl()); @@ -117,29 +115,6 @@ TEST(TickingThreadTest, test_ticks_before_wait_basic) app._threadPool->stop(); } -TEST(TickingThreadTest, test_ticks_before_wait_live_update) -{ - TestComponentRegister testReg = std::make_unique<ComponentRegisterImpl>(); - int threadCount = 1; - MyApp app(threadCount); - // Configure thread pool to send bulks of 5000 ticks each second. - long unsigned int ticksBeforeWaitMs = 5000; - app.start(testReg.getThreadPoolImpl()); - app._threadPool->updateParametersAllThreads( - 1s, 234234ms, ticksBeforeWaitMs); - - // Check that 5000 ticks are received instantly (usually <2 ms) - // (if live update is broken it will take more than an hour). - int maxAttempts = 120000; // a bit more than 120 secs - while (app.getTotalNonCritTicks() < ticksBeforeWaitMs && maxAttempts-->0) { - std::this_thread::sleep_for(1ms); - } - - EXPECT_GT(maxAttempts, 0); - EXPECT_GE(app.getTotalNonCritTicks(), ticksBeforeWaitMs); - app._threadPool->stop(); -} - TEST(TickingThreadTest, test_destroy_without_starting) { TestComponentRegister testReg(std::make_unique<ComponentRegisterImpl>()); diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index f5deb25f983..7281c55ba25 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -14,7 +14,8 @@ ThreadImpl::ThreadImpl(ThreadPoolImpl& pool, vespalib::stringref id, vespalib::duration waitTime, vespalib::duration maxProcessTime, - int ticksBeforeWait) + int ticksBeforeWait, + std::optional<vespalib::CpuUsage::Category> cpu_category) : Thread(id), _pool(pool), _runnable(runnable), @@ -23,7 +24,8 @@ ThreadImpl::ThreadImpl(ThreadPoolImpl& pool, _tickDataPtr(0), _interrupted(false), _joined(false), - _thread(*this) + _thread(*this), + _cpu_category(cpu_category) { _tickData[_tickDataPtr]._lastTick = pool.getClock().getMonotonicTime(); _thread.start(_pool.getThreadPool()); @@ -38,7 +40,12 @@ ThreadImpl::~ThreadImpl() void ThreadImpl::run() { - _runnable.run(*this); + if (_cpu_category.has_value()) { + auto usage = vespalib::CpuUsage::use(_cpu_category.value()); + _runnable.run(*this); + } else { + _runnable.run(*this); + } _pool.unregisterThread(*this); _joined = true; } @@ -109,15 +116,6 @@ ThreadImpl::setTickData(const ThreadTickData& tickData) _tickDataPtr = nextData; } -void -ThreadImpl::updateParameters(vespalib::duration waitTime, - vespalib::duration maxProcessTime, - int ticksBeforeWait) { - _properties.setWaitTime(waitTime); - _properties.setMaxProcessTime(maxProcessTime); - _properties.setTicksBeforeWait(ticksBeforeWait); -} - ThreadTickData ThreadImpl::AtomicThreadTickData::loadRelaxed() const noexcept { diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index 3a24b79bda8..e6f0a21ea20 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h @@ -3,9 +3,11 @@ #pragma once #include <vespa/storageframework/generic/thread/threadpool.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/document_runnable.h> #include <array> #include <atomic> +#include <optional> namespace storage::framework::defaultimplementation { @@ -51,12 +53,14 @@ class ThreadImpl : public Thread std::atomic<bool> _interrupted; bool _joined; BackendThread _thread; + std::optional<vespalib::CpuUsage::Category> _cpu_category; void run(); public: ThreadImpl(ThreadPoolImpl&, Runnable&, vespalib::stringref id, vespalib::duration waitTime, - vespalib::duration maxProcessTime, int ticksBeforeWait); + vespalib::duration maxProcessTime, int ticksBeforeWait, + std::optional<vespalib::CpuUsage::Category> cpu_category); ~ThreadImpl(); bool interrupted() const override; @@ -71,8 +75,6 @@ public: return _properties.getTicksBeforeWait(); } - void updateParameters(vespalib::duration waitTime, vespalib::duration maxProcessTime, int ticksBeforeWait) override; - void setTickData(const ThreadTickData&); ThreadTickData getTickData() const; const ThreadProperties& getProperties() const { return _properties; } diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index 7846c917631..7711eddf51c 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp @@ -48,13 +48,14 @@ ThreadPoolImpl::~ThreadPoolImpl() Thread::UP ThreadPoolImpl::startThread(Runnable& runnable, vespalib::stringref id, vespalib::duration waitTime, - vespalib::duration maxProcessTime, int ticksBeforeWait) + vespalib::duration maxProcessTime, int ticksBeforeWait, + std::optional<vespalib::CpuUsage::Category> cpu_category) { std::lock_guard lock(_threadVectorLock); if (_stopping) { throw IllegalStateException("Threadpool is stopping", VESPA_STRLOC); } - auto thread = std::make_unique<ThreadImpl>(*this, runnable, id, waitTime, maxProcessTime, ticksBeforeWait); + auto thread = std::make_unique<ThreadImpl>(*this, runnable, id, waitTime, maxProcessTime, ticksBeforeWait, cpu_category); _threads.push_back(thread.get()); return thread; } diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index dc80803b57f..d6053a2f128 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -22,7 +22,8 @@ public: ~ThreadPoolImpl() override; Thread::UP startThread(Runnable&, vespalib::stringref id, vespalib::duration waitTime, - vespalib::duration maxProcessTime, int ticksBeforeWait) override; + vespalib::duration maxProcessTime, int ticksBeforeWait, + std::optional<vespalib::CpuUsage::Category> cpu_category) override; void visitThreads(ThreadVisitor&) const override; void registerThread(ThreadImpl&); diff --git a/storageframework/src/vespa/storageframework/generic/component/component.cpp b/storageframework/src/vespa/storageframework/generic/component/component.cpp index 82ca40f780d..f45e1a0b839 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.cpp +++ b/storageframework/src/vespa/storageframework/generic/component/component.cpp @@ -89,10 +89,11 @@ Component::getThreadPool() const // Helper functions for components wanting to start a single thread. Thread::UP -Component::startThread(Runnable& runnable, vespalib::duration waitTime, vespalib::duration maxProcessTime, int ticksBeforeWait) +Component::startThread(Runnable& runnable, vespalib::duration waitTime, vespalib::duration maxProcessTime, + int ticksBeforeWait, std::optional<vespalib::CpuUsage::Category> cpu_category) { return getThreadPool().startThread(runnable, getName(), waitTime, - maxProcessTime, ticksBeforeWait); + maxProcessTime, ticksBeforeWait, cpu_category); } void diff --git a/storageframework/src/vespa/storageframework/generic/component/component.h b/storageframework/src/vespa/storageframework/generic/component/component.h index b7d07243f73..09af1d276f3 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.h +++ b/storageframework/src/vespa/storageframework/generic/component/component.h @@ -71,7 +71,9 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageframework/generic/thread/thread.h> #include <vespa/storageframework/generic/clock/clock.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <atomic> +#include <optional> namespace storage::framework { @@ -115,7 +117,7 @@ class Component : private ManagedComponent void close() override; public: - typedef std::unique_ptr<Component> UP; + using UP = std::unique_ptr<Component>; Component(ComponentRegister&, vespalib::stringref name); virtual ~Component(); @@ -180,7 +182,8 @@ public: Thread::UP startThread(Runnable&, vespalib::duration maxProcessTime = vespalib::duration::zero(), vespalib::duration waitTime = vespalib::duration::zero(), - int ticksBeforeWait = 1); + int ticksBeforeWait = 1, + std::optional<vespalib::CpuUsage::Category> cpu_category = std::nullopt); // Check upgrade flag settings. Note that this flag may change at any time. // Thus the results of these functions should not be cached. diff --git a/storageframework/src/vespa/storageframework/generic/thread/thread.h b/storageframework/src/vespa/storageframework/generic/thread/thread.h index f11c1fbcf91..c17638a0d42 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/thread.h +++ b/storageframework/src/vespa/storageframework/generic/thread/thread.h @@ -13,13 +13,8 @@ #pragma once #include "runnable.h" -#include <vespa/vespalib/stllike/string.h> #include <condition_variable> -namespace vespalib { - class Monitor; -} - namespace storage::framework { class Thread : public ThreadHandle { @@ -29,7 +24,7 @@ public: typedef std::unique_ptr<Thread> UP; Thread(vespalib::stringref id) : _id(id) {} - virtual ~Thread() {} + virtual ~Thread() = default; virtual const vespalib::string& getId() const { return _id; } @@ -50,10 +45,6 @@ public: */ virtual void join() = 0; - virtual void updateParameters(vespalib::duration waitTime, - vespalib::duration maxProcessTime, - int ticksBeforeWait) = 0; - /** * Utility function to interrupt and join a thread, possibly broadcasting * through a monitor after the signalling face. diff --git a/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp b/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp index 29be5fb07a2..480e42c91ef 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp +++ b/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp @@ -7,34 +7,10 @@ namespace storage::framework { ThreadProperties::ThreadProperties(vespalib::duration waitTime, vespalib::duration maxProcessTime, int ticksBeforeWait) + : _maxProcessTime(maxProcessTime), + _waitTime(waitTime), + _ticksBeforeWait(ticksBeforeWait) { - setWaitTime(waitTime); - setMaxProcessTime(maxProcessTime); - setTicksBeforeWait(ticksBeforeWait); -} - - vespalib::duration ThreadProperties::getMaxProcessTime() const { - return _maxProcessTime.load(std::memory_order_relaxed); -} - -vespalib::duration ThreadProperties::getWaitTime() const { - return _waitTime.load(std::memory_order_relaxed); -} - -int ThreadProperties::getTicksBeforeWait() const { - return _ticksBeforeWait.load(std::memory_order_relaxed); -} - -void ThreadProperties::setMaxProcessTime(vespalib::duration maxProcessingTime) { - _maxProcessTime.store(maxProcessingTime); -} - -void ThreadProperties::setWaitTime(vespalib::duration waitTime) { - _waitTime.store(waitTime); -} - -void ThreadProperties::setTicksBeforeWait(int ticksBeforeWait) { - _ticksBeforeWait.store(ticksBeforeWait); } } diff --git a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h b/storageframework/src/vespa/storageframework/generic/thread/threadpool.h index 7a8ff47f669..7607932e079 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h +++ b/storageframework/src/vespa/storageframework/generic/thread/threadpool.h @@ -16,6 +16,8 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageframework/generic/thread/thread.h> #include <vespa/storageframework/generic/clock/time.h> +#include <vespa/vespalib/util/cpu_usage.h> +#include <optional> #include <vector> namespace storage::framework { @@ -32,7 +34,7 @@ private: * Time this thread should maximum use to process before a tick is * registered. (Including wait time if wait time is not set) */ - std::atomic<vespalib::duration> _maxProcessTime; + vespalib::duration _maxProcessTime; /** * Time this thread will wait in a non-interrupted wait cycle. * Used in cases where a wait cycle is registered. As long as no other @@ -40,28 +42,23 @@ private: * wait time here. The deadlock detector should add a configurable * global time period before flagging deadlock anyways. */ - std::atomic<vespalib::duration> _waitTime; + vespalib::duration _waitTime; /** * Number of ticks to be done before a wait. */ - std::atomic_uint _ticksBeforeWait; + uint32_t _ticksBeforeWait; public: ThreadProperties(vespalib::duration waitTime, vespalib::duration maxProcessTime, int ticksBeforeWait); - void setMaxProcessTime(vespalib::duration); - void setWaitTime(vespalib::duration); - void setTicksBeforeWait(int); - - vespalib::duration getMaxProcessTime() const; - vespalib::duration getWaitTime() const; - int getTicksBeforeWait() const; + vespalib::duration getMaxProcessTime() const { return _maxProcessTime; } + vespalib::duration getWaitTime() const { return _waitTime; } + int getTicksBeforeWait() const { return _ticksBeforeWait; } vespalib::duration getMaxCycleTime() const { - return std::max(_maxProcessTime.load(std::memory_order_relaxed), - _waitTime.load(std::memory_order_relaxed)); + return std::max(_maxProcessTime, _waitTime); } }; @@ -88,7 +85,8 @@ struct ThreadPool { vespalib::stringref id, vespalib::duration waitTime, vespalib::duration maxProcessTime, - int ticksBeforeWait) = 0; + int ticksBeforeWait, + std::optional<vespalib::CpuUsage::Category> cpu_category) = 0; virtual void visitThreads(ThreadVisitor&) const = 0; }; diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp index 867247c1d68..1a9cb459f28 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp +++ b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp @@ -107,12 +107,12 @@ private: }; class TickingThreadPoolImpl final : public TickingThreadPool { - vespalib::string _name; + const vespalib::string _name; + const vespalib::duration _waitTime; + const vespalib::duration _maxProcessTime; + const uint32_t _ticksBeforeWait; std::mutex _lock; std::condition_variable _cond; - std::atomic<vespalib::duration> _waitTime; - std::atomic_uint _ticksBeforeWait; - std::atomic<vespalib::duration> _maxProcessTime; std::vector<TickingThreadRunner::SP> _tickers; std::vector<std::shared_ptr<Thread>> _threads; @@ -137,24 +137,14 @@ public: int ticksBeforeWait, vespalib::duration maxProcessTime) : _name(name), _waitTime(waitTime), - _ticksBeforeWait(ticksBeforeWait), - _maxProcessTime(maxProcessTime) {} + _maxProcessTime(maxProcessTime), + _ticksBeforeWait(ticksBeforeWait) + { } ~TickingThreadPoolImpl() override { stop(); } - void updateParametersAllThreads(vespalib::duration waitTime, vespalib::duration maxProcessTime, - int ticksBeforeWait) override { - _waitTime.store(waitTime); - _maxProcessTime.store(maxProcessTime); - _ticksBeforeWait.store(ticksBeforeWait); - // TODO: Add locking so threads not deleted while updating - for (uint32_t i=0; i<_threads.size(); ++i) { - _threads[i]->updateParameters(waitTime, maxProcessTime, ticksBeforeWait); - } - } - void addThread(TickingThread& ticker) override { ThreadIndex index = _tickers.size(); ticker.newThreadCreated(index); @@ -169,9 +159,9 @@ public: _threads.push_back(std::shared_ptr<Thread>(pool.startThread( *_tickers[i], ost.str(), - _waitTime.load(std::memory_order_relaxed), - _maxProcessTime.load(std::memory_order_relaxed), - _ticksBeforeWait.load(std::memory_order_relaxed)))); + _waitTime, + _maxProcessTime, + _ticksBeforeWait, std::nullopt))); } } @@ -227,4 +217,10 @@ TickingThreadPool::createDefault( return std::make_unique<TickingThreadPoolImpl>(name, waitTime, ticksBeforeWait, maxProcessTime); } +TickingThreadPool::UP +TickingThreadPool::createDefault(vespalib::stringref name, vespalib::duration waitTime) +{ + return createDefault(name, waitTime, 1, 5s); +} + } // storage::framework diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h index 1265ebd7203..9ddc47c8f3a 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h +++ b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h @@ -81,14 +81,10 @@ struct TickingThreadPool : public ThreadLock { // TODO STRIPE: Change waitTime default to 100ms when legacy mode is removed. static TickingThreadPool::UP createDefault( vespalib::stringref name, - vespalib::duration waitTime = 5ms, - int ticksBeforeWait = 1, - vespalib::duration maxProcessTime = 5s); - - virtual void updateParametersAllThreads( vespalib::duration waitTime, - vespalib::duration maxProcessTime, - int ticksBeforeWait) = 0; + int ticksBeforeWait, + vespalib::duration maxProcessTime); + static TickingThreadPool::UP createDefault(vespalib::stringref name, vespalib::duration waitTime); ~TickingThreadPool() override = default; diff --git a/storageserver/src/vespa/storageserver/app/process.cpp b/storageserver/src/vespa/storageserver/app/process.cpp index 8480f1427f2..93d19c8e6fc 100644 --- a/storageserver/src/vespa/storageserver/app/process.cpp +++ b/storageserver/src/vespa/storageserver/app/process.cpp @@ -5,6 +5,7 @@ #include <vespa/storage/storageserver/storagenode.h> #include <vespa/storage/storageserver/storagenodecontext.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/log/log.h> LOG_SETUP(".process"); @@ -18,6 +19,8 @@ Process::Process(const config::ConfigUri & configUri) _configSubscriber(_configUri.getContext()) { } +Process::~Process() = default; + void Process::setupConfig(milliseconds subscribeTimeout) { diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h index 407c8cf8881..fcda67fb201 100644 --- a/storageserver/src/vespa/storageserver/app/process.h +++ b/storageserver/src/vespa/storageserver/app/process.h @@ -42,7 +42,7 @@ public: using milliseconds = std::chrono::milliseconds; Process(const config::ConfigUri & configUri); - virtual ~Process() {} + ~Process() override; virtual void setupConfig(milliseconds subscribeTimeout); virtual void createNode() = 0; diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp index 076b847b32e..0765074e315 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp @@ -54,6 +54,19 @@ QueryEnvironment::QueryEnvironment(const string & location_str, QueryEnvironment::~QueryEnvironment() {} +void QueryEnvironment::addGeoLocation(const vespalib::string &field, const vespalib::string &location_str) { + GeoLocationParser locationParser; + if (! locationParser.parseNoField(location_str)) { + LOG(warning, "Location parse error (location: '%s'): %s. Location ignored.", + location_str.c_str(), locationParser.getParseError()); + return; + } + auto loc = locationParser.getGeoLocation(); + if (loc.has_point) { + _locations.push_back(GeoLocationSpec{field, loc}); + } +} + QueryEnvironment::GeoLocationSpecPtrs QueryEnvironment::getAllLocations() const { diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h index a61cdc5ae58..1bc625f4ea3 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h @@ -33,6 +33,8 @@ public: const search::IAttributeManager * attrMgr = nullptr); ~QueryEnvironment(); + void addGeoLocation(const vespalib::string &field, const vespalib::string &location); + // inherit documentation virtual const search::fef::Properties & getProperties() const override { return _properties; } diff --git a/streamingvisitors/src/vespa/searchvisitor/querywrapper.h b/streamingvisitors/src/vespa/searchvisitor/querywrapper.h index 5a277a12318..90343085298 100644 --- a/streamingvisitors/src/vespa/searchvisitor/querywrapper.h +++ b/streamingvisitors/src/vespa/searchvisitor/querywrapper.h @@ -47,6 +47,7 @@ public: bool isPhraseTerm() const { return _parent != nullptr; } bool isFirstPhraseTerm() const { return isPhraseTerm() && getIndex() == 0; } size_t getPosAdjust() const { return _parent != nullptr ? _parent->width() - 1 : 0; } + bool isGeoPosTerm() const { return (_term != nullptr) && _term->isGeoLoc(); } }; typedef std::vector<Term> TermList; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index ed5d0a88bb1..13e5ad1c84b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vsm/common/document.h> +#include <vespa/vsm/vsm/vsm-adapter.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.rankmanager"); diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index ac74f7791db..73baa1de45f 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -56,6 +56,11 @@ RankProcessor::initQueryEnvironment() QueryWrapper::TermList & terms = _query.getTermList(); for (uint32_t i = 0; i < terms.size(); ++i) { + if (terms[i].isGeoPosTerm()) { + const vespalib::string & fieldName = terms[i].getTerm()->index(); + const vespalib::string & locStr = terms[i].getTerm()->getTermString(); + _queryEnv.addGeoLocation(fieldName, locStr); + } if (!terms[i].isPhraseTerm() || terms[i].isFirstPhraseTerm()) { // register 1 term data per phrase QueryTermData & qtd = dynamic_cast<QueryTermData &>(terms[i].getTerm()->getQueryItem()); diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index f4923bf79f1..cf1f65aa05a 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -366,6 +366,25 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-no-log4j</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for apps with log4j deps in compile or provided scope. --> + <excludes> + <exclude>log4j:log4j:*:jar:compile</exclude> + <exclude>log4j:log4j:*:jar:provided</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:compile</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:provided</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> </executions> </plugin> diff --git a/tenant-cd-api/abi-spec.json b/tenant-cd-api/abi-spec.json index 2e277a42378..4840943b7b1 100644 --- a/tenant-cd-api/abi-spec.json +++ b/tenant-cd-api/abi-spec.json @@ -29,6 +29,17 @@ ], "fields": [] }, + "ai.vespa.hosted.cd.InconclusiveTestException": { + "superClass": "java.lang.RuntimeException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()" + ], + "fields": [] + }, "ai.vespa.hosted.cd.IntegrationTest": { "superClass": "java.lang.Object", "interfaces": [ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/InconclusiveTestException.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/InconclusiveTestException.java new file mode 100644 index 00000000000..a77379647dc --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/InconclusiveTestException.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + + +/** + * Signals that a test method can not yield a conclusive result at this time, and must be retried later. + * + * @author jonmv + */ +public class InconclusiveTestException extends RuntimeException { + +} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java index f3bf6f4ed38..a79f3d5fb6d 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java @@ -23,22 +23,47 @@ public class VisitorStatistics { public int getBucketsVisited() { return bucketsVisited; } public void setBucketsVisited(int bucketsVisited) { this.bucketsVisited = bucketsVisited; } + /** + * @return the number of documents matching the document selection in the backend and that + * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc). + */ public long getDocumentsVisited() { return documentsVisited; } public void setDocumentsVisited(long documentsVisited) { this.documentsVisited = documentsVisited; } public long getBytesVisited() { return bytesVisited; } public void setBytesVisited(long bytesVisited) { this.bytesVisited = bytesVisited; } + /** + * @return Number of documents returned to the visitor client by the backend. This number may + * be lower than that returned by getDocumentsVisited() since the client-specified visitor + * instance may further have filtered the set of documents returned by the backend. + */ public long getDocumentsReturned() { return documentsReturned; } public void setDocumentsReturned(long documentsReturned) { this.documentsReturned = documentsReturned; } public long getBytesReturned() { return bytesReturned; } public void setBytesReturned(long bytesReturned) { this.bytesReturned = bytesReturned; } + /** + * @deprecated Use getDocumentsReturned() instead + */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public long getSecondPassDocumentsReturned() { return secondPassDocumentsReturned; } + /** + * @deprecated only applies for deprecated "orderdoc" ID scheme + */ + @Deprecated(since = "7", forRemoval = true)// TODO: Vespa 8: remove public void setSecondPassDocumentsReturned(long secondPassDocumentsReturned) { this.secondPassDocumentsReturned = secondPassDocumentsReturned; } + /** + * @deprecated Use getBytesReturned() instead + */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public long getSecondPassBytesReturned() { return secondPassBytesReturned; } + /** + * @deprecated only applies for deprecated "orderdoc" ID scheme + */ + @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove public void setSecondPassBytesReturned(long secondPassBytesReturned) { this.secondPassBytesReturned = secondPassBytesReturned; } public String toString() { diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index 04a577db554..b6455a7703c 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -39,6 +39,7 @@ public class Distribution { private final boolean distributorAutoOwnershipTransferOnWholeGroupDown; } + @SuppressWarnings("removal") // TODO Vespa 8: remove private ConfigSubscriber configSub; private final AtomicReference<Config> config = new AtomicReference<>(new Config(null, 1, false)); @@ -60,6 +61,7 @@ public class Distribution { return p; } + @SuppressWarnings("removal") // TODO Vespa 8: remove // NOTE: keep in sync with the below private ConfigSubscriber.SingleSubscriber<StorDistributionConfig> configSubscriber = config -> { try { @@ -146,6 +148,7 @@ public class Distribution { } } + @SuppressWarnings("removal") // TODO Vespa 8: remove public Distribution(String configId) { try { configSub = new ConfigSubscriber(); diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java index 225bc17c552..084d0e9185d 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java @@ -4,8 +4,6 @@ package com.yahoo.vdslib.distribution; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.document.BucketId; import com.yahoo.vdslib.state.ClusterState; -import com.yahoo.vdslib.state.Node; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vespa.config.content.StorDistributionConfig; import org.codehaus.jettison.json.JSONArray; @@ -16,8 +14,9 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; + +// TODO: Use config builder instead of ConfigGetter to create test config. public class DistributionTestFactory extends CrossPlatformTestFactory { private static final String testDirectory = "src/tests/distribution/testdata"; diff --git a/vdslib/src/vespa/vdslib/container/visitorstatistics.h b/vdslib/src/vespa/vdslib/container/visitorstatistics.h index f4e03b00bee..d4912bbf1ba 100644 --- a/vdslib/src/vespa/vdslib/container/visitorstatistics.h +++ b/vdslib/src/vespa/vdslib/container/visitorstatistics.h @@ -40,8 +40,8 @@ private: uint64_t _bytesVisited; uint64_t _documentsReturned; uint64_t _bytesReturned; - uint64_t _secondPassDocumentsReturned; - uint64_t _secondPassBytesReturned; + uint64_t _secondPassDocumentsReturned; // TODO remove on Vespa 8 + uint64_t _secondPassBytesReturned; // TODO remove on Vespa 8 }; } diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index 7322c31d858..8a8d273e9db 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -6,13 +6,10 @@ #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/util/bobhash.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/config/config.h> #include <vespa/config/print/asciiconfigwriter.h> -#include <vespa/config/print/asciiconfigreader.h> +#include <vespa/config/print/asciiconfigreader.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/config-stor-distribution.h> -#include <list> -#include <algorithm> #include <cmath> #include <cassert> diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp index 0da172fc789..ce00897acdf 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp @@ -62,8 +62,8 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, ClusterStateBundle::ClusterStateBundle(const ClusterStateBundle&) = default; ClusterStateBundle& ClusterStateBundle::operator=(const ClusterStateBundle&) = default; -ClusterStateBundle::ClusterStateBundle(ClusterStateBundle&&) = default; -ClusterStateBundle& ClusterStateBundle::operator=(ClusterStateBundle&&) = default; +ClusterStateBundle::ClusterStateBundle(ClusterStateBundle&&) noexcept = default; +ClusterStateBundle& ClusterStateBundle::operator=(ClusterStateBundle&&) noexcept = default; ClusterStateBundle::~ClusterStateBundle() = default; @@ -90,7 +90,7 @@ ClusterStateBundle::getVersion() const } bool -ClusterStateBundle::operator==(const ClusterStateBundle &rhs) const +ClusterStateBundle::operator==(const ClusterStateBundle &rhs) const noexcept { if (!(*_baselineClusterState == *rhs._baselineClusterState)) { return false; diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h index ad9e07407a0..52a952e30bb 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h @@ -65,8 +65,8 @@ public: ClusterStateBundle(const ClusterStateBundle&); ClusterStateBundle& operator=(const ClusterStateBundle&); - ClusterStateBundle(ClusterStateBundle&&); - ClusterStateBundle& operator=(ClusterStateBundle&&); + ClusterStateBundle(ClusterStateBundle&&) noexcept; + ClusterStateBundle& operator=(ClusterStateBundle&&) noexcept; ~ClusterStateBundle(); const std::shared_ptr<const ClusterState> &getBaselineClusterState() const; @@ -74,15 +74,15 @@ public: const BucketSpaceStateMapping& getDerivedClusterStates() const noexcept { return _derivedBucketSpaceStates; } - bool block_feed_in_cluster() const { + [[nodiscard]] bool block_feed_in_cluster() const noexcept { return _feed_block.has_value() && _feed_block->block_feed_in_cluster(); } const std::optional<FeedBlock>& feed_block() const { return _feed_block; } uint32_t getVersion() const; bool deferredActivation() const noexcept { return _deferredActivation; } std::string toString() const; - bool operator==(const ClusterStateBundle &rhs) const; - bool operator!=(const ClusterStateBundle &rhs) const { return !operator==(rhs); } + bool operator==(const ClusterStateBundle &rhs) const noexcept; + bool operator!=(const ClusterStateBundle &rhs) const noexcept { return !operator==(rhs); } }; std::ostream& operator<<(std::ostream&, const ClusterStateBundle&); diff --git a/vespa-application-maven-plugin/pom.xml b/vespa-application-maven-plugin/pom.xml index 362e9745256..10c494a7dd4 100644 --- a/vespa-application-maven-plugin/pom.xml +++ b/vespa-application-maven-plugin/pom.xml @@ -45,7 +45,7 @@ <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - <version>2.6</version> + <version>2.7</version> </dependency> </dependencies> <build> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 289a63a1128..d83eab9e339 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.athenz.api.AthenzPolicy; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; import com.yahoo.vespa.athenz.client.ErrorHandler; @@ -288,7 +287,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public Map<AthenzUser, String> listPendingRoleApprovals(AthenzRole athenzRole) { + public Map<AthenzIdentity, String> listPendingRoleApprovals(AthenzRole athenzRole) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true", athenzRole.domain().getName(), athenzRole.roleName())); HttpUriRequest request = RequestBuilder.get() .setUri(uri) @@ -297,16 +296,15 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { return roleEntity.roleMembers().stream() .filter(RoleEntity.Member::pendingApproval) - .filter(re -> AthenzIdentities.USER_PRINCIPAL_DOMAIN.equals(AthenzIdentities.from(re.memberName()).getDomain())) .collect(Collectors.toUnmodifiableMap( - m -> (AthenzUser) AthenzIdentities.from(m.memberName()), + m -> AthenzIdentities.from(m.memberName()), m -> m.auditRef() != null ? m.auditRef() : "<no reason provided>")); } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason) { - URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s/decision", athenzRole.domain().getName(), athenzRole.roleName(), athenzUser.getFullName())); - MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(athenzUser.getFullName(), true, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, Optional<String> reason) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s/decision", athenzRole.domain().getName(), athenzRole.roleName(), athenzIdentity.getFullName())); + MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(athenzIdentity.getFullName(), true, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); var requestBuilder = RequestBuilder.put() .setUri(uri) @@ -392,6 +390,20 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { execute(request, response -> readEntity(response, Void.class)); } + @Override + public void createSubdomain(AthenzDomain parent, String name) { + URI uri = zmsUrl.resolve(String.format("subdomain/%s", parent.getName())); + StringEntity entity = toJsonStringEntity( + Map.of("name", name, + "parent", parent.getName(), + "adminUsers", List.of(identity.getFullName())) // TODO: createSubdomain should receive an adminUsers argument + ); + var request = RequestBuilder.post(uri) + .setEntity(entity) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) { return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token())); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index aa038b5bb23..38d11d33d74 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -61,9 +61,9 @@ public interface ZmsClient extends AutoCloseable { Optional<AthenzPolicy> getPolicy(AthenzDomain domain, String name); - Map<AthenzUser, String> listPendingRoleApprovals(AthenzRole athenzRole); + Map<AthenzIdentity, String> listPendingRoleApprovals(AthenzRole athenzRole); - void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason); + void approvePendingRoleMembership(AthenzRole athenzRole, AthenzIdentity athenzIdentity, Instant expiry, Optional<String> reason); List<AthenzIdentity> listMembers(AthenzRole athenzRole); @@ -81,5 +81,7 @@ public interface ZmsClient extends AutoCloseable { void deleteRole(AthenzRole athenzRole); + void createSubdomain(AthenzDomain parent, String name); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java index 4be82daee43..84d522ee85d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java @@ -6,12 +6,15 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author olaa */ @JsonIgnoreProperties(ignoreUnknown = true) public class PolicyEntity { + private static final Pattern namePattern = Pattern.compile("^(?<domain>[^:]+):policy\\.(?<name>.*)$"); @JsonInclude(JsonInclude.Include.NON_EMPTY) private final List<AssertionEntity> assertions; @@ -19,10 +22,17 @@ public class PolicyEntity { public PolicyEntity(@JsonProperty("name") String name, @JsonProperty("assertions") List<AssertionEntity> assertions) { - this.name = name; + this.name = nameFromResourceString(name); this.assertions = assertions; } + private static String nameFromResourceString(String resource) { + Matcher matcher = namePattern.matcher(resource); + if (!matcher.matches()) + throw new IllegalArgumentException("Could not find policy name from resource string: " + resource); + return matcher.group("name"); + } + public String getName() { return name; } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 79325075a42..137860ba096 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -18,7 +18,7 @@ import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.searchdefinition.parser.ParseException; import org.apache.maven.plugin.AbstractMojo; @@ -114,7 +114,7 @@ public class DocumentGenMojo extends AbstractMojo { annotationTypes = new HashMap<>(); outputDir.mkdirs(); - SchemaBuilder builder = buildSearches(schemasDir); + ApplicationBuilder builder = buildSearches(schemasDir); boolean annotationsExported=false; for (NewDocumentType docType : builder.getModel().getDocumentManager().getTypes()) { @@ -134,21 +134,21 @@ public class DocumentGenMojo extends AbstractMojo { if (project!=null) project.addCompileSourceRoot(outputDirectory.toString()); } - private SchemaBuilder buildSearches(File sdDir) { + private ApplicationBuilder buildSearches(File sdDir) { File[] sdFiles = sdDir.listFiles((dir, name) -> name.endsWith(".sd")); - SchemaBuilder builder = new SchemaBuilder(true); + ApplicationBuilder builder = new ApplicationBuilder(true); for (File f : sdFiles) { try { long modTime = f.lastModified(); if (modTime > newestModifiedTime) { newestModifiedTime = modTime; } - builder.importFile(f.getAbsolutePath()); + builder.addSchemaFile(f.getAbsolutePath()); } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } } - builder.build(); + builder.build(true); for (Schema schema : builder.getSchemaList() ) { this.searches.put(schema.getName(), schema); } diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java index 044f3ae1d14..fdbfae53321 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java @@ -45,6 +45,8 @@ public class JsonFeeder implements Closeable { }); private final FeedClient client; private final OperationParameters protoParameters; + private final AtomicInteger globalInflightOperations = new AtomicInteger(0); + private volatile boolean closed = false; private JsonFeeder(FeedClient client, OperationParameters protoParameters) { this.client = client; @@ -84,6 +86,8 @@ public class JsonFeeder implements Closeable { * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. */ public CompletableFuture<Result> feedSingle(String json) { + if (closed) throw new IllegalStateException("Already closed"); + globalInflightOperations.incrementAndGet(); CompletableFuture<Result> result = new CompletableFuture<>(); try { SingleOperationParserAndExecutor parser = new SingleOperationParserAndExecutor(json.getBytes(UTF_8)); @@ -93,9 +97,11 @@ public class JsonFeeder implements Closeable { } else { result.complete(operationResult); } + globalInflightOperations.decrementAndGet(); }, resultExecutor); } catch (Exception e) { resultExecutor.execute(() -> result.completeExceptionally(wrapException(e))); + globalInflightOperations.decrementAndGet(); } return result; } @@ -125,6 +131,7 @@ public class JsonFeeder implements Closeable { * </pre> * Note that {@code "id"} is an alias for the document put operation. * Exceptional completion will use be an instance of {@link FeedException} or one of its sub-classes. + * The input stream will be closed upon exhaustion, or error. */ public CompletableFuture<Void> feedMany(InputStream jsonStream, ResultCallback resultCallback) { return feedMany(jsonStream, 1 << 26, resultCallback); @@ -139,25 +146,27 @@ public class JsonFeeder implements Closeable { } CompletableFuture<Void> feedMany(InputStream jsonStream, int size, ResultCallback resultCallback) { + if (closed) throw new IllegalStateException("Already closed"); CompletableFuture<Void> overallResult = new CompletableFuture<>(); CompletableFuture<Result> result; - AtomicInteger pending = new AtomicInteger(1); // The below dispatch loop itself is counted as a single pending operation + AtomicInteger localInflightOperations = new AtomicInteger(1); // The below dispatch loop itself is counted as a single pending operation AtomicBoolean finalCallbackInvoked = new AtomicBoolean(); - try { - RingBufferStream buffer = new RingBufferStream(jsonStream, size); + try (RingBufferStream buffer = new RingBufferStream(jsonStream, size)) { while ((result = buffer.next()) != null) { - pending.incrementAndGet(); + localInflightOperations.incrementAndGet(); + globalInflightOperations.incrementAndGet(); result.whenCompleteAsync((r, t) -> { if (!finalCallbackInvoked.get()) { invokeCallback(resultCallback, c -> c.onNextResult(r, (FeedException) t)); } - if (pending.decrementAndGet() == 0 && finalCallbackInvoked.compareAndSet(false, true)) { + if (localInflightOperations.decrementAndGet() == 0 && finalCallbackInvoked.compareAndSet(false, true)) { invokeCallback(resultCallback, ResultCallback::onComplete); overallResult.complete(null); } + globalInflightOperations.decrementAndGet(); }, resultExecutor); } - if (pending.decrementAndGet() == 0 && finalCallbackInvoked.compareAndSet(false, true)) { + if (localInflightOperations.decrementAndGet() == 0 && finalCallbackInvoked.compareAndSet(false, true)) { resultExecutor.execute(() -> { invokeCallback(resultCallback, ResultCallback::onComplete); overallResult.complete(null); @@ -187,6 +196,8 @@ public class JsonFeeder implements Closeable { private static final JsonFactory factory = new JsonFactory(); @Override public void close() throws IOException { + closed = true; + awaitInflightOperations(); client.close(); resultExecutor.shutdown(); try { @@ -198,6 +209,14 @@ public class JsonFeeder implements Closeable { } } + private void awaitInflightOperations() { + try { + while (globalInflightOperations.get() > 0) Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + private FeedException wrapException(Exception e) { if (e instanceof FeedException) return (FeedException) e; if (e instanceof IOException) { @@ -225,7 +244,9 @@ public class JsonFeeder implements Closeable { this.data = new byte[size]; this.size = size; - new Thread(this::fill, "feed-reader").start(); + Thread filler = new Thread(this::fill, "feed-reader"); + filler.setDaemon(true); + filler.start(); this.parserAndExecutor = new RingBufferBackedOperationParserAndExecutor(factory.createParser(this)); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index 2fc7e5af7b4..ac859359bfa 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -54,6 +54,7 @@ class CliArguments { private static final String SILENT_OPTION = "silent"; private static final String VERSION_OPTION = "version"; private static final String STDIN_OPTION = "stdin"; + private static final String DOOM_OPTION = "max-failure-seconds"; private final CommandLine arguments; @@ -149,6 +150,8 @@ class CliArguments { OptionalInt traceLevel() throws CliArgumentsException { return intValue(TRACE_OPTION); } + OptionalInt doomSeconds() throws CliArgumentsException { return intValue(DOOM_OPTION); } + Optional<Duration> timeout() throws CliArgumentsException { OptionalDouble timeout = doubleValue(TIMEOUT_OPTION); return timeout.isPresent() @@ -259,7 +262,7 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(BENCHMARK_OPTION) - .desc("Enable benchmark mode") + .desc("Print statistics to stdout when done") .build()) .addOption(Option.builder() .longOpt(ROUTE_OPTION) @@ -279,8 +282,14 @@ class CliArguments { .type(Number.class) .build()) .addOption(Option.builder() - .longOpt(STDIN_OPTION) - .desc("Read JSON input from standard input") + .longOpt(STDIN_OPTION) + .desc("Read JSON input from standard input") + .build()) + .addOption(Option.builder() + .longOpt(DOOM_OPTION) + .desc("Exit if specified number of seconds ever pass without any successful operations. Disabled by default") + .hasArg() + .type(Number.class) .build()) .addOption(Option.builder() .longOpt(DRYRUN_OPTION) @@ -292,7 +301,7 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(SILENT_OPTION) - .desc("Disable periodic status printing") + .desc("Disable periodic status printing to stderr") .build()) .addOption(Option.builder() .longOpt(SHOW_ERRORS_OPTION) diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 7e036b8dec3..5e904b37588 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Files; +import java.time.Duration; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -94,9 +95,7 @@ public class CliClient { }); latch.await(); - if (cliArgs.benchmarkModeEnabled()) { - printBenchmarkResult(System.nanoTime() - startNanos, successes.get(), failures.get(), feedClient.stats(), systemOut); - } + printBenchmarkResult(System.nanoTime() - startNanos, successes.get(), failures.get(), feedClient.stats(), cliArgs.benchmarkModeEnabled() ? systemOut : systemError); if (fatal.get() != null) throw fatal.get(); } return 0; @@ -137,6 +136,8 @@ public class CliClient { cliArgs.caCertificates().ifPresent(builder::setCaCertificatesFile); cliArgs.headers().forEach(builder::addRequestHeader); builder.setDryrun(cliArgs.dryrunEnabled()); + cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10), + Duration.ofSeconds(doom)))); return builder.build(); } diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh index c4e70c362b0..fc99a68614a 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh @@ -4,6 +4,5 @@ exec java \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m \ ---add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.util.logging.config.file=`dirname $0`/logging.properties \ -cp `dirname $0`/vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.impl.CliClient "$@" diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh index 7dbdc056524..6e9aefd56c7 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh @@ -79,6 +79,5 @@ exec java \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m $(getJavaOptionsIPV46) \ ---add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.util.logging.config.file=${VESPA_HOME}/conf/vespa-feed-client/logging.properties \ -cp ${VESPA_HOME}/lib/jars/vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.impl.CliClient "$@" diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java index 19b93c3172b..fe3dc465814 100644 --- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java @@ -25,7 +25,8 @@ class CliArgumentsTest { "--max-streams-per-connection=128", "--certificate=cert.pem", "--private-key=key.pem", "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value", "--benchmark", - "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", "--show-errors", "--show-all"}); + "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", + "--show-errors", "--show-all", "--max-failure-seconds=30"}); assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); assertEquals(Paths.get("feed.json"), args.inputFile().get()); assertEquals(10, args.connections().getAsInt()); @@ -43,6 +44,7 @@ class CliArgumentsTest { assertEquals("myroute", args.route().get()); assertEquals(Duration.ofMillis(125), args.timeout().get()); assertEquals(9, args.traceLevel().getAsInt()); + assertEquals(30, args.doomSeconds().getAsInt()); assertTrue(args.verboseSpecified()); assertTrue(args.showErrors()); assertTrue(args.showSuccesses()); diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 67b83c07699..323206ab128 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -1,6 +1,7 @@ usage: vespa-feed-client <options> Vespa feed client - --benchmark Enable benchmark mode + --benchmark Print statistics to stdout when + done --ca-certificates <arg> Path to file containing CA X.509 certificates encoded as PEM --certificate <arg> Path to PEM encoded X.509 @@ -16,6 +17,10 @@ Vespa feed client --header <arg> HTTP header on the form 'Name: value' --help + --max-failure-seconds <arg> Exit if specified number of + seconds ever pass without any + successful operations. Disabled + by default --max-streams-per-connection <arg> Maximum number of concurrent streams per HTTP/2 connection --private-key <arg> Path to PEM/PKCS#8 encoded @@ -27,6 +32,7 @@ Vespa feed client --show-errors Print every feed operation failure --silent Disable periodic status printing + to stderr --stdin Read JSON input from standard input --timeout <arg> Feed operation timeout (in diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index 1874bd42e16..b51210d22ea 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -16,6 +16,10 @@ import org.apache.hc.core5.net.URIAuthority; import org.apache.hc.core5.reactor.IOReactorConfig; import org.apache.hc.core5.util.Timeout; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import javax.net.ssl.SSLEngine; + import javax.net.ssl.SSLContext; import java.io.IOException; import java.net.URI; @@ -132,6 +136,8 @@ class ApacheCluster implements Cluster { throw new IllegalStateException("No adequate SSL cipher suites supported by the JVM"); ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create() + .setTlsDetailsFactory(sslEngine -> + new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol())) .setCiphers(allowedCiphers) .setSslContext(sslContext); if (builder.hostnameVerifier != null) diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java index 40049bad217..f59eee25d42 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java @@ -74,8 +74,8 @@ public class BenchmarkingCluster implements Cluster { return getStats(); } } - catch (InterruptedException | ExecutionException ignored) { - throw new RuntimeException(ignored); + catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java index 6fec0029bc3..2d052e8a6af 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java @@ -19,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -298,6 +299,14 @@ class HttpRequestStrategy implements RequestStrategy { inflightById.values().forEach(RetriableFuture::complete); cluster.close(); resultExecutor.shutdown(); + try { + if ( ! resultExecutor.awaitTermination(1, TimeUnit.MINUTES)) + log.log(WARNING, "Failed processing results within 1 minute"); + } + catch (InterruptedException e) { + log.log(WARNING, "Interrupted waiting for results to be processed"); + Thread.currentThread().interrupt(); + } } } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java index 13dc7ad4624..60e8c106b40 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java @@ -174,18 +174,20 @@ class HttpRequestStrategyTest { now.set(605000); assertEquals(OPEN, breaker.state()); // Circuit broken due to failed requests. + strategy.destroy(); + OperationStats stats = strategy.stats(); Map<Integer, Long> codes = new HashMap<>(); codes.put(200, 4L); codes.put(400, 1L); codes.put(429, 2L); codes.put(500, 3L); - assertEquals(codes, strategy.stats().responsesByCode()); - assertEquals(3, strategy.stats().exceptions()); + assertEquals(codes, stats.responsesByCode()); + assertEquals(3, stats.exceptions()); - assertEquals(strategy.stats(), strategy.stats().since(initial)); - assertEquals(0, strategy.stats().since(strategy.stats()).averageLatencyMillis()); - assertEquals(0, strategy.stats().since(strategy.stats()).requests()); - assertEquals(0, strategy.stats().since(strategy.stats()).bytesSent()); + assertEquals(stats, stats.since(initial)); + assertEquals(0, stats.since(stats).averageLatencyMillis()); + assertEquals(0, stats.since(stats).requests()); + assertEquals(0, stats.since(stats).bytesSent()); } @Test diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 3cf669f924a..2530e461354 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -17,7 +17,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>2.8.0</hadoop.version> + <hadoop.version>2.10.1</hadoop.version> <pig.version>0.14.0</pig.version> </properties> @@ -187,16 +187,6 @@ <shadedPattern>shaded.vespa</shadedPattern> </relocation> </relocations> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> </execution> </executions> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java index 89e48a16909..42abe9e6131 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java @@ -148,7 +148,7 @@ public class LegacyVespaRecordWriter extends RecordWriter { while (tokenizer.hasMoreTokens()) { String endpoint = tokenizer.nextToken().trim(); sessionParams.addCluster(new Cluster.Builder().addEndpoint( - Endpoint.create(endpoint, configuration.defaultPort(), configuration.useSSL()) + Endpoint.create(endpoint, configuration.defaultPort(), configuration.useSSL().orElse(false)) ).build()); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java index 00ce396fd09..66ab94574d9 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java @@ -3,10 +3,18 @@ package com.yahoo.vespa.hadoop.mapreduce; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; -import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; import java.io.IOException; +import java.util.Objects; import java.util.Properties; +import java.util.logging.Logger; + +import static com.yahoo.vespa.http.client.config.FeedParams.DataFormat.XML_UTF8; /** * An output specification for writing to Vespa instances in a Map-Reduce job. @@ -18,6 +26,8 @@ import java.util.Properties; @SuppressWarnings("rawtypes") public class VespaOutputFormat extends OutputFormat { + private static final Logger log = Logger.getLogger(VespaOutputFormat.class.getName()); + final Properties configOverride; public VespaOutputFormat() { @@ -33,12 +43,17 @@ public class VespaOutputFormat extends OutputFormat { @Override @SuppressWarnings("deprecation") - public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { + public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { VespaCounters counters = VespaCounters.get(context); VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride); - return configuration.useLegacyClient().orElse(true) - ? new LegacyVespaRecordWriter(configuration, counters) - : new VespaRecordWriter(configuration, counters); + Boolean useLegacyClient = configuration.useLegacyClient().orElse(null); + if (Objects.equals(useLegacyClient, Boolean.TRUE) || configuration.dataFormat() == XML_UTF8) { + log.warning("Feeding with legacy client or XML will no longer be supported on Vespa 8. " + + "See https://docs.vespa.ai/en/vespa8-release-notes.html"); + return new LegacyVespaRecordWriter(configuration, counters); + } else { + return new VespaRecordWriter(configuration, counters); + } } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index ecac9f67615..c381ec87492 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -147,8 +147,9 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { } private static List<URI> endpointUris(VespaConfiguration config) { + String scheme = config.useSSL().orElse(true) ? "https" : "http"; return Arrays.stream(config.endpoint().split(",")) - .map(hostname -> URI.create(String.format("https://%s:%d/", hostname, config.defaultPort()))) + .map(hostname -> URI.create(String.format("%s://%s:%d/", scheme, hostname, config.defaultPort()))) .collect(toList()); } } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index fad14532d38..1421a3fcd43 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -54,8 +54,10 @@ public class VespaConfiguration { } - public boolean useSSL() { - return getBoolean(USE_SSL, false); + public Optional<Boolean> useSSL() { + String raw = getString(USE_SSL); + if (raw == null || raw.trim().isEmpty()) return Optional.empty(); + return Optional.of(Boolean.parseBoolean(raw)); } @@ -181,7 +183,7 @@ public class VespaConfiguration { StringBuilder sb = new StringBuilder(); sb.append(ENDPOINT + ": " + endpoint() + "\n"); sb.append(DEFAULT_PORT + ": " + defaultPort() + "\n"); - sb.append(USE_SSL + ": " + useSSL() + "\n"); + sb.append(USE_SSL + ": " + useSSL().map(Object::toString).orElse("<empty>") + "\n"); sb.append(PROXY_HOST + ": " + proxyHost() + "\n"); sb.append(PROXY_PORT + ": " + proxyPort() + "\n"); sb.append(DRYRUN + ": " + dryrun() +"\n"); diff --git a/vespa-hadoop/src/test/resources/visit_data.json b/vespa-hadoop/src/test/resources/visit_data.json index a48fc9cf1c0..947b9326cc8 100644 --- a/vespa-hadoop/src/test/resources/visit_data.json +++ b/vespa-hadoop/src/test/resources/visit_data.json @@ -1,12 +1,10 @@ -[ -{"id":"id:testapp:metric::clicks-2015110414","fields":{"date":"2015110414","name":"clicks","value":1,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110415","fields":{"date":"2015110415","name":"clicks","value":2,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110416","fields":{"date":"2015110416","name":"clicks","value":4,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110417","fields":{"date":"2015110417","name":"clicks","value":3,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110418","fields":{"date":"2015110418","name":"clicks","value":6,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110419","fields":{"date":"2015110419","name":"clicks","value":3,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110420","fields":{"date":"2015110420","name":"clicks","value":4,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110421","fields":{"date":"2015110421","name":"clicks","value":2,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110422","fields":{"date":"2015110422","name":"clicks","value":7,"application":"testapp"}}, -{"id":"id:testapp:metric::clicks-2015110423","fields":{"date":"2015110423","name":"clicks","value":1,"application":"testapp"}} -] +{"id":"id:testapp:metric::clicks-2015110414","fields":{"date":"2015110414","name":"clicks","value":1,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110415","fields":{"date":"2015110415","name":"clicks","value":2,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110416","fields":{"date":"2015110416","name":"clicks","value":4,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110417","fields":{"date":"2015110417","name":"clicks","value":3,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110418","fields":{"date":"2015110418","name":"clicks","value":6,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110419","fields":{"date":"2015110419","name":"clicks","value":3,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110420","fields":{"date":"2015110420","name":"clicks","value":4,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110421","fields":{"date":"2015110421","name":"clicks","value":2,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110422","fields":{"date":"2015110422","name":"clicks","value":7,"application":"testapp"}} +{"id":"id:testapp:metric::clicks-2015110423","fields":{"date":"2015110423","name":"clicks","value":1,"application":"testapp"}}
\ No newline at end of file diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index b25f54362ed..c4f7fd95ffd 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -170,17 +170,6 @@ </goals> <configuration> <outputFile>target/${project.artifactId}-jar-with-dependencies.jar</outputFile> - <filters> - <filter> - <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>com.yahoo.vespa.http.client.runner.Runner</mainClass> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java index 29ca1e2d4fd..fa214808ae3 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java @@ -16,6 +16,8 @@ import java.time.Instant; import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; /** * Implementation of FeedClient. It is a thin layer on top of multiClusterHandler and multiClusterResultAggregator. @@ -24,6 +26,9 @@ import java.util.concurrent.TimeUnit; */ public class FeedClientImpl implements FeedClient { + private static final Logger log = Logger.getLogger(FeedClientImpl.class.getName()); + private static final AtomicBoolean warningPrinted = new AtomicBoolean(false); + private final Clock clock; private final OperationProcessor operationProcessor; private final long closeTimeoutMs; @@ -46,6 +51,10 @@ public class FeedClientImpl implements FeedClient { sessionParams, timeoutExecutor, clock); + if (warningPrinted.compareAndSet(false, true)) { + log.warning("The vespa-http-client is deprecated and will be removed in Vespa 8. " + + "See https://docs.vespa.ai/en/vespa8-release-notes.html"); + } } @Override diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java index 82c1f7194d0..5637d25df71 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java @@ -52,12 +52,14 @@ public class AggregateTestRunner implements TestRunner { return Status.NOT_STARTED; boolean failed = false; + boolean inconclusive = false; for (int i = 0; i <= current.get(); i++) { if (i == wrapped.size()) - return failed ? Status.FAILURE : Status.SUCCESS; + return failed ? Status.FAILURE : inconclusive ? Status.INCONCLUSIVE : Status.SUCCESS; switch (wrapped.get(i).getStatus()) { case ERROR: return Status.ERROR; + case INCONCLUSIVE: inconclusive = true; break; case FAILURE: failed = true; } } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 6aa36c62416..dfa37780eb5 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -5,6 +5,7 @@ import ai.vespa.cloud.Environment; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; import ai.vespa.hosted.api.TestDescriptor; +import ai.vespa.hosted.cd.InconclusiveTestException; import ai.vespa.hosted.cd.internal.TestRuntimeProvider; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; @@ -204,11 +205,16 @@ public class JunitRunner extends AbstractComponent implements TestRunner { var failures = report.getFailures().stream() .map(failure -> new TestReport.Failure(failure.getTestIdentifier().getUniqueId(), failure.getException())) .collect(Collectors.toList()); + long inconclusive = failures.stream() + .filter(failure -> failure.exception() != null + && failure.exception().getClass().getName().equals(InconclusiveTestException.class.getName())) + .count(); return TestReport.builder() .withSuccessCount(report.getTestsSucceededCount()) .withAbortedCount(report.getTestsAbortedCount()) .withIgnoredCount(report.getTestsSkippedCount()) - .withFailedCount(report.getTestsFailedCount()) + .withFailedCount(report.getTotalFailureCount() - inconclusive) + .withInconclusiveCount(inconclusive) .withFailures(failures) .withLogs(logRecords.values()) .build(); @@ -230,12 +236,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { if (execution == null) return TestRunner.Status.NOT_STARTED; if (!execution.isDone()) return TestRunner.Status.RUNNING; try { - TestReport report = execution.get(); - if (report.isSuccess()) { - return TestRunner.Status.SUCCESS; - } else { - return TestRunner.Status.FAILURE; - } + return execution.get().status(); } catch (InterruptedException|ExecutionException e) { logger.log(Level.WARNING, "Error while getting test report", e); return TestRunner.Status.ERROR; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java index b3ca47e2480..8cced4f8cd6 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java @@ -6,6 +6,10 @@ import java.util.Collections; import java.util.List; import java.util.logging.LogRecord; +import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; + /** * @author mortent */ @@ -14,15 +18,17 @@ public class TestReport { final long totalCount; final long successCount; final long failedCount; + final long inconclusiveCount; final long ignoredCount; final long abortedCount; final List<Failure> failures; final List<LogRecord> logLines; - private TestReport(long totalCount, long successCount, long failedCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) { + private TestReport(long totalCount, long successCount, long failedCount, long inconclusiveCount, long ignoredCount, long abortedCount, List<Failure> failures, List<LogRecord> logLines) { this.totalCount = totalCount; this.successCount = successCount; this.failedCount = failedCount; + this.inconclusiveCount = inconclusiveCount; this.ignoredCount = ignoredCount; this.abortedCount = abortedCount; this.failures = failures; @@ -33,8 +39,8 @@ public class TestReport { return logLines; } - public boolean isSuccess() { - return (failedCount + abortedCount) == 0; + public TestRunner.Status status() { + return failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : SUCCESS; } public static Builder builder(){ @@ -47,13 +53,14 @@ public class TestReport { private long totalCount; private long successCount; private long failedCount; + private long inconclusiveCount; private long ignoredCount; private long abortedCount; private List<Failure> failures = Collections.emptyList(); private List<LogRecord> logLines = Collections.emptyList(); public TestReport build() { - return new TestReport(totalCount, successCount, failedCount, ignoredCount, abortedCount, failures, logLines); + return new TestReport(totalCount, successCount, failedCount, inconclusiveCount, ignoredCount, abortedCount, failures, logLines); } public Builder withTotalCount(long totalCount) { @@ -71,6 +78,11 @@ public class TestReport { return this; } + public Builder withInconclusiveCount(long inconclusiveCount) { + this.inconclusiveCount = inconclusiveCount; + return this; + } + public Builder withIgnoredCount(long ignoredCount) { this.ignoredCount = ignoredCount; return this; @@ -95,6 +107,7 @@ public class TestReport { public static class Failure { + private final String testId; private final Throwable exception; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java index d70a3f60c7d..20ff19266cf 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunner.java @@ -22,7 +22,7 @@ public interface TestRunner { default TestReport getReport() { return null; } enum Status { - NOT_STARTED, RUNNING, FAILURE, ERROR, SUCCESS + NOT_STARTED, RUNNING, FAILURE, INCONCLUSIVE, ERROR, SUCCESS } enum Suite { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index 62601e4dfa0..e7b0fe406c7 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -6,7 +6,7 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.exception.ExceptionUtils; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.SlimeJsonResponse; @@ -27,10 +27,10 @@ import static com.yahoo.jdisc.Response.Status; /** * @author valerijf - * @author jvenstad + * @author jonmv * @author mortent */ -public class TestRunnerHandler extends LoggingRequestHandler { +public class TestRunnerHandler extends ThreadedHttpRequestHandler { private final TestRunner testRunner; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index f131292597c..9ae75fbae43 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; import static com.yahoo.yolean.Exceptions.uncheck; @@ -88,7 +89,12 @@ public class VespaCliTestRunner implements TestRunner { HtmlLogger htmlLogger = new HtmlLogger(); BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); in.lines().forEach(line -> log(htmlLogger.toLog(line))); - status.set(process.waitFor() == 0 ? SUCCESS : process.waitFor() == 3 ? FAILURE : ERROR); + int exitCode = process.waitFor(); + status.set(exitCode == 0 ? SUCCESS : exitCode == 3 ? FAILURE : exitCode == 4 ? INCONCLUSIVE : ERROR); + } + catch (NoTestsException e) { + log(Level.WARNING, "Did not find expected basic HTTP tests", e); + status.set(FAILURE); } catch (Exception e) { if (process != null) @@ -109,7 +115,7 @@ public class VespaCliTestRunner implements TestRunner { ProcessBuilder testRunProcessBuilder(Suite suite, TestConfig config) throws IOException { Path suitePath = getChildDirectory(testsPath, toSuiteDirectoryName(suite)) - .orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'")); + .orElseThrow(() -> new NoTestsException("No tests found, for suite '" + suite + "'")); ProcessBuilder builder = new ProcessBuilder("vespa", "test", suitePath.toAbsolutePath().toString(), "--application", config.application().toFullString(), @@ -166,4 +172,10 @@ public class VespaCliTestRunner implements TestRunner { return new String(SlimeUtils.toJsonBytes(root), UTF_8); } + static class NoTestsException extends RuntimeException { + + private NoTestsException(String message) { super(message); } + + } + } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java index 64f9079643a..a7ddbbf3fcc 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java @@ -12,6 +12,7 @@ import java.util.logging.LogRecord; import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; +import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; import static com.yahoo.vespa.testrunner.TestRunner.Status.NOT_STARTED; import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; @@ -85,6 +86,10 @@ class AggregateTestRunnerTest { second.future.complete(null); assertEquals(SUCCESS, runner.getStatus()); + // An inconclusive test means inconclusive. + first.status = INCONCLUSIVE; + assertEquals(INCONCLUSIVE, runner.getStatus()); + // A failure means failure. second.status = FAILURE; assertEquals(FAILURE, runner.getStatus()); diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java index 9ba8dfdc4dc..46d7b78cdea 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.testrunner; import ai.vespa.hosted.api.TestConfig; +import com.yahoo.vespa.testrunner.VespaCliTestRunner.NoTestsException; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -51,8 +52,8 @@ class VespaCliTestRunnerTest { Path systemTests = Files.createDirectory(tests.resolve("system-test")); assertTrue(runner.isSupported()); - IllegalStateException ise = assertThrows(IllegalStateException.class, - () -> runner.testRunProcessBuilder(TestRunner.Suite.STAGING_TEST, testConfig)); + NoTestsException ise = assertThrows(NoTestsException.class, + () -> runner.testRunProcessBuilder(TestRunner.Suite.STAGING_TEST, testConfig)); assertEquals("No tests found, for suite 'STAGING_TEST'", ise.getMessage()); ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, testConfig); diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index 6c25b4f6329..a8168b6c4f7 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -62,17 +62,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>com.yahoo.vespa.feed.perf.SimpleFeeder</mainClass> diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index d2677e58a00..7e1fbf5d06a 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -148,15 +148,16 @@ consider_fallback VESPA_VALGRIND_OPT "--num-callers=32 \ --show-reachable=yes \ ${VESPA_VALGRIND_SUPPREESSIONS_OPT}" -consider_fallback VESPA_USE_HUGEPAGES_LIST $(get_var "hugepages_list") +consider_fallback VESPA_USE_HUGEPAGES_LIST "$(get_var hugepages_list)" consider_fallback VESPA_USE_HUGEPAGES_LIST "all" -consider_fallback VESPA_USE_MADVISE_LIST $(get_var "madvise_list") +consider_fallback VESPA_USE_MADVISE_LIST "$(get_var madvise_list)" -consider_fallback VESPA_USE_VESPAMALLOC $(get_var "vespamalloc_list") -consider_fallback VESPA_USE_VESPAMALLOC_D $(get_var "vespamallocd_list") -consider_fallback VESPA_USE_VESPAMALLOC_DST $(get_var "vespamallocdst_list") -consider_fallback VESPA_USE_NO_VESPAMALLOC $(get_var "no_vespamalloc_list") +consider_fallback VESPA_USE_VESPAMALLOC "$(get_var vespamalloc_list)" +consider_fallback VESPA_USE_VESPAMALLOC_D "$(get_var vespamallocd_list)" +consider_fallback VESPA_USE_VESPAMALLOC_DST "$(get_var vespamallocdst_list)" +consider_fallback VESPA_USE_NO_VESPAMALLOC "$(get_var no_vespamalloc_list)" +consider_fallback VESPA_USE_NO_VESPAMALLOC "vespa-rpc-invoke vespa-get-config vespa-sentinel-cmd vespa-route vespa-proton-cmd vespa-configproxy-cmd vespa-config-status" # TODO: # if [ "$VESPA_USE_HUGEPAGES_LIST" = "all" ]; then diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh index 52c4c5a0de8..2f0a923b074 100755 --- a/vespabase/src/start-cbinaries.sh +++ b/vespabase/src/start-cbinaries.sh @@ -97,7 +97,7 @@ check_bname_in_value () { configure_valgrind () { no_valgrind=true if which valgrind >/dev/null 2>&1; then - if check_bname_in_value $VESPA_USE_VALGRIND; then + if check_bname_in_value "$VESPA_USE_VALGRIND"; then no_valgrind=false valgrind_log=$VESPA_HOME/tmp/valgrind.$bname.log.$$ case $VESPA_VALGRIND_OPT in @@ -108,7 +108,7 @@ configure_valgrind () { } configure_huge_pages () { - if check_bname_in_value $VESPA_USE_HUGEPAGES_LIST; then + if check_bname_in_value "$VESPA_USE_HUGEPAGES_LIST"; then log_debug_message "Want huge pages for '$bname' since VESPA_USE_HUGEPAGES_LIST=${VESPA_USE_HUGEPAGES_LIST}" export VESPA_USE_HUGEPAGES="yes" fi @@ -134,15 +134,15 @@ configure_use_madvise () { } configure_vespa_malloc () { - if check_bname_in_value $VESPA_USE_NO_VESPAMALLOC; then + if check_bname_in_value "$VESPA_USE_NO_VESPAMALLOC"; then # log_debug_message "Not using vespamalloc for '$bname' since VESPA_USE_NO_VESPAMALLOC=${VESPA_USE_NO_VESPAMALLOC}" return fi suf=vespa/malloc/libvespamalloc.so - if check_bname_in_value $VESPA_USE_VESPAMALLOC_D; then + if check_bname_in_value "$VESPA_USE_VESPAMALLOC_D"; then suf=vespa/malloc/libvespamallocd.so fi - if check_bname_in_value $VESPA_USE_VESPAMALLOC_DST; then + if check_bname_in_value "$VESPA_USE_VESPAMALLOC_DST"; then suf=vespa/malloc/libvespamallocdst16.so fi diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 180ddca2ca5..4d7c2f1cc14 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -20,6 +20,7 @@ import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.fieldset.DocIdOnly; import com.yahoo.document.idstring.IdIdString; import com.yahoo.document.json.DocumentOperationType; @@ -379,6 +380,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { StorageCluster destination = resolveCluster(Optional.of(requireProperty(request, DESTINATION_CLUSTER)), clusters); VisitorParameters parameters = parseParameters(request, path); parameters.setRemoteDataHandler("[Content:cluster=" + destination.name() + "]"); // Bypass indexing. + // TODO Vespa 8: change to DocumentOnly.NAME parameters.setFieldSet(AllFields.NAME); return () -> { visitWithRemote(request, parameters, handler); @@ -1088,6 +1090,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { throw new IllegalArgumentException("Must set 'cluster' parameter to a valid content cluster id when visiting at a root /document/v1/ level"); VisitorParameters parameters = parseCommonParameters(request, path, cluster); + // TODO Vespa 8: change to DocumentOnly.NAME parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(AllFields.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); parameters.visitInconsistentBuckets(true); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 8cc054eb96c..bc1e58a4990 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -4,7 +4,7 @@ package com.yahoo.document.restapi.resource; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Metric; /** @@ -12,7 +12,7 @@ import com.yahoo.jdisc.Metric; * * @author jonmv */ -public class RestApi extends LoggingRequestHandler { +public class RestApi extends ThreadedHttpRequestHandler { @Inject public RestApi() { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index d57ef11f2d3..66955be2325 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -8,6 +8,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.ResourceReference; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; @@ -71,22 +72,25 @@ class ClientFeederV3 { this.hostName = HostName.getLocalhost(); } - public boolean timedOut() { + boolean timedOut() { synchronized (monitor) { return Instant.now().isAfter(prevOpsPerSecTime.plusSeconds(6000)) && ongoingRequests.get() == 0; } } - public void kill() { - // No new requests should be sent to this object, but there can be old one, even though this is very unlikely. - while (ongoingRequests.get() > 0) { - try { - ongoingRequests.wait(100); - } catch (InterruptedException e) { - break; + void kill() { + try (ResourceReference ignored = sourceSession.getReference()) { + // No new requests should be sent to this object, but there can be old one, even though this is very unlikely. + while (ongoingRequests.get() > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + return; + } } + } catch (Exception e) { + log.log(Level.WARNING, "Failed to close reference to source session", e); } - sourceSession.getReference().close(); } private void transferPreviousRepliesToResponse(BlockingQueue<OperationStatus> operations) throws InterruptedException { @@ -98,7 +102,7 @@ class ClientFeederV3 { } } - public HttpResponse handleRequest(HttpRequest request) throws IOException { + HttpResponse handleRequest(HttpRequest request) throws IOException { ongoingRequests.incrementAndGet(); try { FeederSettings feederSettings = new FeederSettings(request); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 4b01835ea3c..2a774995601 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -5,7 +5,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.metrics.DocumentApiMetrics; @@ -34,7 +34,7 @@ import java.util.zip.GZIPInputStream; * * @author Steinar Knutsen */ -public class FeedHandler extends LoggingRequestHandler { +public class FeedHandler extends ThreadedHttpRequestHandler { protected final ReplyHandler feedReplyHandler; private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(3)); @@ -144,20 +144,5 @@ public class FeedHandler extends LoggingRequestHandler { } } - @Override - protected void destroy() { - feedHandlerV3.destroy(); - // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. - // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this - // and started destructing something that required something only the messenger thread could provide. - Thread destroyer = new Thread(() -> { - internalDestroy(); - }); - destroyer.setDaemon(true); - destroyer.start(); - } - - private void internalDestroy() { - super.destroy(); - } + @Override protected void destroy() { feedHandlerV3.destroy(); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 8ec05f158c9..95e98f325e1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.http.server; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; @@ -18,7 +18,6 @@ import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.yolean.Exceptions; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -33,7 +32,7 @@ import java.util.logging.Logger; * * @author dybis */ -public class FeedHandlerV3 extends LoggingRequestHandler { +public class FeedHandlerV3 extends ThreadedHttpRequestHandler { private DocumentTypeManager docTypeManager; private final Map<String, ClientFeederV3> clientFeederByClientId = new HashMap<>(); @@ -53,8 +52,8 @@ public class FeedHandlerV3 extends LoggingRequestHandler { docTypeManager = new DocumentTypeManager(documentManagerConfig); this.sessionCache = sessionCache; feedReplyHandler = new FeedReplyReader(metric, metricsHelper); - cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron")); - cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES); + cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feed-handler-v3-janitor")); + cron.scheduleWithFixedDelay(this::removeOldClients, 3, 3, TimeUnit.SECONDS); this.metric = metric; } @@ -101,19 +100,18 @@ public class FeedHandlerV3 extends LoggingRequestHandler { @Override protected void destroy() { - // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. + // We are forking this to avoid that accidental de-referencing causes any random thread doing destruction. // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this // and started destructing something that required something only the messenger thread could provide. Thread destroyer = new Thread(() -> { - super.destroy(); cron.shutdown(); synchronized (monitor) { - for (ClientFeederV3 client : clientFeederByClientId.values()) { - client.kill(); + for (var iterator = clientFeederByClientId.values().iterator(); iterator.hasNext(); ) { + iterator.next().kill(); + iterator.remove(); } - clientFeederByClientId.clear(); } - }); + }, "feed-handler-v3-adhoc-destroyer"); destroyer.setDaemon(true); destroyer.start(); } @@ -142,9 +140,8 @@ public class FeedHandlerV3 extends LoggingRequestHandler { private void removeOldClients() { synchronized (monitor) { - for (Iterator<Map.Entry<String, ClientFeederV3>> iterator = clientFeederByClientId - .entrySet().iterator(); iterator.hasNext();) { - ClientFeederV3 client = iterator.next().getValue(); + for (var iterator = clientFeederByClientId.values().iterator(); iterator.hasNext(); ) { + ClientFeederV3 client = iterator.next(); if (client.timedOut()) { client.kill(); iterator.remove(); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index edf51d1bcd6..e5da51f0918 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; * Utility class for assigning properties to messages, either from implicit * config values or from explicit values in requests. */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscriber<FeederConfig> { private static final Logger log = Logger.getLogger(MessagePropertyProcessor.class.getName()); diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index d8530fd9d82..0db7c523eb5 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -75,17 +75,6 @@ <artifactId>maven-shade-plugin</artifactId> <configuration> <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> </configuration> <executions> <execution> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index 0eef559da3f..da9f48f44b1 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -2,6 +2,7 @@ package com.yahoo.vespaget; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.fieldset.DocIdOnly; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import org.apache.commons.cli.CommandLine; @@ -67,6 +68,7 @@ public class CommandLineOptions { .longOpt(PRINTIDS_OPTION) .build()); + // TODO Vespa 8: change to DocumentOnly.NAME options.addOption(Option.builder("f") .hasArg(true) .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets) (default '" + AllFields.NAME + "')") @@ -181,8 +183,9 @@ public class CommandLineOptions { if (printIdsOnly) { fieldSet = DocIdOnly.NAME; - } else if (fieldSet.isEmpty()) { - fieldSet = AllFields.NAME; + } else if (fieldSet.isEmpty()) { + // TODO Vespa 8: change to DocumentOnly.NAME + fieldSet = AllFields.NAME; } if (!cluster.isEmpty() && !route.isEmpty()) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index f74f7397ae3..4d79f2f2e1d 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -523,9 +523,6 @@ public class VdsVisit { if (line.hasOption("processtime")) { allParams.setProcessTime(((Number) line.getParsedOptionValue("processtime")).intValue()); } - if (line.hasOption("maxhits")) { - params.setMaxFirstPassHits(((Number)line.getParsedOptionValue("maxhits")).intValue()); - } if (line.hasOption("maxtotalhits")) { params.setMaxTotalHits(((Number)line.getParsedOptionValue("maxtotalhits")).intValue()); } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java index 6af2344e36e..b634e899b74 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespaget; import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.fieldset.DocIdOnly; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import org.junit.Rule; @@ -55,6 +56,7 @@ public class CommandLineOptionsTest { assertFalse(params.help); assertFalse(params.documentIds.hasNext()); assertFalse(params.printIdsOnly); + // TODO Vespa 8: change to DocumentOnly.NAME assertEquals(AllFields.NAME, params.fieldSet); assertEquals("default-get", params.route); assertTrue(params.cluster.isEmpty()); diff --git a/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp index de8500b3e26..8e990c81f29 100644 --- a/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp +++ b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "clusterlist.h" -#include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> #include <sstream> @@ -17,7 +16,7 @@ ClusterList::Cluster::Cluster(const std::string& name, const std::string& config ClusterList::Cluster::Cluster(const Cluster &) = default; ClusterList::Cluster & ClusterList::Cluster::operator = (const Cluster &) = default; -ClusterList::Cluster::~Cluster() {} +ClusterList::Cluster::~Cluster() = default; ClusterList::ClusterList() { diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.h b/vespaclient/src/vespa/vespaclient/vesparoute/application.h index 93b287b4559..b7a33134b0e 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/application.h +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.h @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/app.h> -#include <vespa/messagebus/messagebus.h> -#include <vespa/messagebus/routing/hopblueprint.h> #include "mynetwork.h" #include "params.h" - +#include <vespa/messagebus/messagebus.h> +#include <vespa/messagebus/routing/hopblueprint.h> +#include <vespa/fastos/app.h> +#include <set> namespace vesparoute { /** diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 71e6c0f28f4..a30ee055538 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -2658,8 +2658,7 @@ "public java.util.Optional dimension()", "public java.util.Optional label()", "public java.util.Optional index()", - "public java.lang.String toString()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString()" ], "fields": [] }, @@ -2744,6 +2743,7 @@ "methods": [ "public static com.yahoo.tensor.functions.ToStringContext empty()", "public abstract java.lang.String getBinding(java.lang.String)", + "public java.util.Optional typeContext()", "public abstract com.yahoo.tensor.functions.ToStringContext parent()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java index d133af2ea84..3d1e82743cc 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java +++ b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java @@ -153,44 +153,45 @@ public class BobHash { // handle the last 11 bytes c += k.length; switch (len) { - // all the case statements fall through - case 11: - c += (unsign(k[offset + 10]) << 24); + // all the case statements fall through + case 11: + c += (unsign(k[offset + 10]) << 24); - case 10: - c += (unsign(k[offset + 9]) << 16); + case 10: + c += (unsign(k[offset + 9]) << 16); - case 9: - c += (unsign(k[offset + 8]) << 8); + case 9: + c += (unsign(k[offset + 8]) << 8); - /* the first byte of c is reserved for the length */ - case 8: - b += (unsign(k[offset + 7]) << 24); + /* the first byte of c is reserved for the length */ + case 8: + b += (unsign(k[offset + 7]) << 24); - case 7: - b += (unsign(k[offset + 6]) << 16); + case 7: + b += (unsign(k[offset + 6]) << 16); - case 6: - b += (unsign(k[offset + 5]) << 8); + case 6: + b += (unsign(k[offset + 5]) << 8); - case 5: - b += unsign(k[offset + 4]); + case 5: + b += unsign(k[offset + 4]); - case 4: - a += (unsign(k[offset + 3]) << 24); + case 4: + a += (unsign(k[offset + 3]) << 24); - case 3: - a += (unsign(k[offset + 2]) << 16); + case 3: + a += (unsign(k[offset + 2]) << 16); - case 2: - a += (unsign(k[offset + 1]) << 8); + case 2: + a += (unsign(k[offset + 1]) << 8); - case 1: - a += unsign(k[offset + 0]); + case 1: + a += unsign(k[offset + 0]); - /* case 0: nothing left to add */ + /* case 0: nothing left to add */ } abcBuffer = mix(a, b, c); return abcBuffer[2]; } + } diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 846fed48755..fcbc89307b8 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -139,7 +139,7 @@ public class Compressor { public byte[] decompress(CompressionType compression, byte[] compressedData, int compressedDataOffset, int expectedUncompressedSize, Optional<Integer> expectedCompressedSize) { switch (compression) { - case NONE: case INCOMPRESSIBLE: // return a copy of the requested slize of the input buffer + case NONE: case INCOMPRESSIBLE: // return a copy of the requested slice of the input buffer int endPosition = expectedCompressedSize.isPresent() ? compressedDataOffset + expectedCompressedSize.get() : compressedData.length; return Arrays.copyOfRange(compressedData, compressedDataOffset, endPosition); case LZ4: diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java b/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java index 44bcec4f0eb..7fa5ecfcdee 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; -import com.google.common.util.concurrent.UncheckedTimeoutException; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java index 7e32ca25dc4..01abcc0d610 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java @@ -2,6 +2,8 @@ package com.yahoo.concurrent; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; /** * This is an implementation of {@link Timer} that is backed by an actual system timer. @@ -14,7 +16,24 @@ public enum SystemTimer implements Timer { private volatile long millis; - private SystemTimer() { + private static int detectHz() { + Logger log = Logger.getLogger(SystemTimer.class.getName()); + String hzEnv = System.getenv("VESPA_TIMER_HZ"); + int hz = 1000; + if ((hzEnv != null) && !hzEnv.isBlank()) { + try { + hz = Integer.parseInt(hzEnv); + } catch (NumberFormatException e) { + log.log(Level.WARNING, "Failed parsing VESPA_TIMER_HZ='" + hzEnv + "'", e); + } + }; + hz = Math.min(1000, Math.max(1, hz)); // Capping to valid range [1...1000]hz + log.fine("vespa-system-timer running at " + hz + "hz. VESPA_TIMER_HZ='" + hzEnv + "'"); + return hz; + } + + SystemTimer() { + int napTime = 1000 / detectHz(); millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); Thread thread = new Thread() { @@ -23,7 +42,7 @@ public enum SystemTimer implements Timer { while (true) { millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); try { - Thread.sleep(1); + Thread.sleep(napTime); } catch (InterruptedException e) { break; } @@ -31,6 +50,7 @@ public enum SystemTimer implements Timer { } }; thread.setDaemon(true); + thread.setName("vespa-system-timer"); thread.start(); } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java b/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java new file mode 100644 index 00000000000..b9fa0c6cb3c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.concurrent.TimeoutException; + +/** + * Unchecked alternative for {@link java.util.concurrent.TimeoutException}. + * + * @author bjorncs + */ +public class UncheckedTimeoutException extends RuntimeException { + + public UncheckedTimeoutException() {} + + public UncheckedTimeoutException(TimeoutException cause) { super(cause.getMessage(), cause); } + + public UncheckedTimeoutException(String message) { super(message); } + + public UncheckedTimeoutException(Throwable cause) { super(cause); } + + public UncheckedTimeoutException(String message, Throwable cause) { super(message, cause); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index da22dbdc336..1edf8e4edbe 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.net.HostName; import java.math.BigDecimal; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TypeResolver.java b/vespajlib/src/main/java/com/yahoo/tensor/TypeResolver.java index dad93734b22..3b12b6bdba1 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TypeResolver.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TypeResolver.java @@ -2,13 +2,9 @@ package com.yahoo.tensor; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.tensor.TensorType.Dimension; @@ -61,7 +57,7 @@ public class TypeResolver { static public TensorType peek(TensorType inputType, List<String> peekDimensions) { if (peekDimensions.isEmpty()) { - throw new IllegalArgumentException("peeking no dimensions makes no sense"); + throw new IllegalArgumentException("Peeking no dimensions makes no sense"); } Map<String, Dimension> map = new HashMap<>(); for (Dimension dim : inputType.dimensions()) { @@ -71,7 +67,7 @@ public class TypeResolver { if (map.containsKey(name)) { map.remove(name); } else { - throw new IllegalArgumentException("peeking non-existing dimension "+name+" in type "+inputType); + throw new IllegalArgumentException("Peeking non-existing dimension '" + name + "'"); } } if (map.isEmpty()) { @@ -83,10 +79,10 @@ public class TypeResolver { static public TensorType rename(TensorType inputType, List<String> from, List<String> to) { if (from.isEmpty()) { - throw new IllegalArgumentException("renaming no dimensions"); + throw new IllegalArgumentException("Renaming no dimensions"); } if (from.size() != to.size()) { - throw new IllegalArgumentException("bad rename, from size "+from.size()+" != to.size "+to.size()); + throw new IllegalArgumentException("Bad rename, from size "+from.size()+" != to.size "+to.size()); } Map<String,Dimension> oldDims = new HashMap<>(); for (Dimension dim : inputType.dimensions()) { @@ -100,7 +96,7 @@ public class TypeResolver { var dim = oldDims.remove(oldName); newDims.put(newName, dim.withName(newName)); } else { - logger.log(Level.WARNING, "renaming non-existing dimension "+oldName+" in type "+inputType); + logger.log(Level.WARNING, "Renaming non-existing dimension "+oldName+" in type "+inputType); // throw new IllegalArgumentException("bad rename, dimension "+oldName+" not found"); } } @@ -110,13 +106,13 @@ public class TypeResolver { if (inputType.dimensions().size() == newDims.size()) { return new TensorType(inputType.valueType(), newDims.values()); } else { - throw new IllegalArgumentException("bad rename, lost some dimenions"); + throw new IllegalArgumentException("Bad rename, lost some dimensions"); } } static public TensorType cell_cast(TensorType inputType, Value toCellType) { if (toCellType != Value.DOUBLE && inputType.dimensions().isEmpty()) { - throw new IllegalArgumentException("cannot cast "+inputType+" to valueType"+toCellType); + throw new IllegalArgumentException("Cannot cast "+inputType+" to valueType"+toCellType); } return new TensorType(toCellType, inputType.dimensions()); } @@ -192,7 +188,7 @@ public class TypeResolver { if (allOk) { return join(lhs, rhs); } else { - throw new IllegalArgumentException("types in merge() dimensions mismatch: "+lhs+" != "+rhs); + throw new IllegalArgumentException("Types in merge() dimensions mismatch: "+lhs+" != "+rhs); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java index a376536015a..dbc8396d701 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java @@ -58,7 +58,7 @@ public class VariableTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return name; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java index 16ca7104f8d..55dd8a7bc8a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java @@ -48,7 +48,7 @@ public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "argmax(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java index fcdc1233550..f1f0b9d67b0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java @@ -48,7 +48,7 @@ public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "argmin(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java index 8c6c27e171a..09f84e6747e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java @@ -107,7 +107,7 @@ public class CellCast<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "cell_cast(" + argument.toString(context) + ", " + valueType + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java index 32a4c8cd2ff..6d4b15be991 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java @@ -285,7 +285,7 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "concat(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + dimension + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java index 1544369ba2f..a0fd9272f54 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java @@ -47,6 +47,6 @@ public class ConstantTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti public Tensor evaluate(EvaluationContext<NAMETYPE> context) { return constant; } @Override - public String toString(ToStringContext context) { return constant.toString(); } + public String toString(ToStringContext<NAMETYPE> context) { return constant.toString(); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java index 2c0fa483021..92d89ec68f7 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java @@ -41,7 +41,7 @@ public class Diag<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYP } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "diag(" + dimensionNames().collect(Collectors.joining(",")) + ")" + diagFunction; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java index 97126ad88a7..46992115c23 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java @@ -46,11 +46,11 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens TensorType type() { return type; } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return type().toString() + ":" + contentToString(context); } - abstract String contentToString(ToStringContext context); + abstract String contentToString(ToStringContext<NAMETYPE> context); /** Creates a dynamic tensor function. The cell addresses must match the type. */ public static <NAMETYPE extends Name> DynamicTensor<NAMETYPE> from(TensorType type, Map<TensorAddress, ScalarFunction<NAMETYPE>> cells) { @@ -80,7 +80,7 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens } @Override - String contentToString(ToStringContext context) { + String contentToString(ToStringContext<NAMETYPE> context) { if (type().dimensions().isEmpty()) { if (cells.isEmpty()) return "{}"; return "{" + cells.values().iterator().next().toString(context) + "}"; @@ -121,7 +121,7 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens } @Override - String contentToString(ToStringContext context) { + String contentToString(ToStringContext<NAMETYPE> context) { if (type().dimensions().isEmpty()) { if (cells.isEmpty()) return "{}"; return "{" + cells.get(0).toString(context) + "}"; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java index 8fc246a7d9d..c049e5d41da 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java @@ -41,7 +41,7 @@ public class Expand<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "expand(" + argument.toString(context) + ", " + dimensionName + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index 89e981df49e..54e83fa472f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -117,9 +117,9 @@ public class Generate<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } @Override - public String toString(ToStringContext context) { return type + "(" + generatorToString(context) + ")"; } + public String toString(ToStringContext<NAMETYPE> context) { return type + "(" + generatorToString(context) + ")"; } - private String generatorToString(ToStringContext context) { + private String generatorToString(ToStringContext<NAMETYPE> context) { if (freeGenerator != null) return freeGenerator.toString(); else @@ -183,11 +183,11 @@ public class Generate<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } /** A context which adds the bindings of the generate dimension names to the given context. */ - private class GenerateToStringContext implements ToStringContext { + private class GenerateToStringContext implements ToStringContext<NAMETYPE> { - private final ToStringContext context; + private final ToStringContext<NAMETYPE> context; - public GenerateToStringContext(ToStringContext context) { + public GenerateToStringContext(ToStringContext<NAMETYPE> context) { this.context = context; } @@ -200,7 +200,7 @@ public class Generate<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } @Override - public ToStringContext parent() { return context; } + public ToStringContext<NAMETYPE> parent() { return context; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 0d4aeb5c37d..52bef482fb4 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -75,7 +75,7 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "join(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + combinator + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java index 903d0b2dcd9..f47202d1b9f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java @@ -39,7 +39,7 @@ public class L1Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "l1_normalize(" + argument.toString(context) + ", " + dimension + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java index c862aa4eaf6..8f4e2f466d4 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java @@ -41,7 +41,7 @@ public class L2Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "l2_normalize(" + argument.toString(context) + ", " + dimension + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java index 40620cb95fe..46772d8cbff 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java @@ -71,7 +71,7 @@ public class Map<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "map(" + argument.toString(context) + ", " + mapper + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java index 810e01011fe..8ac6d711c48 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java @@ -45,7 +45,7 @@ public class Matmul<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "matmul(" + argument1.toString(context) + ", " + argument2.toString(context) + ", " + dimension + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java index bd42e95a59e..adc84225a63 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java @@ -70,7 +70,7 @@ public class Merge<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "merge(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + merger + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java index 8cf1964585a..18c5db8e3a7 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java @@ -38,7 +38,7 @@ public class Random<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "random(" + dimensionNames().collect(Collectors.joining(",")) + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java index 7d5b11d6672..45b827db900 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java @@ -42,7 +42,7 @@ public class Range<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETY } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "range(" + dimensionNames().collect(Collectors.joining(",")) + ")" + rangeFunction; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java index 79209fd8f09..8841cff15e9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java @@ -86,7 +86,7 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "reduce(" + argument.toString(context) + ", " + aggregator + commaSeparated(dimensions) + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java index 9b56fefb5f0..7505355beed 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java @@ -314,7 +314,7 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "reduce_join(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + combinator + ", " + @@ -324,8 +324,8 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N private static class MultiDimensionIterator { - private long[] bounds; - private long[] iterator; + private final long[] bounds; + private final long[] iterator; private int remaining; MultiDimensionIterator(TensorType type) { @@ -364,9 +364,11 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N remaining -= 1; } + @Override public String toString() { return Arrays.toString(iterator); } + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java index 67ede7f6540..a434ecba5cc 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java @@ -128,7 +128,7 @@ public class Rename<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "rename(" + argument.toString(context) + ", " + toVectorString(fromDimensions) + ", " + toVectorString(toDimensions) + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java index 11e52aad73e..0e0dc9a9aa8 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java @@ -20,6 +20,6 @@ public interface ScalarFunction<NAMETYPE extends Name> extends Function<Evaluati /** Returns this as a tensor function, or empty if it cannot be represented as a tensor function */ default Optional<TensorFunction<NAMETYPE>> asTensorFunction() { return Optional.empty(); } - default String toString(ToStringContext context) { return toString(); } + default String toString(ToStringContext<NAMETYPE> context) { return toString(); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java index a6f71dacf30..da7581c39f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java @@ -121,14 +121,13 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY private TensorType resultType(TensorType argumentType) { List<String> peekDimensions; - - // Special case where a single indexed or mapped dimension is sliced if (subspaceAddress.size() == 1 && subspaceAddress.get(0).dimension().isEmpty()) { + // Special case where a single indexed or mapped dimension is sliced if (subspaceAddress.get(0).index().isPresent()) { peekDimensions = findDimensions(argumentType.dimensions(), TensorType.Dimension::isIndexed); if (peekDimensions.size() > 1) { throw new IllegalArgumentException(this + " slices a single indexed dimension, cannot be applied " + - "to " + argumentType + ", which has multiple"); + "to " + argumentType + ", which has multiple"); } } else { @@ -141,20 +140,28 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY else { // general slicing peekDimensions = subspaceAddress.stream().map(d -> d.dimension().get()).collect(Collectors.toList()); } - return TypeResolver.peek(argumentType, peekDimensions); + try { + return TypeResolver.peek(argumentType, peekDimensions); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException(this + " cannot slice type " + argumentType, e); + } } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { StringBuilder b = new StringBuilder(argument.toString(context)); - if (subspaceAddress.size() == 1 && subspaceAddress.get(0).dimension().isEmpty()) { + if (context.typeContext().isEmpty() + && subspaceAddress.size() == 1 && subspaceAddress.get(0).dimension().isEmpty()) { // use short forms if (subspaceAddress.get(0).index().isPresent()) b.append("[").append(subspaceAddress.get(0).index().get().toString(context)).append("]"); else b.append("{").append(subspaceAddress.get(0).label().get()).append("}"); } - else { - b.append("{").append(subspaceAddress.stream().map(i -> i.toString(context)).collect(Collectors.joining(", "))).append("}"); + else { // general form + b.append("{").append(subspaceAddress.stream() + .map(i -> i.toString(context, this)) + .collect(Collectors.joining(", "))).append("}"); } return b.toString(); } @@ -221,12 +228,22 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY @Override public String toString() { - return toString(ToStringContext.empty()); + return toString(null, null); } - public String toString(ToStringContext context) { + String toString(ToStringContext<NAMETYPE> context, Slice<NAMETYPE> owner) { StringBuilder b = new StringBuilder(); - dimension.ifPresent(d -> b.append(d).append(":")); + Optional<String> dimensionName = dimension; + if (context != null && dimensionName.isEmpty()) { // This isn't just toString(): Output canonical form or fail + TensorType type = context.typeContext().isPresent() ? owner.argument.type(context.typeContext().get()) : null; + if (type == null || type.dimensions().size() != 1) + throw new IllegalArgumentException("The tensor dimension name being sliced by " + owner + + " cannot be uniquely resolved. Use the full form " + + "slice{myDimensionName: ..."); + else + dimensionName = Optional.of(type.dimensions().get(0).name()); + } + dimensionName.ifPresent(d -> b.append(d).append(":")); if (label != null) b.append(label); else diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java index 13420a12e8f..9ea9040831b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java @@ -46,7 +46,7 @@ public class Softmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAME } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "softmax(" + argument.toString(context) + ", " + dimension + ")"; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java index 81d3692bd94..1e1d1d3b5b9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java @@ -60,7 +60,7 @@ public abstract class TensorFunction<NAMETYPE extends Name> { * * @param context a context which must be passed to all nested functions when requesting the string value */ - public abstract String toString(ToStringContext context); + public abstract String toString(ToStringContext<NAMETYPE> context); /** Returns this as a scalar function, or empty if it cannot be represented as a scalar function */ public Optional<ScalarFunction<NAMETYPE>> asScalarFunction() { return Optional.empty(); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java index 1c8da9a1dca..233779fcebe 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java @@ -1,31 +1,42 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; +import com.yahoo.tensor.evaluation.Name; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Optional; + /** * A context which is passed down to all nested functions when returning a string representation. * * @author bratseth */ -public interface ToStringContext { +public interface ToStringContext<NAMETYPE extends Name> { - static ToStringContext empty() { return new EmptyStringContext(); } + static <NAMETYPE extends Name> ToStringContext<NAMETYPE> empty() { return new EmptyStringContext<NAMETYPE>(); } /** Returns the name an identifier is bound to, or null if not bound in this context */ String getBinding(String name); /** + * Returns the context used to resolve types in this, if present. + * In some functions serialization depends on type information. + */ + default Optional<TypeContext<NAMETYPE>> typeContext() { return Optional.empty(); } + + /** * Returns the parent context of this (the context we're in scope of when this is created), * or null if this is the root. */ - ToStringContext parent(); + ToStringContext<NAMETYPE> parent(); - class EmptyStringContext implements ToStringContext { + class EmptyStringContext<NAMETYPE extends Name> implements ToStringContext<NAMETYPE> { @Override public String getBinding(String name) { return null; } @Override - public ToStringContext parent() { return null; } + public ToStringContext<NAMETYPE> parent() { return null; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java index 112a0d43796..0223ad4d588 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java @@ -44,7 +44,7 @@ public class XwPlusB<NAMETYPE extends Name> extends CompositeTensorFunction<NAME } @Override - public String toString(ToStringContext context) { + public String toString(ToStringContext<NAMETYPE> context) { return "xw_plus_b(" + x.toString(context) + ", " + w.toString(context) + ", " + b.toString(context) + ", " + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGenerator.java b/vespajlib/src/main/java/com/yahoo/text/internal/SnippetGenerator.java index 61a7be25ec8..a8263b999c7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGenerator.java +++ b/vespajlib/src/main/java/com/yahoo/text/internal/SnippetGenerator.java @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.text; +package com.yahoo.text.internal; /** + * Truncate text to a snippet suitable for logging. + * * @author hakon */ public class SnippetGenerator { diff --git a/vespajlib/src/main/java/com/yahoo/text/internal/package-info.java b/vespajlib/src/main/java/com/yahoo/text/internal/package-info.java new file mode 100644 index 00000000000..bbbf9238528 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/internal/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +@ExportPackage +package com.yahoo.text.internal; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java index b18a0f397f6..64cf9dd522c 100644 --- a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java +++ b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.time; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import java.time.Clock; import java.time.Duration; diff --git a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java index 8f089bfc8d6..7d2c780bcdb 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa; +import java.nio.file.Files; +import java.nio.file.Path; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; @@ -9,6 +11,7 @@ import java.io.FileReader; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -84,7 +87,9 @@ public class VersionTagger { String className = format == Format.SIMPLE ? "VespaVersion" : "Vtag"; String outFile = dirName + "/" + className +".java"; - FileOutputStream out = new FileOutputStream(outFile); + Path outPath = Path.of(outFile); + Path tmpPath = Path.of(outFile + ".tmp"); + var out = Files.newOutputStream(tmpPath); OutputStreamWriter writer = new OutputStreamWriter(out); System.err.println("generating: " + outFile); @@ -129,6 +134,17 @@ public class VersionTagger { } writer.write("}\n"); writer.close(); + out.close(); + if (Files.exists(outPath)) { + byte[] tmpBytes = Files.readAllBytes(tmpPath); + byte[] oldBytes = Files.readAllBytes(outPath); + if (Arrays.equals(tmpBytes, oldBytes)) { + Files.delete(tmpPath); + return; + } + } + Files.deleteIfExists(outPath); + Files.move(tmpPath, outPath); } } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java index 7f2f0deea66..604c29e7289 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import org.junit.Test; import java.time.Duration; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGeneratorTest.java b/vespajlib/src/test/java/com/yahoo/text/internal/SnippetGeneratorTest.java index a0b2df8783f..54c5a37f23c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGeneratorTest.java +++ b/vespajlib/src/test/java/com/yahoo/text/internal/SnippetGeneratorTest.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.text; +package com.yahoo.text.internal; import org.junit.Test; @@ -56,4 +56,4 @@ public class SnippetGeneratorTest { "This is a long text that should be snippeted", 50, "This is a long text that should be snippeted"); } -}
\ No newline at end of file +} diff --git a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java index bad95883df9..c296137dfb6 100644 --- a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java +++ b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.time; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.test.ManualClock; import org.junit.Test; diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index eb7e1f7d4c0..09cf354d027 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -85,6 +85,7 @@ vespa_define_module( src/tests/net/tls/policy_checking_certificate_verifier src/tests/net/tls/protocol_snooping src/tests/net/tls/transport_options + src/tests/nice src/tests/objects/nbostream src/tests/optimized src/tests/overload @@ -101,6 +102,7 @@ vespa_define_module( src/tests/require src/tests/runnable_pair src/tests/sha1 + src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr src/tests/signalhandler diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index 2b6d3ee4613..da41d75b479 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -139,22 +139,28 @@ TEST("rounding of large mmaped buffer") { EXPECT_EQUAL(MemoryAllocator::HUGEPAGE_SIZE*12ul, buf.size()); } +void verifyExtension(Alloc& buf, size_t currSZ, size_t newSZ) { + bool expectSuccess = (currSZ != newSZ); + void* oldPtr = buf.get(); + EXPECT_EQUAL(currSZ, buf.size()); + EXPECT_EQUAL(expectSuccess, buf.resize_inplace(currSZ + 1)); + EXPECT_EQUAL(oldPtr, buf.get()); + EXPECT_EQUAL(newSZ, buf.size()); +} + TEST("heap alloc can not be extended") { Alloc buf = Alloc::allocHeap(100); - void * oldPtr = buf.get(); - EXPECT_EQUAL(100ul, buf.size()); - EXPECT_FALSE(buf.resize_inplace(101)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(100ul, buf.size()); + verifyExtension(buf, 100, 100); +} + +TEST("mmap alloc cannot be extended from zero") { + Alloc buf = Alloc::allocMMap(0); + verifyExtension(buf, 0, 0); } TEST("auto alloced heap alloc can not be extended") { Alloc buf = Alloc::alloc(100); - void * oldPtr = buf.get(); - EXPECT_EQUAL(100ul, buf.size()); - EXPECT_FALSE(buf.resize_inplace(101)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(100ul, buf.size()); + verifyExtension(buf, 100, 100); } TEST("auto alloced heap alloc can not be extended, even if resize will be mmapped") { @@ -166,15 +172,6 @@ TEST("auto alloced heap alloc can not be extended, even if resize will be mmappe EXPECT_EQUAL(100ul, buf.size()); } -void verifyExtension(Alloc & buf, size_t currSZ, size_t newSZ) { - bool expectSuccess = (currSZ != newSZ); - void * oldPtr = buf.get(); - EXPECT_EQUAL(currSZ, buf.size()); - EXPECT_EQUAL(expectSuccess, buf.resize_inplace(currSZ+1)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(newSZ, buf.size()); -} - void ensureRoomForExtension(const Alloc & buf, Alloc & reserved) { // Normally mmapping starts at the top and grows down in address space. // Then there is no room to extend the last mapping. @@ -253,6 +250,11 @@ TEST("heap alloc can not be shrinked") { EXPECT_EQUAL(101ul, buf.size()); } +TEST("heap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::allocHeap(101); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("mmap alloc can be shrinked") { Alloc buf = Alloc::allocMMap(4097); void * oldPtr = buf.get(); @@ -262,6 +264,11 @@ TEST("mmap alloc can be shrinked") { EXPECT_EQUAL(4_Ki, buf.size()); } +TEST("mmap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::allocMMap(4097); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced heap alloc can not be shrinked") { Alloc buf = Alloc::alloc(101); void * oldPtr = buf.get(); @@ -271,6 +278,11 @@ TEST("auto alloced heap alloc can not be shrinked") { EXPECT_EQUAL(101ul, buf.size()); } +TEST("auto alloced heap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::alloc(101); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced mmap alloc can be shrinked") { static constexpr size_t SZ = MemoryAllocator::HUGEPAGE_SIZE; Alloc buf = Alloc::alloc(SZ + 1); @@ -281,6 +293,11 @@ TEST("auto alloced mmap alloc can be shrinked") { EXPECT_EQUAL(SZ, buf.size()); } +TEST("auto alloced mmap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::alloc(MemoryAllocator::HUGEPAGE_SIZE + 1); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced mmap alloc can not be shrinked below HUGEPAGE_SIZE/2 + 1 ") { static constexpr size_t SZ = MemoryAllocator::HUGEPAGE_SIZE; Alloc buf = Alloc::alloc(SZ + 1); diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp index 719623c9401..511c5ae11f1 100644 --- a/vespalib/src/tests/array/array_test.cpp +++ b/vespalib/src/tests/array/array_test.cpp @@ -2,7 +2,9 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/util/round_up_to_page_size.h> #include <vespa/vespalib/util/size_literals.h> #include <deque> #include <atomic> @@ -27,6 +29,11 @@ std::ostream & operator << (std::ostream & os, const Array<T> & a) } +using alloc::Alloc; +using alloc::MemoryAllocator; +using MyMemoryAllocator = vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MyMemoryAllocator::Stats; + class Clever { public: Clever() : _counter(&_global) { (*_counter)++; } @@ -324,31 +331,77 @@ TEST("test move assignment") struct UnreserveFixture { Array<int> arr; - UnreserveFixture() : arr(1025, 7, alloc::Alloc::allocMMap(0)) + UnreserveFixture() : arr(page_ints() + 1, 7, alloc::Alloc::allocMMap(0)) { - EXPECT_EQUAL(1025u, arr.size()); - EXPECT_EQUAL(2048u, arr.capacity()); + EXPECT_EQUAL(page_ints() + 1, arr.size()); + EXPECT_EQUAL(2 * page_ints(), arr.capacity()); + } + + static size_t page_ints() { + return round_up_to_page_size(1) / sizeof(int); } }; TEST_F("require that try_unreserve() fails if wanted capacity >= current capacity", UnreserveFixture) { - EXPECT_FALSE(f.arr.try_unreserve(2048)); + EXPECT_FALSE(f.arr.try_unreserve(2 * UnreserveFixture::page_ints())); } TEST_F("require that try_unreserve() fails if wanted capacity < current size", UnreserveFixture) { - EXPECT_FALSE(f.arr.try_unreserve(1024)); + EXPECT_FALSE(f.arr.try_unreserve(UnreserveFixture::page_ints())); } TEST_F("require that try_unreserve() succeedes if mmap can be shrinked", UnreserveFixture) { int *oldPtr = &f.arr[0]; f.arr.resize(512); - EXPECT_TRUE(f.arr.try_unreserve(1023)); - EXPECT_EQUAL(1_Ki, f.arr.capacity()); + EXPECT_TRUE(f.arr.try_unreserve(UnreserveFixture::page_ints() - 1)); + EXPECT_EQUAL(UnreserveFixture::page_ints(), f.arr.capacity()); int *newPtr = &f.arr[0]; EXPECT_EQUAL(oldPtr, newPtr); } +struct Fixture { + AllocStats stats; + std::unique_ptr<MemoryAllocator> allocator; + Alloc initial_alloc; + Array<int> arr; + + Fixture(); + ~Fixture(); +}; + +Fixture::Fixture() + : stats(), + allocator(std::make_unique<MyMemoryAllocator>(stats)), + initial_alloc(Alloc::alloc_with_allocator(allocator.get())), + arr(initial_alloc) +{ +} + +Fixture::~Fixture() = default; + +TEST_F("require that memory allocator can be set", Fixture) +{ + f.arr.resize(1); + EXPECT_EQUAL(AllocStats(1, 0), f.stats); +} + +TEST_F("require that memory allocator is preserved across reset", Fixture) +{ + f.arr.resize(1); + f.arr.reset(); + f.arr.resize(1); + EXPECT_EQUAL(AllocStats(2, 1), f.stats); +} + +TEST_F("require that created array uses same memory allocator", Fixture) +{ + auto arr2 = f.arr.create(); + EXPECT_EQUAL(AllocStats(0, 0), f.stats); + arr2.resize(1); + EXPECT_EQUAL(AllocStats(1, 0), f.stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp index 98a7bd780a7..83f49d6c73b 100644 --- a/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp +++ b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp @@ -2,11 +2,16 @@ #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/test/thread_meets.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <sys/resource.h> #include <thread> using namespace vespalib; +using namespace vespalib::test; +using vespalib::make_string_short::fmt; bool verbose = false; size_t loop_cnt = 10; @@ -16,6 +21,16 @@ using Sampler = vespalib::cpu_usage::ThreadSampler; //----------------------------------------------------------------------------- +class EndTime { +private: + steady_time _end_time; +public: + EndTime(duration test_time) : _end_time(steady_clock::now() + test_time) {} + bool operator()() const { return (steady_clock::now() >= _end_time); } +}; + +//----------------------------------------------------------------------------- + void be_busy(duration d) { if (d > 0ms) { volatile int tmp = 123; @@ -45,22 +60,30 @@ void verify_sampling(size_t thread_id, size_t num_threads, std::vector<Sampler*> TEST_BARRIER(); // #1 auto t0 = steady_clock::now(); std::vector<duration> pre_usage = sample(samplers); + auto pre_total = cpu_usage::total_cpu_usage(); TEST_BARRIER(); // #2 TEST_BARRIER(); // #3 auto t1 = steady_clock::now(); std::vector<duration> post_usage = sample(samplers); + auto post_total = cpu_usage::total_cpu_usage(); TEST_BARRIER(); // #4 double wall = to_s(t1 - t0); - std::vector<double> load(4, 0.0); + std::vector<double> util(4, 0.0); for (size_t i = 0; i < 4; ++i) { - load[i] = to_s(post_usage[i] - pre_usage[i]) / wall; + util[i] = to_s(post_usage[i] - pre_usage[i]) / wall; } - EXPECT_GREATER(load[3], load[0]); - fprintf(stderr, "loads: { %.2f, %.2f, %.2f, %.2f }\n", load[0], load[1], load[2], load[3]); + double total_util = to_s(post_total - pre_total) / wall; + EXPECT_GREATER(util[3], util[0]); + // NB: cannot expect total_util to be greater than util[3] + // here due to mock utils being 'as expected' while valgrind + // will cut all utils in about half. + EXPECT_GREATER(total_util, util[0]); + fprintf(stderr, "utils: { %.3f, %.3f, %.3f, %.3f }\n", util[0], util[1], util[2], util[3]); + fprintf(stderr, "total util: %.3f\n", total_util); } else { int idx = (thread_id - 1); - double target_load = double(thread_id - 1) / (num_threads - 2); - auto sampler = cpu_usage::create_thread_sampler(force_mock, target_load); + double target_util = double(thread_id - 1) / (num_threads - 2); + auto sampler = cpu_usage::create_thread_sampler(force_mock, target_util); samplers[idx] = sampler.get(); TEST_BARRIER(); // #1 TEST_BARRIER(); // #2 @@ -74,7 +97,7 @@ void verify_sampling(size_t thread_id, size_t num_threads, std::vector<Sampler*> //----------------------------------------------------------------------------- -TEST_MT_F("require that dummy thread-based CPU usage sampling with known expected load works", 5, std::vector<Sampler*>(4, nullptr)) { +TEST_MT_F("require that dummy thread-based CPU usage sampling with known expected util works", 5, std::vector<Sampler*>(4, nullptr)) { TEST_DO(verify_sampling(thread_id, num_threads, f1, true)); } @@ -89,6 +112,353 @@ TEST("measure thread CPU clock overhead") { fprintf(stderr, "approx overhead per sample (thread CPU clock): %f us\n", min_time_us); } +TEST("measure total cpu usage overhead") { + duration d; + double min_time_us = BenchmarkTimer::benchmark([&d]() noexcept { d = cpu_usage::total_cpu_usage(); }, budget) * 1000000.0; + fprintf(stderr, "approx overhead per RUsage sample: %f us\n", min_time_us); +} + +//----------------------------------------------------------------------------- + +void verify_category(CpuUsage::Category cat, size_t idx, const vespalib::string &name) { + switch (cat) { // make sure we known all categories + case CpuUsage::Category::SETUP: + case CpuUsage::Category::READ: + case CpuUsage::Category::WRITE: + case CpuUsage::Category::COMPACT: + case CpuUsage::Category::OTHER: + EXPECT_EQUAL(CpuUsage::index_of(cat), idx); + EXPECT_EQUAL(CpuUsage::name_of(cat), name); + } +} + +TEST("require that CPU categories are as expected") { + TEST_DO(verify_category(CpuUsage::Category::SETUP, 0u, "setup")); + TEST_DO(verify_category(CpuUsage::Category::READ, 1u, "read")); + TEST_DO(verify_category(CpuUsage::Category::WRITE, 2u, "write")); + TEST_DO(verify_category(CpuUsage::Category::COMPACT, 3u, "compact")); + TEST_DO(verify_category(CpuUsage::Category::OTHER, 4u, "other")); + EXPECT_EQUAL(CpuUsage::num_categories, 5u); +} + +TEST("require that empty sample is zero") { + CpuUsage::Sample sample; + EXPECT_EQUAL(sample.size(), CpuUsage::num_categories); + for (uint32_t i = 0; i < sample.size(); ++i) { + EXPECT_EQUAL(sample[i].count(), 0); + } +} + +TEST("require that cpu samples can be manipulated and inspected") { + CpuUsage::Sample a; + CpuUsage::Sample b; + const CpuUsage::Sample &c = a; + a[CpuUsage::Category::SETUP] = 1ms; + a[CpuUsage::Category::READ] = 2ms; + a[CpuUsage::Category::WRITE] = 3ms; + a[CpuUsage::Category::COMPACT] = 4ms; + a[CpuUsage::Category::OTHER] = 5ms; + for (uint32_t i = 0; i < b.size(); ++i) { + b[i] = 10ms * (i + 1); + } + a.merge(b); + for (uint32_t i = 0; i < c.size(); ++i) { + EXPECT_EQUAL(c[i], 11ms * (i + 1)); + } + EXPECT_EQUAL(c[CpuUsage::Category::SETUP], 11ms); + EXPECT_EQUAL(c[CpuUsage::Category::READ], 22ms); + EXPECT_EQUAL(c[CpuUsage::Category::WRITE], 33ms); + EXPECT_EQUAL(c[CpuUsage::Category::COMPACT], 44ms); + EXPECT_EQUAL(c[CpuUsage::Category::OTHER], 55ms); +} + +//----------------------------------------------------------------------------- + +struct CpuUsage::Test { + struct BlockingTracker : ThreadTracker { + std::atomic<size_t> called; + ThreadMeets::Nop sync_entry; + ThreadMeets::Swap<Sample> swap_sample; + BlockingTracker() + : called(0), sync_entry(2), swap_sample() {} + Sample sample() noexcept override { + if (called++) { + return Sample(); + } + sync_entry(); + return swap_sample(Sample()); + } + }; + struct SimpleTracker : ThreadTracker { + Sample my_sample; + std::atomic<size_t> called; + SimpleTracker(Sample sample) noexcept + : my_sample(sample), called(0) {} + Sample sample() noexcept override { + ++called; + return my_sample; + } + }; + struct Fixture { + CpuUsage my_usage; + std::shared_ptr<BlockingTracker> blocking; + std::vector<std::shared_ptr<SimpleTracker>> simple_list; + Fixture() : my_usage() {} + void add_blocking() { + ASSERT_TRUE(!blocking); + blocking = std::make_unique<BlockingTracker>(); + my_usage.add_thread(blocking); + } + void add_simple(Sample sample) { + auto simple = std::make_shared<SimpleTracker>(sample); + simple_list.push_back(simple); + my_usage.add_thread(simple); + } + void add_remove_simple(Sample sample) { + auto simple = std::make_shared<SimpleTracker>(sample); + my_usage.add_thread(simple); + my_usage.remove_thread(simple); + } + size_t count_threads() { + Guard guard(my_usage._lock); + return my_usage._threads.size(); + } + bool is_sampling() { + Guard guard(my_usage._lock); + return my_usage._sampling; + } + size_t count_conflicts() { + Guard guard(my_usage._lock); + if (!my_usage._conflict) { + return 0; + } + return my_usage._conflict->waiters; + } + size_t count_simple_samples() { + size_t result = 0; + for (const auto &simple: simple_list) { + result += simple->called; + } + return result; + } + TimedSample sample() { return my_usage.sample_or_wait(); } + ~Fixture() { + if (blocking) { + my_usage.remove_thread(std::move(blocking)); + } + for (auto &simple: simple_list) { + my_usage.remove_thread(std::move(simple)); + } + ASSERT_EQUAL(count_threads(), 0u); + } + }; + struct TrackerImpl { + ThreadTrackerImpl impl; + TrackerImpl(cpu_usage::ThreadSampler::UP sampler) + : impl(std::move(sampler)) {} + CpuUsage::Sample sample() { return impl.sample(); } + CpuUsage::Category set_category(CpuUsage::Category cat) { return impl.set_category(cat); } + }; +}; + +TEST_F("require that CpuUsage sample calls sample on thread trackers", CpuUsage::Test::Fixture()) { + CpuUsage::Sample sample; + sample[CpuUsage::Category::READ] = 10ms; + f1.add_simple(sample); + f1.add_simple(sample); + f1.add_simple(sample); + EXPECT_EQUAL(f1.count_threads(), 3u); + auto result = f1.sample(); + EXPECT_EQUAL(result.second[CpuUsage::Category::READ], duration(30ms)); + EXPECT_EQUAL(f1.count_simple_samples(), 3u); + result = f1.sample(); + EXPECT_EQUAL(result.second[CpuUsage::Category::READ], duration(60ms)); + EXPECT_EQUAL(f1.count_simple_samples(), 6u); +} + +TEST_F("require that threads added and removed between CpuUsage sample calls are tracked", CpuUsage::Test::Fixture()) { + CpuUsage::Sample sample; + sample[CpuUsage::Category::READ] = 10ms; + auto result = f1.sample(); + EXPECT_EQUAL(result.second[CpuUsage::Category::READ], duration(0ms)); + f1.add_remove_simple(sample); + f1.add_remove_simple(sample); + f1.add_remove_simple(sample); + EXPECT_EQUAL(f1.count_threads(), 0u); + result = f1.sample(); + EXPECT_EQUAL(result.second[CpuUsage::Category::READ], duration(30ms)); + result = f1.sample(); + EXPECT_EQUAL(result.second[CpuUsage::Category::READ], duration(30ms)); +} + +TEST_MT_FF("require that sample conflicts are resolved correctly", 5, CpuUsage::Test::Fixture(), std::vector<CpuUsage::TimedSample>(num_threads - 1)) { + if (thread_id == 0) { + CpuUsage::Sample s1; + s1[CpuUsage::Category::SETUP] = 10ms; + CpuUsage::Sample s2; + s2[CpuUsage::Category::READ] = 20ms; + CpuUsage::Sample s3; + s3[CpuUsage::Category::WRITE] = 30ms; + CpuUsage::Sample s4; + s4[CpuUsage::Category::COMPACT] = 40ms; + f1.add_blocking(); + f1.add_simple(s1); // should be sampled + EXPECT_TRUE(!f1.is_sampling()); + EXPECT_EQUAL(f1.count_conflicts(), 0u); + TEST_BARRIER(); // #1 + f1.blocking->sync_entry(); + EXPECT_TRUE(f1.is_sampling()); + while (f1.count_conflicts() < (num_threads - 2)) { + // wait for appropriate number of conflicts + std::this_thread::sleep_for(1ms); + } + f1.add_simple(s2); // should NOT be sampled (pending add) + f1.add_remove_simple(s3); // should be sampled (pending remove); + EXPECT_EQUAL(f1.count_threads(), 2u); + EXPECT_TRUE(f1.is_sampling()); + EXPECT_EQUAL(f1.count_conflicts(), (num_threads - 2)); + f1.blocking->swap_sample(s4); + TEST_BARRIER(); // #2 + EXPECT_TRUE(!f1.is_sampling()); + EXPECT_EQUAL(f1.count_conflicts(), 0u); + EXPECT_EQUAL(f1.count_threads(), 3u); + EXPECT_EQUAL(f2[0].second[CpuUsage::Category::SETUP], duration(10ms)); + EXPECT_EQUAL(f2[0].second[CpuUsage::Category::READ], duration(0ms)); + EXPECT_EQUAL(f2[0].second[CpuUsage::Category::WRITE], duration(30ms)); + EXPECT_EQUAL(f2[0].second[CpuUsage::Category::COMPACT], duration(40ms)); + for (size_t i = 1; i < (num_threads - 1); ++i) { + EXPECT_EQUAL(f2[i].first, f2[0].first); + EXPECT_EQUAL(f2[i].second[CpuUsage::Category::SETUP], f2[0].second[CpuUsage::Category::SETUP]); + EXPECT_EQUAL(f2[i].second[CpuUsage::Category::READ], f2[0].second[CpuUsage::Category::READ]); + EXPECT_EQUAL(f2[i].second[CpuUsage::Category::WRITE], f2[0].second[CpuUsage::Category::WRITE]); + EXPECT_EQUAL(f2[i].second[CpuUsage::Category::COMPACT], f2[0].second[CpuUsage::Category::COMPACT]); + } + } else { + TEST_BARRIER(); // #1 + f2[thread_id - 1] = f1.sample(); + TEST_BARRIER(); // #2 + } +} + +//----------------------------------------------------------------------------- + +struct DummySampler : public cpu_usage::ThreadSampler { + duration &ref; + DummySampler(duration &ref_in) : ref(ref_in) {} + duration sample() const noexcept override { return ref; } +}; + +TEST("require that thread tracker implementation can track cpu use") { + duration t = duration::zero(); + CpuUsage::Test::TrackerImpl tracker(std::make_unique<DummySampler>(t)); + t += 10ms; + tracker.set_category(CpuUsage::Category::SETUP); + t += 15ms; + tracker.set_category(CpuUsage::Category::READ); + t += 10ms; + auto sample = tracker.sample(); + EXPECT_EQUAL(sample[CpuUsage::Category::SETUP], duration(15ms)); + EXPECT_EQUAL(sample[CpuUsage::Category::READ], duration(10ms)); + EXPECT_EQUAL(sample[CpuUsage::Category::WRITE], duration(0ms)); + t += 15ms; + tracker.set_category(CpuUsage::Category::WRITE); + t += 10ms; + sample = tracker.sample(); + EXPECT_EQUAL(sample[CpuUsage::Category::SETUP], duration(0ms)); + EXPECT_EQUAL(sample[CpuUsage::Category::READ], duration(15ms)); + EXPECT_EQUAL(sample[CpuUsage::Category::WRITE], duration(10ms)); +} + +TEST("require that thread tracker implementation reports previous CPU category") { + duration t = duration::zero(); + CpuUsage::Test::TrackerImpl tracker(std::make_unique<DummySampler>(t)); + EXPECT_EQUAL(CpuUsage::index_of(CpuUsage::Category::OTHER), + CpuUsage::index_of(tracker.set_category(CpuUsage::Category::SETUP))); + EXPECT_EQUAL(CpuUsage::index_of(CpuUsage::Category::SETUP), + CpuUsage::index_of(tracker.set_category(CpuUsage::Category::READ))); + EXPECT_EQUAL(CpuUsage::index_of(CpuUsage::Category::READ), + CpuUsage::index_of(tracker.set_category(CpuUsage::Category::READ))); +} + +TEST("require that thread tracker implementation does not track OTHER cpu use") { + duration t = duration::zero(); + CpuUsage::Test::TrackerImpl tracker(std::make_unique<DummySampler>(t)); + t += 10ms; + tracker.set_category(CpuUsage::Category::OTHER); + t += 15ms; + tracker.set_category(CpuUsage::Category::READ); + tracker.set_category(CpuUsage::Category::OTHER); + t += 15ms; + auto sample = tracker.sample(); + EXPECT_EQUAL(sample[CpuUsage::Category::READ], duration(0ms)); + EXPECT_EQUAL(sample[CpuUsage::Category::OTHER], duration(0ms)); +} + +//----------------------------------------------------------------------------- + +void do_sample_cpu_usage(const EndTime &end_time) { + auto my_usage = CpuUsage::use(CpuUsage::Category::SETUP); + CpuUtil cpu(8ms); + while (!end_time()) { + std::this_thread::sleep_for(verbose ? 1s : 10ms); + auto util = cpu.get_util(); + vespalib::string body; + for (size_t i = 0; i < util.size(); ++i) { + if (!body.empty()) { + body.append(", "); + } + body.append(fmt("%s: %.3f", CpuUsage::name_of(CpuUsage::Category(i)).c_str(), util[i])); + } + fprintf(stderr, "CPU: %s\n", body.c_str()); + } +} + +void do_full_work(CpuUsage::Category cat, const EndTime &end_time) { + auto my_usage = CpuUsage::use(cat); + while (!end_time()) { + be_busy(4ms); + } +} + +void do_some_work(CpuUsage::Category cat, const EndTime &end_time) { + auto my_usage = CpuUsage::use(cat); + while (!end_time()) { + be_busy(4ms); + std::this_thread::sleep_for(4ms); + } +} + +void do_nested_work(CpuUsage::Category cat1, CpuUsage::Category cat2, const EndTime &end_time) { + auto my_usage1 = CpuUsage::use(cat1); + while (!end_time()) { + be_busy(4ms); + auto my_usage2 = CpuUsage::use(cat2); + be_busy(4ms); + } +} + +void do_external_work(CpuUsage::Category cat, const EndTime &end_time) { + auto my_usage1 = CpuUsage::use(CpuUsage::Category::SETUP); + while (!end_time()) { + std::thread thread([cat](){ + auto my_usage2 = CpuUsage::use(cat); + be_busy(4ms); + }); + thread.join(); + } +} + +TEST_MT_F("use top-level API to sample CPU usage", 5, EndTime(verbose ? 10s : 100ms)) { + switch (thread_id) { + case 0: return do_sample_cpu_usage(f1); + case 1: return do_full_work(CpuUsage::Category::WRITE, f1); + case 2: return do_some_work(CpuUsage::Category::READ, f1); + case 3: return do_nested_work(CpuUsage::Category::OTHER, CpuUsage::Category::READ, f1); + case 4: return do_external_work(CpuUsage::Category::COMPACT, f1); + default: TEST_FATAL("missing thread id case"); + } +} + //----------------------------------------------------------------------------- int main(int argc, char **argv) { diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index c58e357a9a1..2672ed70f6d 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/memory_allocator.h> #include <vespa/vespalib/util/size_literals.h> @@ -19,6 +20,9 @@ using vespalib::ArrayRef; using generation_t = vespalib::GenerationHandler::generation_t; using MemStats = vespalib::datastore::test::MemStats; using BufferStats = vespalib::datastore::test::BufferStats; +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; namespace { @@ -40,18 +44,20 @@ struct Fixture using value_type = EntryT; using ReferenceStore = vespalib::hash_map<EntryRef, EntryVector>; + AllocStats stats; ArrayStoreType store; ReferenceStore refStore; generation_t generation; Fixture(uint32_t maxSmallArraySize, bool enable_free_lists = true) : store(ArrayStoreConfig(maxSmallArraySize, ArrayStoreConfig::AllocSpec(16, RefT::offsetSize(), 8_Ki, - ALLOC_GROW_FACTOR)).enable_free_lists(enable_free_lists)), + ALLOC_GROW_FACTOR)).enable_free_lists(enable_free_lists), + std::make_unique<MemoryAllocatorObserver>(stats)), refStore(), generation(1) {} Fixture(const ArrayStoreConfig &storeCfg) - : store(storeCfg), + : store(storeCfg, std::make_unique<MemoryAllocatorObserver>(stats)), refStore(), generation(1) {} @@ -162,10 +168,10 @@ TEST("require that we test with trivial and non-trivial types") TEST_F("control static sizes", NumberFixture(3)) { #ifdef _LIBCPP_VERSION - EXPECT_EQUAL(424u, sizeof(f.store)); + EXPECT_EQUAL(440u, sizeof(f.store)); EXPECT_EQUAL(296u, sizeof(NumberFixture::ArrayStoreType::DataStoreType)); #else - EXPECT_EQUAL(456u, sizeof(f.store)); + EXPECT_EQUAL(472u, sizeof(f.store)); EXPECT_EQUAL(328u, sizeof(NumberFixture::ArrayStoreType::DataStoreType)); #endif EXPECT_EQUAL(96u, sizeof(NumberFixture::ArrayStoreType::SmallArrayType)); @@ -447,4 +453,9 @@ TEST_F("require that offset in EntryRefT is within bounds when allocating memory f.assertStoreContent(); } +TEST_F("require that provided memory allocator is used", NumberFixture(3)) +{ + EXPECT_EQUAL(AllocStats(4, 0), f.stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/datastore/datastore/.gitignore b/vespalib/src/tests/datastore/datastore/.gitignore new file mode 100644 index 00000000000..d033739fd78 --- /dev/null +++ b/vespalib/src/tests/datastore/datastore/.gitignore @@ -0,0 +1 @@ +/core.* diff --git a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp index 417b92af9ac..599cb209e6c 100644 --- a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp @@ -68,7 +68,7 @@ struct DataStoreFixedSizeHashTest : public ::testing::Test DataStoreFixedSizeHashTest::DataStoreFixedSizeHashTest() : _generation_handler(), _generation_holder(), - _allocator(), + _allocator({}), _store(_allocator.get_data_store()), _comp(std::make_unique<MyCompare>(_store)), _hash_map(), diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp index 796e19a97d1..13f9ae251b6 100644 --- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp @@ -132,7 +132,7 @@ struct DataStoreShardedHashTest : public ::testing::Test DataStoreShardedHashTest::DataStoreShardedHashTest() : _generationHandler(), - _allocator(), + _allocator({}), _store(_allocator.get_data_store()), _hash_map(std::make_unique<MyCompare>(_store)), _writer(1, 128_Ki), diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 917c91f2dff..7f279689985 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/datastore/buffer_stats.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/traits.h> #include <vector> @@ -21,6 +22,9 @@ using namespace vespalib::datastore; using vespalib::ArrayRef; using generation_t = vespalib::GenerationHandler::generation_t; using vespalib::datastore::test::BufferStats; +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; template <typename UniqueStoreT> struct TestBaseValues { @@ -39,6 +43,7 @@ struct TestBase : public ::testing::Test { using ReferenceStoreValueType = std::conditional_t<std::is_same_v<ValueType, const char *>, std::string, ValueType>; using ReferenceStore = std::map<EntryRef, std::pair<ReferenceStoreValueType,uint32_t>>; + AllocStats stats; UniqueStoreType store; ReferenceStore refStore; generation_t generation; @@ -148,7 +153,8 @@ struct TestBase : public ::testing::Test { template <typename UniqueStoreTypeAndDictionaryType> TestBase<UniqueStoreTypeAndDictionaryType>::TestBase() - : store(), + : stats(), + store(std::make_unique<MemoryAllocatorObserver>(stats)), refStore(), generation(1) { @@ -424,6 +430,15 @@ TYPED_TEST(TestBase, store_can_be_enumerated) EXPECT_EQ(2u, enumValue2); } +TYPED_TEST(TestBase, provided_memory_allocator_is_used) +{ + if constexpr (std::is_same_v<const char *, typename TestFixture::ValueType>) { + EXPECT_EQ(AllocStats(18, 0), this->stats); + } else { + EXPECT_EQ(AllocStats(1, 0), this->stats); + } +} + #pragma GCC diagnostic pop TEST_F(DoubleTest, nan_is_handled) diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp index 158b85f6bf5..21330c22166 100644 --- a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/datastore/buffer_stats.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/traits.h> #include <vector> @@ -11,6 +12,10 @@ using namespace vespalib::datastore; using vespalib::MemoryUsage; using generation_t = vespalib::GenerationHandler::generation_t; using BufferStats = vespalib::datastore::test::BufferStats; +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MemoryAllocatorObserver::Stats; + namespace { @@ -24,10 +29,12 @@ template <typename RefT = EntryRefT<22>> struct TestBase : public ::testing::Test { using EntryRefType = RefT; + AllocStats stats; UniqueStoreStringAllocator<EntryRefType> allocator; generation_t generation; TestBase() - : allocator(), + : stats(), + allocator(std::make_unique<MemoryAllocatorObserver>(stats)), generation(1) {} void assert_add(const char *input) { @@ -170,6 +177,11 @@ TEST_F(StringTest, free_list_is_never_used_for_move) assert_buffer_state(ref2, BufferStats().used(4).hold(0).dead(2).extra_used(2002)); } +TEST_F(StringTest, provided_memory_allocator_is_used) +{ + EXPECT_EQ(AllocStats(18, 0), stats); +} + TEST_F(SmallOffsetStringTest, new_underlying_buffer_is_allocated_when_current_is_full) { uint32_t first_buffer_id = get_buffer_id(add(small.c_str())); @@ -184,6 +196,7 @@ TEST_F(SmallOffsetStringTest, new_underlying_buffer_is_allocated_when_current_is uint32_t buffer_id = get_buffer_id(add(small.c_str())); EXPECT_EQ(second_buffer_id, buffer_id); } + EXPECT_LT(18, stats.alloc_cnt); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/nice/CMakeLists.txt b/vespalib/src/tests/nice/CMakeLists.txt new file mode 100644 index 00000000000..a97ec52c22c --- /dev/null +++ b/vespalib/src/tests/nice/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_nice_test_app TEST + SOURCES + nice_test.cpp + DEPENDS + vespalib +) +if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + vespa_add_test(NAME vespalib_nice_test_app COMMAND vespalib_nice_test_app) +endif() diff --git a/vespalib/src/tests/nice/nice_test.cpp b/vespalib/src/tests/nice/nice_test.cpp new file mode 100644 index 00000000000..ec1d25be3bd --- /dev/null +++ b/vespalib/src/tests/nice/nice_test.cpp @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/nice.h> +#include <vespa/vespalib/test/thread_meets.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <unistd.h> +#include <functional> +#include <thread> + +using vespalib::Runnable; +using vespalib::be_nice; +using vespalib::test::ThreadMeets; + +double how_nice(int now, int target) { + int max = 19; + int wanted_param = (target - now); + int num_zones = ((max + 1) - now); + // make sure we are in the middle of the wanted nice zone + double result = (0.5 + wanted_param) / num_zones; + fprintf(stderr, " ... using how_nice=%g to get from %d to %d in nice value\n", result, now, target); + return result; +} + +struct RunFun : Runnable { + std::function<void()> my_fun; + RunFun(std::function<void()> fun_in) : my_fun(fun_in) {} + void run() override { my_fun(); } +}; + +int my_init_fun(Runnable &target) { + target.run(); + return 1; +} + +std::thread run_with_init(std::function<void()> my_fun, Runnable::init_fun_t init_fun = my_init_fun) { + return std::thread([init_fun, my_fun] + { + RunFun run_fun(my_fun); + init_fun(run_fun); + }); +} + +TEST("require that initial nice value is 0") { + EXPECT_EQUAL(nice(0), 0); +} + +TEST("require that nice value is tracked per thread") { + ThreadMeets::Nop barrier(5); + std::vector<std::thread> threads; + for (int i = 0; i < 5; ++i) { + threads.push_back(run_with_init([my_barrier = &barrier, i] + { + nice(i); + (*my_barrier)(); + EXPECT_EQUAL(nice(0), i); + })); + } + for (auto &thread: threads) { + thread.join(); + } +} + +void verify_max_nice_value() { + int now = nice(0); + now = nice(19 - now); + EXPECT_EQUAL(now, 19); + now = nice(1); + EXPECT_EQUAL(now, 19); +} + +TEST("require that max nice value is 19") { + auto thread = run_with_init([]{ verify_max_nice_value(); }); + thread.join(); +} + +TEST("require that nice value can be set with init function") { + for (int i = 0; i <= 19; ++i) { + auto thread = run_with_init([i]() + { + EXPECT_EQUAL(nice(0), i); + }, be_nice(my_init_fun, how_nice(0, i))); + thread.join(); + } +} + +TEST("require that niceness can be nested and will act on a limited nice value range") { + auto thread1 = run_with_init([]{ EXPECT_EQUAL(nice(0), 7); }, + be_nice(be_nice(my_init_fun, how_nice(3, 7)), how_nice(0, 3))); + auto thread2 = run_with_init([]{ EXPECT_EQUAL(nice(0), 15); }, + be_nice(be_nice(my_init_fun, how_nice(10, 15)), how_nice(0, 10))); + auto thread3 = run_with_init([]{ EXPECT_EQUAL(nice(0), 19); }, + be_nice(be_nice(my_init_fun, how_nice(10, 19)), how_nice(0, 10))); + thread1.join(); + thread2.join(); + thread3.join(); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/shared_operation_throttler/CMakeLists.txt b/vespalib/src/tests/shared_operation_throttler/CMakeLists.txt new file mode 100644 index 00000000000..6e977cdb59f --- /dev/null +++ b/vespalib/src/tests/shared_operation_throttler/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_shared_operation_throttler_test_app TEST + SOURCES + shared_operation_throttler_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_shared_operation_throttler_test_app COMMAND vespalib_shared_operation_throttler_test_app) diff --git a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp new file mode 100644 index 00000000000..d9b6ae7f908 --- /dev/null +++ b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp @@ -0,0 +1,212 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/util/shared_operation_throttler.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/barrier.h> +#include <thread> + +namespace vespalib { + +using ThrottleToken = SharedOperationThrottler::Token; + +struct DynamicThrottleFixture { + std::unique_ptr<SharedOperationThrottler> _throttler; + + DynamicThrottleFixture() { + SharedOperationThrottler::DynamicThrottleParams params; + params.window_size_increment = 1; + params.min_window_size = 1; + _throttler = SharedOperationThrottler::make_dynamic_throttler(params); + } +}; + +TEST("unlimited throttler does not throttle") { + // We technically can't test that the unlimited throttler _never_ throttles, but at + // least check that it doesn't throttle _twice_, and then induce from this ;) + auto throttler = SharedOperationThrottler::make_unlimited_throttler(); + auto token1 = throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); + auto token2 = throttler->blocking_acquire_one(); + EXPECT_TRUE(token2.valid()); + // Window size should be zero (i.e. unlimited) for unlimited throttler + EXPECT_EQUAL(throttler->current_window_size(), 0u); +} + +TEST_F("dynamic throttler respects initial window size", DynamicThrottleFixture()) { + auto token1 = f1._throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); + auto token2 = f1._throttler->try_acquire_one(); + EXPECT_FALSE(token2.valid()); + + EXPECT_EQUAL(f1._throttler->current_window_size(), 1u); +} + +TEST_F("blocking acquire returns immediately if slot available", DynamicThrottleFixture()) { + auto token = f1._throttler->blocking_acquire_one(); + EXPECT_TRUE(token.valid()); + token.reset(); + token = f1._throttler->blocking_acquire_one(600s); // Should never block. + EXPECT_TRUE(token.valid()); +} + +TEST_F("blocking call woken up if throttle slot available", DynamicThrottleFixture()) { + vespalib::Barrier barrier(2); + std::thread t([&] { + auto token = f1._throttler->try_acquire_one(); + assert(token.valid()); + barrier.await(); + while (f1._throttler->waiting_threads() != 1) { + std::this_thread::sleep_for(100us); + } + // Implicit token release at thread scope exit + }); + barrier.await(); + auto token = f1._throttler->blocking_acquire_one(); + EXPECT_TRUE(token.valid()); + t.join(); +} + +TEST_F("time-bounded blocking acquire waits for timeout", DynamicThrottleFixture()) { + auto window_filling_token = f1._throttler->try_acquire_one(); + auto before = std::chrono::steady_clock::now(); + // Will block for at least 1ms. Since no window slot will be available by that time, + // an invalid token should be returned. + auto token = f1._throttler->blocking_acquire_one(1ms); + auto after = std::chrono::steady_clock::now(); + EXPECT_TRUE((after - before) >= 1ms); + EXPECT_FALSE(token.valid()); +} + +TEST("default constructed token is invalid") { + ThrottleToken token; + EXPECT_FALSE(token.valid()); + token.reset(); // no-op + EXPECT_FALSE(token.valid()); +} + +TEST_F("token destruction frees up throttle window slot", DynamicThrottleFixture()) { + { + auto token = f1._throttler->try_acquire_one(); + EXPECT_TRUE(token.valid()); + } + auto token = f1._throttler->try_acquire_one(); + EXPECT_TRUE(token.valid()); +} + +TEST_F("token can be moved and reset", DynamicThrottleFixture()) { + auto token1 = f1._throttler->try_acquire_one(); + auto token2 = std::move(token1); // move ctor + EXPECT_TRUE(token2.valid()); + EXPECT_FALSE(token1.valid()); + ThrottleToken token3; + token3 = std::move(token2); // move assignment op + EXPECT_TRUE(token3.valid()); + EXPECT_FALSE(token2.valid()); + + // Trying to fetch new token should not succeed due to active token and win size of 1 + token1 = f1._throttler->try_acquire_one(); + EXPECT_FALSE(token1.valid()); + // Resetting the token should free up the slot in the window + token3.reset(); + token1 = f1._throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); +} + +// Note on test semantics: these tests are adapted from a subset of the MessageBus +// throttling tests. Some tests have been simplified due to no longer having access +// to the low-level DynamicThrottlePolicy API. + +struct WindowFixture { + uint64_t _milli_time; + std::unique_ptr<SharedOperationThrottler> _throttler; + + WindowFixture(uint32_t window_size_increment = 5, + uint32_t min_window_size = 20, + uint32_t max_window_size = INT_MAX) + : _milli_time(0), + _throttler() + { + SharedOperationThrottler::DynamicThrottleParams params; + params.resize_rate = 1; + params.window_size_increment = window_size_increment; + params.min_window_size = min_window_size; + params.max_window_size = max_window_size; + params.window_size_decrement_factor = 2; + params.window_size_backoff = 0.9; + _throttler = SharedOperationThrottler::make_dynamic_throttler(params, [&]() noexcept { + return steady_time(std::chrono::milliseconds(_milli_time)); + }); + } + + std::vector<SharedOperationThrottler::Token> fill_entire_throttle_window() { + std::vector<SharedOperationThrottler::Token> tokens; + while (true) { + auto token = _throttler->try_acquire_one(); + if (!token.valid()) { + break; + } + tokens.emplace_back(std::move(token)); + } + return tokens; + } + + uint32_t attempt_converge_on_stable_window_size(uint32_t max_pending) { + for (uint32_t i = 0; i < 999; ++i) { + auto tokens = fill_entire_throttle_window(); + uint32_t num_pending = static_cast<uint32_t>(tokens.size()); + + uint64_t trip_time = (num_pending < max_pending) ? 1000 : 1000 + (num_pending - max_pending) * 1000; + _milli_time += trip_time; + // Throttle window slots implicitly freed up as tokens are destructed. + } + uint32_t ret = _throttler->current_window_size(); + fprintf(stderr, "attempt_converge_on_stable_window_size() = %u\n", ret); + return ret; + } +}; + +TEST_F("window size changes dynamically based on throughput", WindowFixture()) { + uint32_t window_size = f1.attempt_converge_on_stable_window_size(100); + ASSERT_TRUE(window_size >= 90 && window_size <= 105); + + window_size = f1.attempt_converge_on_stable_window_size(200); + ASSERT_TRUE(window_size >= 180 && window_size <= 205); + + window_size = f1.attempt_converge_on_stable_window_size(50); + ASSERT_TRUE(window_size >= 45 && window_size <= 55); + + window_size = f1.attempt_converge_on_stable_window_size(500); + ASSERT_TRUE(window_size >= 450 && window_size <= 505); + + window_size = f1.attempt_converge_on_stable_window_size(100); + ASSERT_TRUE(window_size >= 90 && window_size <= 115); +} + +TEST_F("window size is reset after idle time period", WindowFixture(5, 1)) { + double window_size = f1.attempt_converge_on_stable_window_size(100); + ASSERT_TRUE(window_size >= 90 && window_size <= 110); + + f1._milli_time += 30001; // Not yet past 60s idle time + auto tokens = f1.fill_entire_throttle_window(); + ASSERT_TRUE(tokens.size() >= 90 && tokens.size() <= 110); + tokens.clear(); + + f1._milli_time += 60001; // Idle time passed + tokens = f1.fill_entire_throttle_window(); + EXPECT_EQUAL(tokens.size(), 1u); // Reduced to minimum window size +} + +TEST_F("minimum window size is respected", WindowFixture(5, 150, INT_MAX)) { + double window_size = f1.attempt_converge_on_stable_window_size(200); + ASSERT_TRUE(window_size >= 150 && window_size <= 210); +} + +TEST_F("maximum window size is respected", WindowFixture(5, 1, 50)) { + double window_size = f1.attempt_converge_on_stable_window_size(100); + ASSERT_TRUE(window_size >= 40 && window_size <= 50); +} + +} + +TEST_MAIN() { + TEST_RUN_ALL(); +} diff --git a/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp b/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp index 81c8271f755..5b267c3b9e9 100644 --- a/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp +++ b/vespalib/src/tests/shared_string_repo/shared_string_repo_test.cpp @@ -101,8 +101,8 @@ std::unique_ptr<Handles> copy_strong_handles(const Handles &handles) { return result; } -std::unique_ptr<std::vector<string_id>> make_weak_handles(const Handles &handles) { - return std::make_unique<std::vector<string_id>>(handles.view()); +std::unique_ptr<StringIdVector> make_weak_handles(const Handles &handles) { + return std::make_unique<StringIdVector>(handles.view()); } //----------------------------------------------------------------------------- @@ -202,7 +202,7 @@ struct Fixture { std::vector<vespalib::string> get_direct_result; std::unique_ptr<Handles> strong; std::unique_ptr<Handles> strong_copy; - std::unique_ptr<std::vector<string_id>> weak; + std::unique_ptr<StringIdVector> weak; auto copy_strings_task = [&](){ copy_strings_result = copy_strings(work); }; auto copy_and_hash_task = [&](){ copy_and_hash_result = copy_and_hash(work); }; auto local_enum_task = [&](){ local_enum_result = local_enum(work); }; diff --git a/vespalib/src/tests/spin_lock/spin_lock_test.cpp b/vespalib/src/tests/spin_lock/spin_lock_test.cpp index 54ad354f584..78e35a3e8d1 100644 --- a/vespalib/src/tests/spin_lock/spin_lock_test.cpp +++ b/vespalib/src/tests/spin_lock/spin_lock_test.cpp @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/spin_lock.h> +#include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/testkit/test_kit.h> #include <array> using namespace vespalib; +using namespace vespalib::atomic; bool verbose = false; double budget = 0.25; @@ -25,15 +27,15 @@ struct MyState { void update() { std::array<size_t,SZ> tmp; for (size_t i = 0; i < SZ; ++i) { - tmp[i] = state[i]; + store_ref_relaxed(tmp[i], load_ref_relaxed(state[i])); } for (size_t i = 0; i < SZ; ++i) { - state[i] = tmp[i] + 1; + store_ref_relaxed(state[i], load_ref_relaxed(tmp[i]) + 1); } } bool check(size_t expect) const { - for (size_t value: state) { - if (value != expect) { + for (const auto& value: state) { + if (load_ref_relaxed(value) != expect) { return false; } } diff --git a/vespalib/src/tests/stllike/asciistream_test.cpp b/vespalib/src/tests/stllike/asciistream_test.cpp index be0bc1cb694..3042595e18c 100644 --- a/vespalib/src/tests/stllike/asciistream_test.cpp +++ b/vespalib/src/tests/stllike/asciistream_test.cpp @@ -409,27 +409,14 @@ AsciistreamTest::testWriteThenRead() void AsciistreamTest::testGetLine() { - asciistream is(""); - EXPECT_TRUE(is.getlines().empty()); - is = asciistream("line 1"); - std::vector<string> v = is.getlines(); - EXPECT_EQUAL(1u, v.size()); - EXPECT_EQUAL("line 1", v[0]); - is = asciistream("line 1\nline 2"); - v = is.getlines(); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL("line 1", v[0]); - EXPECT_EQUAL("line 2", v[1]); - is = asciistream("line 1\nline 2\n\n"); - v = is.getlines(); - EXPECT_EQUAL(3u, v.size()); - EXPECT_EQUAL("line 1", v[0]); - EXPECT_EQUAL("line 2", v[1]); - EXPECT_EQUAL("", v[2]); - is = asciistream("line 1"); + asciistream is = asciistream("line 1\nline 2\nline 3"); string s; getline(is, s); EXPECT_EQUAL("line 1", s); + getline(is, s); + EXPECT_EQUAL("line 2", s); + getline(is, s); + EXPECT_EQUAL("line 3", s); } #define VERIFY_DOUBLE_SERIALIZATION(value, expected, format, precision) { \ diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index cf84ab03a25..14802a60ff1 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -1,11 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/rcuvector.h> #include <vespa/vespalib/util/size_literals.h> using namespace vespalib; +using vespalib::alloc::Alloc; +using vespalib::alloc::MemoryAllocator; +using MyMemoryAllocator = vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MyMemoryAllocator::Stats; + bool assertUsage(const MemoryUsage & exp, const MemoryUsage & act) { @@ -288,4 +294,70 @@ TEST("test small expand") g.trimHoldLists(2); } +struct Fixture { + using generation_t = GenerationHandler::generation_t; + + AllocStats stats; + std::unique_ptr<MemoryAllocator> allocator; + Alloc initial_alloc; + GenerationHolder g; + RcuVectorBase<int> arr; + + Fixture(); + ~Fixture(); + void transfer_and_trim(generation_t transfer_gen, generation_t trim_gen) + { + g.transferHoldLists(transfer_gen); + g.trimHoldLists(trim_gen); + } +}; + +Fixture::Fixture() + : stats(), + allocator(std::make_unique<MyMemoryAllocator>(stats)), + initial_alloc(Alloc::alloc_with_allocator(allocator.get())), + g(), + arr(g, initial_alloc) +{ + arr.reserve(100); +} + +Fixture::~Fixture() = default; + +TEST_F("require that memory allocator can be set", Fixture) +{ + EXPECT_EQUAL(AllocStats(2, 0), f.stats); + f.transfer_and_trim(1, 2); + EXPECT_EQUAL(AllocStats(2, 1), f.stats); +} + +TEST_F("require that memory allocator is preserved across reset", Fixture) +{ + f.arr.reset(); + f.arr.reserve(100); + EXPECT_EQUAL(AllocStats(4, 1), f.stats); + f.transfer_and_trim(1, 2); + EXPECT_EQUAL(AllocStats(4, 3), f.stats); +} + +TEST_F("require that created replacement vector uses same memory allocator", Fixture) +{ + auto arr2 = f.arr.create_replacement_vector(); + EXPECT_EQUAL(AllocStats(2, 0), f.stats); + arr2.reserve(100); + EXPECT_EQUAL(AllocStats(3, 0), f.stats); + f.transfer_and_trim(1, 2); + EXPECT_EQUAL(AllocStats(3, 1), f.stats); +} + +TEST_F("require that ensure_size and shrink use same memory allocator", Fixture) +{ + f.arr.ensure_size(2000); + EXPECT_EQUAL(AllocStats(3, 0), f.stats); + f.arr.shrink(1000); + EXPECT_EQUAL(AllocStats(4, 0), f.stats); + f.transfer_and_trim(1, 2); + EXPECT_EQUAL(AllocStats(4, 3), f.stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/wakeup/wakeup_bench.cpp b/vespalib/src/tests/wakeup/wakeup_bench.cpp index 1d9817508d3..c39b8899159 100644 --- a/vespalib/src/tests/wakeup/wakeup_bench.cpp +++ b/vespalib/src/tests/wakeup/wakeup_bench.cpp @@ -5,6 +5,7 @@ #include <condition_variable> #include <thread> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/cpu_usage.h> #ifdef __linux__ #include <linux/futex.h> @@ -121,18 +122,33 @@ struct UsePipe : State { set_wakeup(); char token = 'T'; [[maybe_unused]] ssize_t res = write(pipefd[1], &token, 1); - assert(res == 1); + // assert(res == 1); } void stop() { set_stop(); char token = 'T'; [[maybe_unused]] ssize_t res = write(pipefd[1], &token, 1); - assert(res == 1); + // assert(res == 1); } void wait() { char token_trash[128]; [[maybe_unused]] ssize_t res = read(pipefd[0], token_trash, sizeof(token_trash)); - assert(res == 1); + // assert(res == 1); + } +}; + +struct UseAtomic : State { + void wakeup() { + set_wakeup(); + value.notify_one(); + } + void stop() { + set_stop(); + value.notify_one(); + } + void wait() { + value.wait(0); + // assert(!is_ready()); } }; @@ -162,9 +178,11 @@ struct Wakeup : T { using T::should_stop; using T::set_ready; using T::wait; + cpu_usage::ThreadSampler::UP cpu; std::thread thread; Wakeup() : thread([this]{ run(); }) {} void run() { + cpu = cpu_usage::create_thread_sampler(); while (!should_stop()) { set_ready(); wait(); @@ -211,6 +229,15 @@ void wait_until_ready(const T &list) { } template <typename T> +duration sample_cpu(T &list) { + duration result = duration::zero(); + for (auto *item: list) { + result += item->cpu->sample(); + } + return result; +} + +template <typename T> auto perform_wakeups(T &list, size_t target) __attribute__((noinline)); template <typename T> auto perform_wakeups(T &list, size_t target) { @@ -239,11 +266,17 @@ void benchmark() { perform_wakeups(list, WAKE_CNT / 64); } auto t1 = steady_clock::now(); + auto cpu0 = sample_cpu(list); auto res = perform_wakeups(list, WAKE_CNT); auto t2 = steady_clock::now(); + auto cpu1 = sample_cpu(list); wait_until_ready(list); destroy_list(list); - fprintf(stderr, "wakeups per second: %zu (skipped: %zu)\n", size_t(res.first / to_s(t2 - t1)), res.second); + double run_time = to_s(t2 - t1); + double cpu_time = to_s(cpu1 - cpu0); + double cpu_load = (cpu_time / (N * run_time)); + fprintf(stderr, "wakeups per second: %zu (skipped: %zu, cpu load: %.3f)\n", + size_t(res.first / run_time), res.second, cpu_load); } TEST(WakeupBench, using_spin) { benchmark<Wakeup<UseSpin>>(); } @@ -251,6 +284,7 @@ TEST(WakeupBench, using_spin_yield) { benchmark<Wakeup<UseSpinYield>>(); } TEST(WakeupBench, using_cond) { benchmark<Wakeup<UseCond>>(); } TEST(WakeupBench, using_cond_nolock) { benchmark<Wakeup<UseCondNolock>>(); } TEST(WakeupBench, using_pipe) { benchmark<Wakeup<UsePipe>>(); } +TEST(WakeupBench, using_atomic) { benchmark<Wakeup<UseAtomic>>(); } #ifdef __linux__ TEST(WakeupBench, using_futex) { benchmark<Wakeup<UseFutex>>(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/array_value.h b/vespalib/src/vespa/vespalib/data/slime/array_value.h index b1b257e8476..216d116498c 100644 --- a/vespalib/src/vespa/vespalib/data/slime/array_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/array_value.h @@ -5,6 +5,7 @@ #include "value.h" #include "nix_value.h" #include "value_factory.h" +#include <vespa/vespalib/stllike/allocator.h> #include <vector> namespace vespalib::slime { @@ -18,7 +19,7 @@ class ArrayValue final : public Value private: SymbolTable &_symbolTable; Stash &_stash; - std::vector<Value*> _values; + std::vector<Value*, vespalib::allocator_large<Value*>> _values; protected: Cursor &addLeaf(const ValueFactory &input) override { diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index d628843279d..a2f7a4d4ff4 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -12,8 +12,11 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT entryref.cpp entry_ref_filter.cpp fixed_size_hash_map.cpp + large_array_buffer_type.cpp sharded_hash_map.cpp + small_array_buffer_type.cpp unique_store.cpp + unique_store_buffer_type.cpp unique_store_string_allocator.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.cpp b/vespalib/src/vespa/vespalib/datastore/array_store.cpp index 0e03b36d2f9..b03f21402a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.cpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.cpp @@ -5,10 +5,4 @@ namespace vespalib::datastore { -template class BufferType<vespalib::Array<uint8_t>>; -template class BufferType<vespalib::Array<uint32_t>>; -template class BufferType<vespalib::Array<int32_t>>; -template class BufferType<vespalib::Array<std::string>>; -template class BufferType<vespalib::Array<AtomicEntryRef>>; - } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index d9b62c310b5..ed3af451b04 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -9,6 +9,8 @@ #include "entryref.h" #include "atomic_entry_ref.h" #include "i_compaction_context.h" +#include "large_array_buffer_type.h" +#include "small_array_buffer_type.h" #include <vespa/vespalib/util/array.h> namespace vespalib::datastore { @@ -32,27 +34,15 @@ public: using SmallArrayType = BufferType<EntryT>; using LargeArray = vespalib::Array<EntryT>; using AllocSpec = ArrayStoreConfig::AllocSpec; - private: - class LargeArrayType : public BufferType<LargeArray> { - private: - using ParentType = BufferType<LargeArray>; - using ParentType::_emptyEntry; - using CleanContext = typename ParentType::CleanContext; - public: - LargeArrayType(const AllocSpec &spec); - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - }; - - uint32_t _largeArrayTypeId; uint32_t _maxSmallArraySize; DataStoreType _store; - std::vector<SmallArrayType> _smallArrayTypes; - LargeArrayType _largeArrayType; + std::vector<SmallArrayBufferType<EntryT>> _smallArrayTypes; + LargeArrayBufferType<EntryT> _largeArrayType; using generation_t = vespalib::GenerationHandler::generation_t; - void initArrayTypes(const ArrayStoreConfig &cfg); + void initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator); // 1-to-1 mapping between type ids and sizes for small arrays is enforced during initialization. uint32_t getTypeId(size_t arraySize) const { return arraySize; } size_t getArraySize(uint32_t typeId) const { return typeId; } @@ -68,7 +58,7 @@ private: } public: - ArrayStore(const ArrayStoreConfig &cfg); + ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator); ~ArrayStore(); EntryRef add(const ConstArrayRef &array); ConstArrayRef get(EntryRef ref) const { diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index bbbd52c354d..00c1615b173 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -6,40 +6,23 @@ #include "compaction_spec.h" #include "entry_ref_filter.h" #include "datastore.hpp" +#include "large_array_buffer_type.hpp" +#include "small_array_buffer_type.hpp" #include <atomic> #include <algorithm> namespace vespalib::datastore { template <typename EntryT, typename RefT> -ArrayStore<EntryT, RefT>::LargeArrayType::LargeArrayType(const AllocSpec &spec) - : BufferType<LargeArray>(1, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor) -{ -} - -template <typename EntryT, typename RefT> -void -ArrayStore<EntryT, RefT>::LargeArrayType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) -{ - LargeArray *elem = static_cast<LargeArray *>(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { - cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); - *elem = _emptyEntry; - ++elem; - } -} - -template <typename EntryT, typename RefT> void -ArrayStore<EntryT, RefT>::initArrayTypes(const ArrayStoreConfig &cfg) +ArrayStore<EntryT, RefT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); _smallArrayTypes.reserve(_maxSmallArraySize); for (uint32_t arraySize = 1; arraySize <= _maxSmallArraySize; ++arraySize) { const AllocSpec &spec = cfg.specForSize(arraySize); - _smallArrayTypes.emplace_back(arraySize, spec.minArraysInBuffer, spec.maxArraysInBuffer, - spec.numArraysForNewBuffer, spec.allocGrowFactor); + _smallArrayTypes.emplace_back(arraySize, spec, memory_allocator); } for (auto & type : _smallArrayTypes) { uint32_t typeId = _store.addType(&type); @@ -48,14 +31,14 @@ ArrayStore<EntryT, RefT>::initArrayTypes(const ArrayStoreConfig &cfg) } template <typename EntryT, typename RefT> -ArrayStore<EntryT, RefT>::ArrayStore(const ArrayStoreConfig &cfg) +ArrayStore<EntryT, RefT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : _largeArrayTypeId(0), _maxSmallArraySize(cfg.maxSmallArraySize()), _store(), _smallArrayTypes(), - _largeArrayType(cfg.specForSize(0)) + _largeArrayType(cfg.specForSize(0), memory_allocator) { - initArrayTypes(cfg); + initArrayTypes(cfg, std::move(memory_allocator)); _store.init_primary_buffers(); if (cfg.enable_free_lists()) { _store.enableFreeLists(); diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp new file mode 100644 index 00000000000..f4ccb27abad --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.cpp @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "large_array_buffer_type.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class BufferType<Array<uint8_t>>; +template class BufferType<Array<uint32_t>>; +template class BufferType<Array<int32_t>>; +template class BufferType<Array<std::string>>; +template class BufferType<Array<AtomicEntryRef>>; + +template class LargeArrayBufferType<uint8_t>; +template class LargeArrayBufferType<uint32_t>; +template class LargeArrayBufferType<int32_t>; +template class LargeArrayBufferType<std::string>; +template class LargeArrayBufferType<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h new file mode 100644 index 00000000000..50d15d4a27c --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "array_store_config.h" +#include "buffer_type.h" +#include <vespa/vespalib/util/array.h> +#include <memory> + +namespace vespalib::alloc { class MemoryAllocator; } + +namespace vespalib::datastore { + +/* + * Class representing buffer type for large arrays in ArrayStore + */ +template <typename EntryT> +class LargeArrayBufferType : public BufferType<Array<EntryT>> +{ + using AllocSpec = ArrayStoreConfig::AllocSpec; + using ArrayType = Array<EntryT>; + using ParentType = BufferType<ArrayType>; + using ParentType::_emptyEntry; + using CleanContext = typename ParentType::CleanContext; + std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; +public: + LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept; + ~LargeArrayBufferType() override; + void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; +}; + +extern template class LargeArrayBufferType<uint8_t>; +extern template class LargeArrayBufferType<uint32_t>; +extern template class LargeArrayBufferType<int32_t>; +extern template class LargeArrayBufferType<std::string>; +extern template class LargeArrayBufferType<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp new file mode 100644 index 00000000000..aeeef8166c6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "large_array_buffer_type.h" +#include <vespa/vespalib/util/array.hpp> + +namespace vespalib::datastore { + +template <typename EntryT> +LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<Array<EntryT>>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + _memory_allocator(std::move(memory_allocator)) +{ +} + +template <typename EntryT> +LargeArrayBufferType<EntryT>::~LargeArrayBufferType() = default; + +template <typename EntryT> +void +LargeArrayBufferType<EntryT>::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +{ + ArrayType* elem = static_cast<ArrayType*>(buffer) + offset; + for (size_t i = 0; i < numElems; ++i) { + cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); + *elem = _emptyEntry; + ++elem; + } +} + +template <typename EntryT> +const vespalib::alloc::MemoryAllocator* +LargeArrayBufferType<EntryT>::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp new file mode 100644 index 00000000000..06a4c6007a9 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.cpp @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "small_array_buffer_type.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class SmallArrayBufferType<uint8_t>; +template class SmallArrayBufferType<uint32_t>; +template class SmallArrayBufferType<int32_t>; +template class SmallArrayBufferType<std::string>; +template class SmallArrayBufferType<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h new file mode 100644 index 00000000000..4e4568c3d16 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "array_store_config.h" +#include "buffer_type.h" +#include <memory> + +namespace vespalib::alloc { class MemoryAllocator; } + +namespace vespalib::datastore { + +/* + * Class representing buffer type for small arrays in ArrayStore + */ +template <typename EntryT> +class SmallArrayBufferType : public BufferType<EntryT> +{ + using AllocSpec = ArrayStoreConfig::AllocSpec; + std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; +public: + SmallArrayBufferType(const SmallArrayBufferType&) = delete; + SmallArrayBufferType& operator=(const SmallArrayBufferType&) = delete; + SmallArrayBufferType(SmallArrayBufferType&&) noexcept = default; + SmallArrayBufferType& operator=(SmallArrayBufferType&&) noexcept = default; + SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept; + ~SmallArrayBufferType() override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; +}; + +extern template class SmallArrayBufferType<uint8_t>; +extern template class SmallArrayBufferType<uint32_t>; +extern template class SmallArrayBufferType<int32_t>; +extern template class SmallArrayBufferType<std::string>; +extern template class SmallArrayBufferType<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp new file mode 100644 index 00000000000..414804417eb --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "small_array_buffer_type.h" + +namespace vespalib::datastore { + +template <typename EntryT> +SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<EntryT>(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + _memory_allocator(std::move(memory_allocator)) +{ +} + +template <typename EntryT> +SmallArrayBufferType<EntryT>::~SmallArrayBufferType() = default; + +template <typename EntryT> +const vespalib::alloc::MemoryAllocator* +SmallArrayBufferType<EntryT>::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store.cpp index 0d740c30c84..dcd9b38fab8 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.cpp @@ -5,14 +5,6 @@ namespace vespalib::datastore { -template class BufferType<UniqueStoreEntry<int8_t>>; -template class BufferType<UniqueStoreEntry<int16_t>>; -template class BufferType<UniqueStoreEntry<int32_t>>; -template class BufferType<UniqueStoreEntry<int64_t>>; -template class BufferType<UniqueStoreEntry<uint32_t>>; -template class BufferType<UniqueStoreEntry<float>>; -template class BufferType<UniqueStoreEntry<double>>; - using namespace btree; VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(EntryRef, NoAggregated, uniquestore::DefaultDictionaryTraits::INTERNAL_SLOTS); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index aea98f406e8..81034ab4210 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -13,6 +13,8 @@ #include "unique_store_comparator.h" #include "unique_store_entry.h" +namespace vespalib::alloc { class MemoryAllocator; } + namespace vespalib::datastore { template <typename Allocator> @@ -47,8 +49,8 @@ private: using generation_t = vespalib::GenerationHandler::generation_t; public: - UniqueStore(); - UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict); + UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator); + UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict, std::shared_ptr<alloc::MemoryAllocator> memory_allocator); ~UniqueStore(); void set_dictionary(std::unique_ptr<IUniqueStoreDictionary> dict); UniqueStoreAddResult add(EntryConstRefType value); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index b73b714a6bc..b1a7db56545 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -27,14 +27,14 @@ using DefaultUniqueStoreDictionary = UniqueStoreDictionary<DefaultDictionary>; } template <typename EntryT, typename RefT, typename Compare, typename Allocator> -UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore() - : UniqueStore<EntryT, RefT, Compare, Allocator>(std::make_unique<uniquestore::DefaultUniqueStoreDictionary>(std::unique_ptr<EntryComparator>())) +UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::shared_ptr<alloc::MemoryAllocator> memory_allocator) + : UniqueStore<EntryT, RefT, Compare, Allocator>(std::make_unique<uniquestore::DefaultUniqueStoreDictionary>(std::unique_ptr<EntryComparator>()), std::move(memory_allocator)) { } template <typename EntryT, typename RefT, typename Compare, typename Allocator> -UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict) - : _allocator(), +UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) + : _allocator(std::move(memory_allocator)), _store(_allocator.get_data_store()), _dict(std::move(dict)) { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h index 0648f466bae..025165ee0e0 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h @@ -5,9 +5,12 @@ #include "datastore.h" #include "entryref.h" #include "unique_store_add_result.h" +#include "unique_store_buffer_type.h" #include "unique_store_entry.h" #include "i_compactable.h" +namespace vespalib::alloc { class MemoryAllocator; } + namespace vespalib::datastore { /** @@ -23,13 +26,12 @@ public: using EntryConstRefType = const EntryType &; using WrappedEntryType = UniqueStoreEntry<EntryType>; using RefType = RefT; - using UniqueStoreBufferType = BufferType<WrappedEntryType>; private: DataStoreType _store; - UniqueStoreBufferType _typeHandler; + UniqueStoreBufferType<WrappedEntryType> _typeHandler; public: - UniqueStoreAllocator(); + UniqueStoreAllocator(std::shared_ptr<alloc::MemoryAllocator> memory_allocator); ~UniqueStoreAllocator() override; EntryRef allocate(const EntryType& value); void hold(EntryRef ref); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp index eb634503c9d..04a229d4ffa 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp @@ -3,6 +3,7 @@ #pragma once #include "unique_store_allocator.h" +#include "unique_store_buffer_type.hpp" #include "unique_store_value_filter.h" #include "datastore.hpp" #include <vespa/vespalib/util/size_literals.h> @@ -13,10 +14,10 @@ constexpr size_t NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER = 1_Ki; constexpr float ALLOC_GROW_FACTOR = 0.2; template <typename EntryT, typename RefT> -UniqueStoreAllocator<EntryT, RefT>::UniqueStoreAllocator() +UniqueStoreAllocator<EntryT, RefT>::UniqueStoreAllocator(std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : ICompactable(), _store(), - _typeHandler(1, 2u, RefT::offsetSize(), NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR) + _typeHandler(2u, RefT::offsetSize(), NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR, std::move(memory_allocator)) { auto typeId = _store.addType(&_typeHandler); assert(typeId == 0u); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.cpp new file mode 100644 index 00000000000..3441a9435a1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.cpp @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "unique_store_buffer_type.hpp" +#include "unique_store_entry.h" + +namespace vespalib::datastore { + +template class BufferType<UniqueStoreEntry<int8_t>>; +template class BufferType<UniqueStoreEntry<int16_t>>; +template class BufferType<UniqueStoreEntry<int32_t>>; +template class BufferType<UniqueStoreEntry<int64_t>>; +template class BufferType<UniqueStoreEntry<uint32_t>>; +template class BufferType<UniqueStoreEntry<float>>; +template class BufferType<UniqueStoreEntry<double>>; + +template class UniqueStoreBufferType<UniqueStoreEntry<int8_t>>; +template class UniqueStoreBufferType<UniqueStoreEntry<int16_t>>; +template class UniqueStoreBufferType<UniqueStoreEntry<int32_t>>; +template class UniqueStoreBufferType<UniqueStoreEntry<int64_t>>; +template class UniqueStoreBufferType<UniqueStoreEntry<uint32_t>>; +template class UniqueStoreBufferType<UniqueStoreEntry<float>>; +template class UniqueStoreBufferType<UniqueStoreEntry<double>>; + +}; + diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.h new file mode 100644 index 00000000000..2aba785dd91 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "buffer_type.h" +#include <memory> + +namespace vespalib::alloc { class MemoryAllocator; } + +namespace vespalib::datastore { + +/* + * Class representing buffer type for a normal unique store allocator. + */ +template <typename WrappedEntry> +class UniqueStoreBufferType : public BufferType<WrappedEntry> +{ + std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; +public: + UniqueStoreBufferType(uint32_t min_arrays, uint32_t max_arrays, + uint32_t num_arrays_for_new_buffer, float alloc_grow_factor, + std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept; + ~UniqueStoreBufferType() override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.hpp new file mode 100644 index 00000000000..c99033106ee --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_buffer_type.hpp @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "unique_store_buffer_type.h" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template <typename WrappedEntry> +UniqueStoreBufferType<WrappedEntry>::UniqueStoreBufferType(uint32_t min_arrays, uint32_t max_arrays, + uint32_t num_arrays_for_new_buffer, float alloc_grow_factor, + std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<WrappedEntry>(1u, min_arrays, max_arrays, num_arrays_for_new_buffer, alloc_grow_factor), + _memory_allocator(std::move(memory_allocator)) +{ +} + +template <typename WrappedEntry> +UniqueStoreBufferType<WrappedEntry>::~UniqueStoreBufferType() = default; + +template <typename WrappedEntry> +const vespalib::alloc::MemoryAllocator* +UniqueStoreBufferType<WrappedEntry>::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp index 664f1e4432e..9c639067615 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp @@ -33,8 +33,9 @@ get_type_id(size_t string_len) } -UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays) - : BufferType<char>(array_size, 2u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR) +UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : BufferType<char>(array_size, 2u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR), + _memory_allocator(std::move(memory_allocator)) { } @@ -68,8 +69,15 @@ UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, ElemCou assert(e == e_end); } -UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays) - : BufferType<UniqueStoreEntry<std::string>>(array_size, 2u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR) +const vespalib::alloc::MemoryAllocator* +UniqueStoreSmallStringBufferType::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + +UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) + : BufferType<UniqueStoreEntry<std::string>>(array_size, 2u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR), + _memory_allocator(std::move(memory_allocator)) { } @@ -86,6 +94,12 @@ UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, Elem } } +const vespalib::alloc::MemoryAllocator* +UniqueStoreExternalStringBufferType::get_memory_allocator() const +{ + return _memory_allocator.get(); +} + template class UniqueStoreStringAllocator<EntryRefT<22>>; template class BufferType<UniqueStoreEntry<std::string>>; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h index 7b4c578f248..be5fa8f6c1e 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h @@ -10,6 +10,8 @@ #include <cassert> #include <string> +namespace vespalib::alloc { class MemoryAllocator; } + namespace vespalib::datastore { namespace string_allocator { @@ -56,22 +58,26 @@ public: * bytes */ class UniqueStoreSmallStringBufferType : public BufferType<char> { + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; public: - UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays); + UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreSmallStringBufferType() override; void destroyElements(void *, ElemCount) override; void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; /* * Buffer type for external strings in unique store. */ class UniqueStoreExternalStringBufferType : public BufferType<UniqueStoreEntry<std::string>> { + std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; public: - UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays); + UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreExternalStringBufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; /** @@ -101,7 +107,7 @@ private: static uint32_t get_type_id(const char *value); public: - UniqueStoreStringAllocator(); + UniqueStoreStringAllocator(std::shared_ptr<alloc::MemoryAllocator> memory_allocator); ~UniqueStoreStringAllocator() override; EntryRef allocate(const char *value); void hold(EntryRef ref); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp index 90c44200fbb..71ea16bcde2 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp @@ -8,14 +8,14 @@ namespace vespalib::datastore { template <typename RefT> -UniqueStoreStringAllocator<RefT>::UniqueStoreStringAllocator() +UniqueStoreStringAllocator<RefT>::UniqueStoreStringAllocator(std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : ICompactable(), _store(), _type_handlers() { - _type_handlers.emplace_back(std::make_unique<UniqueStoreExternalStringBufferType>(1, RefT::offsetSize())); + _type_handlers.emplace_back(std::make_unique<UniqueStoreExternalStringBufferType>(1, RefT::offsetSize(), memory_allocator)); for (auto size : string_allocator::array_sizes) { - _type_handlers.emplace_back(std::make_unique<UniqueStoreSmallStringBufferType>(size, RefT::offsetSize())); + _type_handlers.emplace_back(std::make_unique<UniqueStoreSmallStringBufferType>(size, RefT::offsetSize(), memory_allocator)); } uint32_t exp_type_id = 0; for (auto &type_handler : _type_handlers) { diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp index bedd0f119c6..eb127c7051a 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp @@ -3,15 +3,14 @@ #include "asciistream.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/locale/c.h> #include <vespa/fastos/file.h> #include <limits> -#include <stdexcept> #include <cassert> -#include <cmath> #include <charconv> +#include <vector> #include <vespa/log/log.h> LOG_SETUP(".vespalib.stllike.asciistream"); @@ -19,20 +18,23 @@ LOG_SETUP(".vespalib.stllike.asciistream"); namespace vespalib { namespace { - std::vector<string> getPrecisions(const char type) { - std::vector<string> result(VESPALIB_ASCIISTREAM_MAX_PRECISION + 1); - for (uint32_t i=0; i<result.size(); ++i) { - char buf[8]; - int count = snprintf(buf, sizeof(buf), "%%.%u%c", i, type); - assert(size_t(count) < sizeof(buf)); // Assert no truncation. - (void) count; - result[i] = buf; - } - return result; + +std::vector<string> +getPrecisions(const char type) { + std::vector<string> result(VESPALIB_ASCIISTREAM_MAX_PRECISION + 1); + for (uint32_t i=0; i<result.size(); ++i) { + char buf[8]; + int count = snprintf(buf, sizeof(buf), "%%.%u%c", i, type); + assert(size_t(count) < sizeof(buf)); // Assert no truncation. + (void) count; + result[i] = buf; } - std::vector<string> fixedPrecisions = getPrecisions('f'); - std::vector<string> scientificPrecisions = getPrecisions('e'); - std::vector<string> autoPrecisions = getPrecisions('g'); + return result; +} +std::vector<string> fixedPrecisions = getPrecisions('f'); +std::vector<string> scientificPrecisions = getPrecisions('e'); +std::vector<string> autoPrecisions = getPrecisions('g'); + } asciistream & @@ -93,7 +95,8 @@ asciistream::asciistream(const asciistream & rhs) : { } -asciistream & asciistream::operator = (const asciistream & rhs) +asciistream & +asciistream::operator = (const asciistream & rhs) { if (this != &rhs) { asciistream newStream(rhs); @@ -108,7 +111,8 @@ asciistream::asciistream(asciistream && rhs) noexcept swap(rhs); } -asciistream & asciistream::operator = (asciistream && rhs) noexcept +asciistream & +asciistream::operator = (asciistream && rhs) noexcept { if (this != &rhs) { swap(rhs); @@ -116,7 +120,8 @@ asciistream & asciistream::operator = (asciistream && rhs) noexcept return *this; } -void asciistream::swap(asciistream & rhs) noexcept +void +asciistream::swap(asciistream & rhs) noexcept { std::swap(_rPos, rhs._rPos); // If read-only, _wbuf is empty and _rbuf is set @@ -150,7 +155,8 @@ void throwUnderflow(size_t pos) __attribute__((noinline)); template <typename T> T strToInt(T & v, const char *begin, const char *end) __attribute__((noinline)); -void throwInputError(int e, const char * t, const char * buf) +void +throwInputError(int e, const char * t, const char * buf) { if (e == 0) { throw IllegalArgumentException("Failed decoding a " + string(t) + " from '" + string(buf) + "'.", VESPA_STRLOC); @@ -163,7 +169,8 @@ void throwInputError(int e, const char * t, const char * buf) } } -void throwInputError(std::errc e, const char * t, const char * buf) { +void +throwInputError(std::errc e, const char * t, const char * buf) { if (e == std::errc::invalid_argument) { throw IllegalArgumentException("Illegal " + string(t) + " value '" + string(buf) + "'.", VESPA_STRLOC); } else if (e == std::errc::result_out_of_range) { @@ -173,12 +180,14 @@ void throwInputError(std::errc e, const char * t, const char * buf) { } } -void throwUnderflow(size_t pos) +void +throwUnderflow(size_t pos) { throw IllegalArgumentException(make_string("buffer underflow at pos %ld.", pos), VESPA_STRLOC); } -int getValue(double & val, const char *buf) +int +getValue(double & val, const char *buf) { char *ebuf; errno = 0; @@ -189,7 +198,8 @@ int getValue(double & val, const char *buf) return ebuf - buf; } -int getValue(float & val, const char *buf) +int +getValue(float & val, const char *buf) { char *ebuf; errno = 0; @@ -201,7 +211,8 @@ int getValue(float & val, const char *buf) } template <typename T> -T strToInt(T & v, const char *begin, const char *end) +T +strToInt(T & v, const char *begin, const char *end) { const char * curr = begin; for (;(curr < end) && std::isspace(*curr); curr++); @@ -226,7 +237,8 @@ T strToInt(T & v, const char *begin, const char *end) } -asciistream & asciistream::operator >> (bool & v) +asciistream & +asciistream::operator >> (bool & v) { for (;(_rPos < length()) && std::isspace(_rbuf[_rPos]); _rPos++); if (_rPos < length()) { @@ -237,7 +249,8 @@ asciistream & asciistream::operator >> (bool & v) return *this; } -asciistream & asciistream::operator >> (char & v) +asciistream & +asciistream::operator >> (char & v) { for (;(_rPos < length()) && std::isspace(_rbuf[_rPos]); _rPos++); if (_rPos < length()) { @@ -248,7 +261,8 @@ asciistream & asciistream::operator >> (char & v) return *this; } -asciistream & asciistream::operator >> (signed char & v) +asciistream & +asciistream::operator >> (signed char & v) { for (;(_rPos < length()) && std::isspace(_rbuf[_rPos]); _rPos++); if (_rPos < length()) { @@ -259,7 +273,8 @@ asciistream & asciistream::operator >> (signed char & v) return *this; } -asciistream & asciistream::operator >> (unsigned char & v) +asciistream & +asciistream::operator >> (unsigned char & v) { for (;(_rPos < length()) && std::isspace(_rbuf[_rPos]); _rPos++); if (_rPos < length()) { @@ -270,55 +285,64 @@ asciistream & asciistream::operator >> (unsigned char & v) return *this; } -asciistream & asciistream::operator >> (unsigned short & v) +asciistream & +asciistream::operator >> (unsigned short & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (unsigned int & v) +asciistream & +asciistream::operator >> (unsigned int & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (unsigned long & v) +asciistream & +asciistream::operator >> (unsigned long & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (unsigned long long & v) +asciistream & +asciistream::operator >> (unsigned long long & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (short & v) +asciistream & +asciistream::operator >> (short & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (int & v) +asciistream & +asciistream::operator >> (int & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (long & v) +asciistream & +asciistream::operator >> (long & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (long long & v) +asciistream & +asciistream::operator >> (long long & v) { _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } -asciistream & asciistream::operator >> (double & v) +asciistream & +asciistream::operator >> (double & v) { double l(0); _rPos += getValue(l, &_rbuf[_rPos]); @@ -326,7 +350,8 @@ asciistream & asciistream::operator >> (double & v) return *this; } -asciistream & asciistream::operator >> (float & v) +asciistream & +asciistream::operator >> (float & v) { float l(0); _rPos += getValue(l, &_rbuf[_rPos]); @@ -334,7 +359,8 @@ asciistream & asciistream::operator >> (float & v) return *this; } -void asciistream::eatWhite() +void +asciistream::eatWhite() { for (;(_rPos < length()) && isspace(_rbuf[_rPos]); _rPos++); } @@ -344,7 +370,8 @@ void asciistream::eatNonWhite() for (;(_rPos < length()) && !isspace(_rbuf[_rPos]); _rPos++); } -asciistream & asciistream::operator >> (std::string & v) +asciistream & +asciistream::operator >> (std::string & v) { eatWhite(); size_t start(_rPos); @@ -353,7 +380,8 @@ asciistream & asciistream::operator >> (std::string & v) return *this; } -asciistream & asciistream::operator >> (string & v) +asciistream & +asciistream::operator >> (string & v) { eatWhite(); size_t start(_rPos); @@ -366,7 +394,8 @@ asciistream & asciistream::operator >> (string & v) namespace { const char * _C_char = "0123456789abcdefg"; -char * prependInt(char * tmp, Base base) +char * +prependInt(char * tmp, Base base) { if (base == bin) { tmp[1] = 'b'; @@ -376,7 +405,8 @@ char * prependInt(char * tmp, Base base) return tmp + 2; } -char * prependSign(bool sign, char * tmp) +char * +prependSign(bool sign, char * tmp) { if (sign) { tmp[0] = '-'; @@ -389,7 +419,8 @@ template <uint8_t base> uint8_t printInt(unsigned long long r, char * tmp, uint8_t i) __attribute__((noinline)); template <uint8_t base> -uint8_t printInt(unsigned long long r, char * tmp, uint8_t i) +uint8_t +printInt(unsigned long long r, char * tmp, uint8_t i) { for(; r; i--, r/=base) { uint8_t d = r%base; @@ -401,7 +432,8 @@ uint8_t printInt(unsigned long long r, char * tmp, uint8_t i) } -asciistream & asciistream::operator << (long long v) +asciistream & +asciistream::operator << (long long v) { char tmp[72]; uint8_t i(sizeof(tmp)); @@ -432,14 +464,16 @@ asciistream & asciistream::operator << (long long v) return *this; } -void asciistream::doReallyFill(size_t currWidth) +void +asciistream::doReallyFill(size_t currWidth) { for (; _width > currWidth; currWidth++) { write(&_fill, 1); } } -asciistream & asciistream::operator << (unsigned long long v) +asciistream & +asciistream::operator << (unsigned long long v) { char tmp[72]; uint8_t i(sizeof(tmp)); @@ -477,13 +511,15 @@ struct BaseStateSaver { }; } -asciistream& asciistream::operator<<(const void* p) +asciistream & +asciistream::operator<<(const void* p) { BaseStateSaver saver(*this, _base); return *this << "0x" << hex << reinterpret_cast<uint64_t>(p); } -asciistream & asciistream::operator << (float v) +asciistream & +asciistream::operator << (float v) { if (_floatSpec == fixed) { printFixed(v); @@ -493,7 +529,8 @@ asciistream & asciistream::operator << (float v) return *this; } -asciistream & asciistream::operator << (double v) +asciistream & +asciistream::operator << (double v) { if (_floatSpec == fixed) { printFixed(v); @@ -516,20 +553,20 @@ void asciistream::printFixed(T v) } namespace { - bool hasDotOrIsScientific(const char* string, size_t len) { - for (size_t i=0; i<len; ++i) { - switch (string[i]) { - case '.': - case ',': - case 'e': - case 'E': - return true; - default: - break; - } +bool hasDotOrIsScientific(const char* string, size_t len) { + for (size_t i=0; i<len; ++i) { + switch (string[i]) { + case '.': + case ',': + case 'e': + case 'E': + return true; + default: + break; } - return false; } + return false; +} } template <typename T> @@ -548,7 +585,8 @@ void asciistream::printScientific(T v) } } -void asciistream::write(const void * buf, size_t len) +void +asciistream::write(const void * buf, size_t len) { if (_rPos > 0 && _rPos == length()) { clear(); @@ -564,16 +602,8 @@ void asciistream::write(const void * buf, size_t len) _rbuf = _wbuf; } -std::vector<string> asciistream::getlines(char delim) -{ - std::vector<string> lines; - while (!eof()) { - lines.push_back(getline(delim)); - } - return lines; -} - -string asciistream::getline(char delim) +string +asciistream::getline(char delim) { string line; const size_t start(_rPos); @@ -588,7 +618,8 @@ string asciistream::getline(char delim) return line; } -asciistream asciistream::createFromFile(stringref fileName) +asciistream +asciistream::createFromFile(stringref fileName) { FastOS_File file(vespalib::string(fileName).c_str()); asciistream is; @@ -597,32 +628,35 @@ asciistream asciistream::createFromFile(stringref fileName) if (sz < 0) { throw IoException("Failed getting size of file " + fileName + " : Error=" + file.getLastErrorString(), IoException::UNSPECIFIED, VESPA_STRLOC); } - MallocPtr buf(sz); - ssize_t actual = file.Read(buf, sz); + alloc::Alloc buf = alloc::Alloc::alloc(sz); + ssize_t actual = file.Read(buf.get(), sz); if (actual != sz) { asciistream e; e << "Failed reading " << sz << " bytes from file " << fileName; throw IoException(e.str() + " : Error=" + file.getLastErrorString(), IoException::UNSPECIFIED, VESPA_STRLOC); } - is << stringref(buf.c_str(), buf.size()); + is << stringref(static_cast<const char *>(buf.get()), sz); } return is; } -asciistream asciistream::createFromDevice(stringref fileName) +asciistream +asciistream::createFromDevice(stringref fileName) { FastOS_File file(vespalib::string(fileName).c_str()); asciistream is; if (file.OpenReadOnly()) { - MallocPtr buf(64_Ki); - for (ssize_t actual = file.Read(buf, buf.size()); actual > 0; actual = file.Read(buf, buf.size())) { - is << stringref(buf.c_str(), actual); + constexpr size_t SZ = 64_Ki; + auto buf = std::make_unique<char []>(SZ); + for (ssize_t actual = file.Read(buf.get(), SZ); actual > 0; actual = file.Read(buf.get(), SZ)) { + is << stringref(buf.get(), actual); } } return is; } -ssize_t getline(asciistream & is, string & line, char delim) +ssize_t +getline(asciistream & is, string & line, char delim) { line = is.getline(delim); return line.size(); diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.h b/vespalib/src/vespa/vespalib/stllike/asciistream.h index 6a4b4378634..b333be6cec2 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.h +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vector> namespace vespalib { @@ -153,7 +152,6 @@ public: static asciistream createFromFile(stringref fileName); static asciistream createFromDevice(stringref fileName); string getline(char delim='\n'); - std::vector<string> getlines(char delim='\n'); char getFill() const noexcept { return _fill; } size_t getWidth() const noexcept { return static_cast<size_t>(_width); } // match input type of setw Base getBase() const noexcept { return _base; } diff --git a/vespalib/src/vespa/vespalib/test/CMakeLists.txt b/vespalib/src/vespa/vespalib/test/CMakeLists.txt index d658b28f94b..a60eb15a4d4 100644 --- a/vespalib/src/vespa/vespalib/test/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/test/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(vespalib_vespalib_test OBJECT make_tls_options_for_testing.cpp memory_allocator_observer.cpp peer_policy_utils.cpp + thread_meets.cpp time_tracer.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/test/thread_meets.cpp b/vespalib/src/vespa/vespalib/test/thread_meets.cpp new file mode 100644 index 00000000000..9d23e0eab28 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/thread_meets.cpp @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "thread_meets.h" + +namespace vespalib::test { + +void +ThreadMeets::Nop::mingle() +{ +} + +} diff --git a/vespalib/src/vespa/vespalib/test/thread_meets.h b/vespalib/src/vespa/vespalib/test/thread_meets.h new file mode 100644 index 00000000000..62ca7779935 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/thread_meets.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/rendezvous.h> + +namespace vespalib::test { + +/** + * Generally useful rendezvous implementations. + **/ +struct ThreadMeets { + // can be used as a simple thread barrier + struct Nop : vespalib::Rendezvous<bool,bool> { + Nop(size_t N) : vespalib::Rendezvous<bool,bool>(N) {} + void operator()() { rendezvous(false); } + void mingle() override; + }; + // swap values between 2 threads + template <typename T> + struct Swap : vespalib::Rendezvous<T,T> { + using vespalib::Rendezvous<T,T>::in; + using vespalib::Rendezvous<T,T>::out; + using vespalib::Rendezvous<T,T>::rendezvous; + Swap() : vespalib::Rendezvous<T,T>(2) {} + T operator()(T input) { return rendezvous(input); } + void mingle() override { + out(1) = in(0); + out(0) = in(1); + } + }; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 32f679d22d7..752bf60c688 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -42,6 +42,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT mmap_file_allocator.cpp mmap_file_allocator_factory.cpp monitored_refcount.cpp + nice.cpp printable.cpp priority_queue.cpp random.cpp @@ -56,6 +57,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT runnable_pair.cpp sequence.cpp sha1.cpp + shared_operation_throttler.cpp shared_string_repo.cpp sig_catch.cpp signalhandler.cpp diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 69f2eedcecb..bf5f0200600 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -381,6 +381,9 @@ MMapAllocator::sresize_inplace(PtrAndSize current, size_t newSize) { size_t MMapAllocator::extend_inplace(PtrAndSize current, size_t newSize) { + if (current.second == 0u) { + return 0u; + } PtrAndSize got = MMapAllocator::salloc(newSize - current.second, static_cast<char *>(current.first)+current.second); if ((static_cast<const char *>(current.first) + current.second) == static_cast<const char *>(got.first)) { return current.second + got.second; @@ -477,6 +480,9 @@ Alloc::allocHeap(size_t sz) bool Alloc::resize_inplace(size_t newSize) { + if (newSize == 0u) { + return size() == 0u; + } size_t extendedSize = _allocator->resize_inplace(_alloc, newSize); if (extendedSize >= newSize) { _alloc.second = extendedSize; diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index d25fb6f6a7c..4066894b4e3 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -62,6 +62,13 @@ public: std::swap(_alloc, rhs._alloc); std::swap(_allocator, rhs._allocator); } + void reset() { + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + _alloc.first = nullptr; + _alloc.second = 0u; + } + } Alloc create(size_t sz) const noexcept { return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz); } diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index 30d87cd98f6..cb5d5c7cc63 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -135,6 +135,7 @@ public: std::destroy(array(0), array(_sz)); _sz = 0; } + void reset(); bool empty() const { return _sz == 0; } T & operator [] (size_t i) { return *array(i); } const T & operator [] (size_t i) const { return *array(i); } @@ -145,6 +146,7 @@ public: rhs._sz = 0; return std::move(rhs._array); } + Array<T> create() const; private: T * array(size_t i) { return static_cast<T *>(_array.get()) + i; } const T * array(size_t i) const { return static_cast<const T *>(_array.get()) + i; } diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index e9070f5759c..72178f0391b 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -205,5 +205,20 @@ void Array<T>::cleanup() Alloc().swap(_array); } +template <typename T> +void Array<T>::reset() +{ + std::destroy(array(0), array(_sz)); + _sz = 0; + _array.reset(); +} + +template <typename T> +Array<T> +Array<T>::create() const +{ + return Array<T>(_array); // Use same memory allocator +} + } diff --git a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp index 73e70e7fd89..8f3dd8ab006 100644 --- a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp +++ b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp @@ -2,11 +2,11 @@ #pragma once -#include <stdint.h> -#include <stdlib.h> +#include "traits.h" +#include <cstdint> +#include <cstdlib> #include <cassert> #include <algorithm> -#include "traits.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/atomic.h b/vespalib/src/vespa/vespalib/util/atomic.h new file mode 100644 index 00000000000..8bf258ffa50 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/atomic.h @@ -0,0 +1,151 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <atomic> +#include <type_traits> +#include <version> + +/** + * Utility functions for single value atomic memory accesses. + * + * store/load_ref_* functions can be used to provide well-defined atomic + * memory access to memory locations that aren't explicitly wrapped in std::atomic + * objects. In this case, all potentially racing loads/stores _must_ be through + * atomic utility functions (or atomic_ref). + * + * Non-ref store/load_* functions are just syntactic sugar to make code using + * atomics more readable, but additionally adds sanity checks that all atomics + * are always lock-free. + */ + +namespace vespalib::atomic { + +// +// std::atomic_ref<T> helpers +// + +namespace detail { +template <typename T> struct is_std_atomic : std::false_type {}; +template <typename T> struct is_std_atomic<std::atomic<T>> : std::true_type {}; +template <typename T> inline constexpr bool is_std_atomic_v = is_std_atomic<T>::value; +} + +// TODO can generalize atomic_ref code once no special casing is needed + +template <typename T1, typename T2> +constexpr void store_ref_relaxed(T1& lhs, T2&& v) noexcept { + static_assert(!detail::is_std_atomic_v<T1>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<T1>::is_always_lock_free); + std::atomic_ref<T1>(lhs).store(std::forward<T2>(v), std::memory_order_relaxed); +#else + // TODO replace with compiler intrinsic + lhs = std::forward<T2>(v); +#endif +} + +template <typename T1, typename T2> +constexpr void store_ref_release(T1& lhs, T2&& v) noexcept { + static_assert(!detail::is_std_atomic_v<T1>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<T1>::is_always_lock_free); + std::atomic_ref<T1>(lhs).store(std::forward<T2>(v), std::memory_order_release); +#else + // TODO replace with compiler intrinsic + lhs = std::forward<T2>(v); + std::atomic_thread_fence(std::memory_order_release); +#endif +} + +template <typename T1, typename T2> +constexpr void store_ref_seq_cst(T1& lhs, T2&& v) noexcept { + static_assert(!detail::is_std_atomic_v<T1>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<T1>::is_always_lock_free); + std::atomic_ref<T1>(lhs).store(std::forward<T2>(v), std::memory_order_seq_cst); +#else + // TODO replace with compiler intrinsic + lhs = std::forward<T2>(v); + std::atomic_thread_fence(std::memory_order_seq_cst); +#endif +} + +template <typename T> +[[nodiscard]] constexpr T load_ref_relaxed(const T& a) noexcept { + static_assert(!detail::is_std_atomic_v<T>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<const T>::is_always_lock_free); + return std::atomic_ref<const T>(a).load(std::memory_order_relaxed); +#else + // TODO replace with compiler intrinsic + return a; +#endif +} + +template <typename T> +[[nodiscard]] constexpr T load_ref_acquire(const T& a) noexcept { + static_assert(!detail::is_std_atomic_v<T>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<const T>::is_always_lock_free); + return std::atomic_ref<const T>(a).load(std::memory_order_acquire); +#else + // TODO replace with compiler intrinsic + std::atomic_thread_fence(std::memory_order_acquire); + return a; +#endif +} + +template <typename T> +[[nodiscard]] constexpr T load_ref_seq_cst(const T& a) noexcept { + static_assert(!detail::is_std_atomic_v<T>, "atomic ref function invoked with a std::atomic, probably not intended"); +#if __cpp_lib_atomic_ref + static_assert(std::atomic_ref<const T>::is_always_lock_free); + return std::atomic_ref<const T>(a).load(std::memory_order_seq_cst); +#else + // TODO replace with compiler intrinsic + std::atomic_thread_fence(std::memory_order_seq_cst); + return a; +#endif +} + +// +// std::atomic<T> helpers +// + +template <typename T1, typename T2> +constexpr void store_relaxed(std::atomic<T1>& lhs, T2&& v) noexcept { + static_assert(std::atomic<T1>::is_always_lock_free); + lhs.store(std::forward<T2>(v), std::memory_order_relaxed); +} + +template <typename T1, typename T2> +constexpr void store_release(std::atomic<T1>& lhs, T2&& v) noexcept { + static_assert(std::atomic<T1>::is_always_lock_free); + lhs.store(std::forward<T2>(v), std::memory_order_release); +} + +template <typename T1, typename T2> +constexpr void store_seq_cst(std::atomic<T1>& lhs, T2&& v) noexcept { + static_assert(std::atomic<T1>::is_always_lock_free); + lhs.store(std::forward<T2>(v), std::memory_order_seq_cst); +} + +template <typename T> +[[nodiscard]] constexpr T load_relaxed(const std::atomic<T>& a) noexcept { + static_assert(std::atomic<T>::is_always_lock_free); + return a.load(std::memory_order_relaxed); +} + +template <typename T> +[[nodiscard]] constexpr T load_acquire(const std::atomic<T>& a) noexcept { + static_assert(std::atomic<T>::is_always_lock_free); + return a.load(std::memory_order_acquire); +} + +template <typename T> +[[nodiscard]] constexpr T load_seq_cst(const std::atomic<T>& a) noexcept { + static_assert(std::atomic<T>::is_always_lock_free); + return a.load(std::memory_order_seq_cst); +} + +} // vespalib::atomic diff --git a/vespalib/src/vespa/vespalib/util/child_process.cpp b/vespalib/src/vespa/vespalib/util/child_process.cpp index 7193a445f9a..694bed60f1b 100644 --- a/vespalib/src/vespa/vespalib/util/child_process.cpp +++ b/vespalib/src/vespa/vespalib/util/child_process.cpp @@ -67,7 +67,9 @@ ChildProcess::Reader::OnReceiveData(const void *data, size_t length) return; } if (buf == nullptr) { // EOF - _gotEOF = true; + if (--_num_streams == 0) { + _gotEOF = true; + } } else { _queue.push(std::string(buf, length)); } @@ -107,11 +109,12 @@ ChildProcess::Reader::updateEOF() } -ChildProcess::Reader::Reader() +ChildProcess::Reader::Reader(int num_streams) : _lock(), _cond(), _queue(), _data(), + _num_streams(num_streams), _gotEOF(false), _waitCnt(0), _readEOF(false) @@ -203,7 +206,7 @@ ChildProcess::checkProc() ChildProcess::ChildProcess(const char *cmd) - : _reader(), + : _reader(1), _proc(cmd, true, &_reader), _running(false), _failed(false), @@ -213,6 +216,17 @@ ChildProcess::ChildProcess(const char *cmd) _failed = !_running; } +ChildProcess::ChildProcess(const char *cmd, capture_stderr_tag) + : _reader(2), + _proc(cmd, true, &_reader, &_reader), + _running(false), + _failed(false), + _exitCode(-918273645) +{ + _running = _proc.CreateWithShell(); + _failed = !_running; +} + ChildProcess::~ChildProcess() = default; diff --git a/vespalib/src/vespa/vespalib/util/child_process.h b/vespalib/src/vespa/vespalib/util/child_process.h index 646a2c7c6c9..877c56a8cb1 100644 --- a/vespalib/src/vespa/vespalib/util/child_process.h +++ b/vespalib/src/vespa/vespalib/util/child_process.h @@ -28,6 +28,7 @@ private: std::condition_variable _cond; std::queue<std::string> _queue; std::string _data; + int _num_streams; bool _gotEOF; int _waitCnt; bool _readEOF; @@ -38,7 +39,7 @@ private: void updateEOF(); public: - Reader(); + Reader(int num_streams); ~Reader() override; uint32_t read(char *buf, uint32_t len, int msTimeout); @@ -57,6 +58,7 @@ private: public: ChildProcess(const ChildProcess &) = delete; ChildProcess &operator=(const ChildProcess &) = delete; + struct capture_stderr_tag{}; /** * @brief Run a child process @@ -66,6 +68,15 @@ public: **/ explicit ChildProcess(const char *cmd); + /** + * @brief Run a child process + * + * Starts a process running the given command. stderr is + * redirected into stdout. + * @param cmd A shell command line to run + **/ + explicit ChildProcess(const char *cmd, capture_stderr_tag); + /** @brief destructor doing cleanup if needed */ ~ChildProcess(); diff --git a/vespalib/src/vespa/vespalib/util/count_down_latch.h b/vespalib/src/vespa/vespalib/util/count_down_latch.h index d543d773909..613a60e90c5 100644 --- a/vespalib/src/vespa/vespalib/util/count_down_latch.h +++ b/vespalib/src/vespa/vespalib/util/count_down_latch.h @@ -22,7 +22,7 @@ namespace vespalib { class CountDownLatch { private: - std::mutex _lock; + mutable std::mutex _lock; std::condition_variable _cond; uint32_t _count; @@ -44,7 +44,7 @@ public: * blocked in the await method will be unblocked. **/ void countDown() { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); if (_count != 0) { --_count; if (_count == 0) { @@ -59,7 +59,7 @@ public: * reduce the count to 0. **/ void await() { - std::unique_lock<std::mutex> guard(_lock); + std::unique_lock guard(_lock); _cond.wait(guard, [this]() { return (_count == 0); }); } @@ -72,7 +72,7 @@ public: * @return true if the counter reached 0, false if we timed out **/ bool await(vespalib::duration maxwait) { - std::unique_lock<std::mutex> guard(_lock); + std::unique_lock guard(_lock); return _cond.wait_for(guard, maxwait, [this]() { return (_count == 0); }); } @@ -82,7 +82,10 @@ public: * * @return current count **/ - uint32_t getCount() const { return _count; } + [[nodiscard]] uint32_t getCount() const noexcept { + std::lock_guard guard(_lock); + return _count; + } /** * Empty. Needs to be virtual to reduce compiler warnings. diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp index 4eee0a63870..97dbedad66f 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp @@ -3,6 +3,10 @@ #include "cpu_usage.h" #include "require.h" #include <pthread.h> +#include <optional> +#include <cassert> + +#include <sys/resource.h> namespace vespalib { @@ -13,11 +17,11 @@ namespace { class DummyThreadSampler : public ThreadSampler { private: steady_time _start; - double _load; + double _util; public: - DummyThreadSampler(double load) : _start(steady_clock::now()), _load(load) {} - duration sample() const override { - return from_s(to_s(steady_clock::now() - _start) * _load); + DummyThreadSampler(double util) : _start(steady_clock::now()), _util(util) {} + duration sample() const noexcept override { + return from_s(to_s(steady_clock::now() - _start) * _util); } }; @@ -30,9 +34,10 @@ public: LinuxThreadSampler() : _my_clock() { REQUIRE_EQ(pthread_getcpuclockid(pthread_self(), &_my_clock), 0); } - duration sample() const override { + duration sample() const noexcept override { timespec ts; - REQUIRE_EQ(clock_gettime(_my_clock, &ts), 0); + memset(&ts, 0, sizeof(ts)); + clock_gettime(_my_clock, &ts); return from_timespec(ts); } }; @@ -41,16 +46,246 @@ public: } // <unnamed> -ThreadSampler::UP create_thread_sampler(bool force_mock_impl, double expected_load) { +duration total_cpu_usage() noexcept { + timespec ts; + memset(&ts, 0, sizeof(ts)); + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + return from_timespec(ts); +} + +ThreadSampler::UP create_thread_sampler(bool force_mock_impl, double expected_util) { if (force_mock_impl) { - return std::make_unique<DummyThreadSampler>(expected_load); + return std::make_unique<DummyThreadSampler>(expected_util); } #ifdef __linux__ return std::make_unique<LinuxThreadSampler>(); #endif - return std::make_unique<DummyThreadSampler>(expected_load); + return std::make_unique<DummyThreadSampler>(expected_util); } } // cpu_usage +CpuUsage::ThreadTrackerImpl::ThreadTrackerImpl(cpu_usage::ThreadSampler::UP sampler) + : _lock(), + _cat(Category::OTHER), + _old_usage(), + _sampler(std::move(sampler)), + _pending() +{ +} + +CpuUsage::Category +CpuUsage::ThreadTrackerImpl::set_category(Category new_cat) noexcept +{ + // only owning thread may change category + if (new_cat == _cat) { + return new_cat; + } + Guard guard(_lock); + duration new_usage = _sampler->sample(); + if (_cat != Category::OTHER) { + _pending[_cat] += (new_usage - _old_usage); + } + _old_usage = new_usage; + auto old_cat = _cat; + _cat = new_cat; + return old_cat; +} + +CpuUsage::Sample +CpuUsage::ThreadTrackerImpl::sample() noexcept +{ + Guard guard(_lock); + if (_cat != Category::OTHER) { + duration new_usage = _sampler->sample(); + _pending[_cat] += (new_usage - _old_usage); + _old_usage = new_usage; + } + Sample sample = _pending; + _pending = Sample(); + return sample; +} + +vespalib::string & +CpuUsage::name_of(Category cat) +{ + static std::array<vespalib::string,num_categories> names = {"setup", "read", "write", "compact", "other"}; + return names[index_of(cat)]; +} + +CpuUsage::Category +CpuUsage::MyUsage::set_cpu_category_for_this_thread(Category cat) noexcept +{ + struct Wrapper { + std::shared_ptr<ThreadTrackerImpl> self; + Wrapper() : self(std::make_shared<ThreadTrackerImpl>(cpu_usage::create_thread_sampler())) { + CpuUsage::self().add_thread(self); + } + ~Wrapper() { + self->set_category(CpuUsage::Category::OTHER); + CpuUsage::self().remove_thread(std::move(self)); + } + }; + thread_local Wrapper wrapper; + return wrapper.self->set_category(cat); +} + +CpuUsage::CpuUsage() + : _lock(), + _usage(), + _threads(), + _sampling(false), + _conflict(), + _pending_add(), + _pending_remove() +{ +} + +CpuUsage & +CpuUsage::self() +{ + static CpuUsage me; + return me; +} + +void +CpuUsage::do_add_thread(const Guard &, ThreadTracker::SP tracker) +{ + assert(!_sampling); + auto *key = tracker.get(); + auto [ignore, was_inserted] = _threads.emplace(key, std::move(tracker)); + assert(was_inserted); +} + +void +CpuUsage::do_remove_thread(const Guard &, ThreadTracker::SP tracker) +{ + assert(!_sampling); + _usage.merge(tracker->sample()); + auto was_removed = _threads.erase(tracker.get()); + assert(was_removed); +} + +void +CpuUsage::add_thread(ThreadTracker::SP tracker) +{ + Guard guard(_lock); + if (_sampling) { + _pending_add.push_back(std::move(tracker)); + } else { + do_add_thread(guard, std::move(tracker)); + } +} + +void +CpuUsage::remove_thread(ThreadTracker::SP tracker) +{ + Guard guard(_lock); + if (_sampling) { + _pending_remove.push_back(std::move(tracker)); + } else { + do_remove_thread(guard, std::move(tracker)); + } +} + +void +CpuUsage::handle_pending(const Guard &guard) +{ + for (auto &thread: _pending_add) { + do_add_thread(guard, std::move(thread)); + } + _pending_add.clear(); + for (auto &thread: _pending_remove) { + do_remove_thread(guard, std::move(thread)); + } + _pending_remove.clear(); +} + +CpuUsage::TimedSample +CpuUsage::do_sample() +{ + assert(_sampling); + Sample my_sample; + std::optional<std::promise<TimedSample>> my_promise; + auto t = steady_clock::now(); + for (const auto &entry: _threads) { + my_sample.merge(entry.first->sample()); + } + { + Guard guard(_lock); + _sampling = false; + handle_pending(guard); + if (_conflict) { + my_promise = std::move(_conflict->sample_promise); + _conflict.reset(); + } + my_sample.merge(_usage); + _usage = my_sample; + } + auto total = cpu_usage::total_cpu_usage(); + for (size_t i = 0; i < index_of(Category::OTHER); ++i) { + total -= my_sample[i]; + } + my_sample[Category::OTHER] = std::max(total, duration::zero()); + TimedSample result{t, my_sample}; + if (my_promise.has_value()) { + my_promise.value().set_value(result); + } + return result; +} + +CpuUsage::TimedSample +CpuUsage::sample_or_wait() +{ + std::shared_future<TimedSample> my_future; + { + Guard guard(_lock); + if (_sampling) { + if (!_conflict) { + _conflict = std::make_unique<SampleConflict>(); + } + my_future = _conflict->future_sample; + _conflict->waiters++; + } else { + _sampling = true; + } + } + if (my_future.valid()) { + return my_future.get(); + } else { + return do_sample(); + } +} + +CpuUsage::TimedSample +CpuUsage::sample() +{ + return self().sample_or_wait(); +} + +Runnable::init_fun_t +CpuUsage::wrap(Runnable::init_fun_t init, Category cat) +{ + return [init,cat](Runnable &target) { + auto my_usage = CpuUsage::use(cat); + return init(target); + }; +} + +Executor::Task::UP +CpuUsage::wrap(Executor::Task::UP task, Category cat) +{ + struct CpuTask : Executor::Task { + UP task; + Category cat; + CpuTask(UP task_in, Category cat_in) + : task(std::move(task_in)), cat(cat_in) {} + void run() override { + auto my_usage = CpuUsage::use(cat); + task->run(); + } + }; + return std::make_unique<CpuTask>(std::move(task), cat); +} + } // namespace diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.h b/vespalib/src/vespa/vespalib/util/cpu_usage.h index 09509a984b5..31309d88f5f 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.h +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.h @@ -1,25 +1,218 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include "runnable.h" +#include "executor.h" +#include "spin_lock.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/stllike/string.h> +#include <array> #include <memory> +#include <future> +#include <vector> +#include <map> namespace vespalib { namespace cpu_usage { /** + * Samples the total CPU usage of this process so far. + **/ +duration total_cpu_usage() noexcept; + +/** * Samples the total CPU usage of the thread that created it. Note * that this must not be used after thread termination. Enables * sampling the CPU usage of a thread from outside the thread. **/ struct ThreadSampler { using UP = std::unique_ptr<ThreadSampler>; - virtual duration sample() const = 0; + virtual duration sample() const noexcept = 0; virtual ~ThreadSampler() {} }; -ThreadSampler::UP create_thread_sampler(bool force_mock_impl = false, double expected_load = 0.16); +ThreadSampler::UP create_thread_sampler(bool force_mock_impl = false, double expected_util = 0.16); } // cpu_usage +/** + * Tracks accumulative cpu usage across threads and work + * categories. Use the 'use' function to signal what kind of CPU you + * are using in the current thread. Use the 'sample' function to get a + * complete view of CPU usage so far. + * + * The 'use' function returns a MyUsage object that needs to be kept + * alive for as long as the current thread should contribute to the + * specified cpu use. Note that MyUsage instances may shadow each + * other, but must be destructed in reverse construction order. + **/ +class CpuUsage +{ +public: + // The kind of work performed by a thread. Used to separate + // different kinds of CPU usage. Note that categories are + // exclusive/non-overlapping; a thread can only perform one kind + // of CPU work at any specific time. + enum class Category { + SETUP = 0, // usage related to system setup (init/(re-)config/etc.) + READ = 1, // usage related to reading data from the system + WRITE = 2, // usage related to writing data to the system + COMPACT = 3, // usage related to internal data re-structuring + OTHER = 4 // all other cpu usage not in the categories above + }; + static vespalib::string &name_of(Category cat); + static constexpr size_t index_of(Category cat) { return static_cast<size_t>(cat); } + static constexpr size_t num_categories = 5; + + template <typename T> + class PerCategory { + private: + std::array<T,num_categories> _array; + public: + PerCategory() : _array() {} + size_t size() const { return _array.size(); } + T &operator[](size_t idx) { return _array[idx]; } + T &operator[](Category cat) { return _array[index_of(cat)]; } + const T &operator[](size_t idx) const { return _array[idx]; } + const T &operator[](Category cat) const { return _array[index_of(cat)]; } + }; + + // A sample contains how much CPU has been spent in each category. + class Sample : public PerCategory<duration> { + public: + void merge(const Sample &rhs) { + for (size_t i = 0; i < size(); ++i) { + (*this)[i] += rhs[i]; + } + } + }; + + // a sample tagged with the time it was taken + using TimedSample = std::pair<steady_time, Sample>; + + // Used by threads to signal what kind of CPU they are currently + // using. The thread will contribute to the declared CPU usage + // category while this object lives. MyUsage instances may shadow + // each other, but must be destructed in reverse construction + // order. The preferred way to use this class is by doing: + // + // auto my_usage = CpuUsage::use(my_cat); + class MyUsage { + private: + Category _old_cat; + static Category set_cpu_category_for_this_thread(Category cat) noexcept; + public: + MyUsage(Category cat) + : _old_cat(set_cpu_category_for_this_thread(cat)) {} + MyUsage(MyUsage &&) = delete; + MyUsage(const MyUsage &) = delete; + MyUsage &operator=(MyUsage &&) = delete; + MyUsage &operator=(const MyUsage &) = delete; + ~MyUsage() { set_cpu_category_for_this_thread(_old_cat); } + }; + + // grant extra access for testing + struct Test; + +private: + using Guard = std::lock_guard<SpinLock>; + + // Used when multiple threads call the 'sample' function at the + // same time. One thread will sample while the others will wait + // for the result. + struct SampleConflict { + std::promise<TimedSample> sample_promise; + std::shared_future<TimedSample> future_sample; + size_t waiters; + SampleConflict() : sample_promise(), + future_sample(sample_promise.get_future()), + waiters(0) {} + }; + + // Interface used to perform destructive sampling of the CPU spent + // in various categories since the last time it was sampled. + struct ThreadTracker { + using SP = std::shared_ptr<ThreadTracker>; + virtual Sample sample() noexcept = 0; + virtual ~ThreadTracker() {} + }; + + class ThreadTrackerImpl : public ThreadTracker { + private: + SpinLock _lock; + Category _cat; + duration _old_usage; + cpu_usage::ThreadSampler::UP _sampler; + Sample _pending; + + public: + ThreadTrackerImpl(cpu_usage::ThreadSampler::UP sampler); + // only called by owning thread + Category set_category(Category new_cat) noexcept; + Sample sample() noexcept override; + }; + + SpinLock _lock; + Sample _usage; + std::map<ThreadTracker*,ThreadTracker::SP> _threads; + bool _sampling; + std::unique_ptr<SampleConflict> _conflict; + std::vector<ThreadTracker::SP> _pending_add; + std::vector<ThreadTracker::SP> _pending_remove; + + CpuUsage(); + CpuUsage(CpuUsage &&) = delete; + CpuUsage(const CpuUsage &) = delete; + CpuUsage &operator=(CpuUsage &&) = delete; + CpuUsage &operator=(const CpuUsage &) = delete; + + static CpuUsage &self(); + + void do_add_thread(const Guard &guard, ThreadTracker::SP tracker); + void do_remove_thread(const Guard &guard, ThreadTracker::SP tracker); + + void add_thread(ThreadTracker::SP tracker); + void remove_thread(ThreadTracker::SP tracker); + + void handle_pending(const Guard &guard); + TimedSample do_sample(); + TimedSample sample_or_wait(); + +public: + static MyUsage use(Category cat) { return MyUsage(cat); } + static TimedSample sample(); + static Runnable::init_fun_t wrap(Runnable::init_fun_t init, Category cat); + static Executor::Task::UP wrap(Executor::Task::UP task, Category cat); +}; + +/** + * Simple class used to track cpu utilization over time. + **/ +class CpuUtil +{ +private: + duration _min_delay; + CpuUsage::TimedSample _old_sample; + CpuUsage::PerCategory<double> _util; + +public: + CpuUtil(duration min_delay = 850ms) + : _min_delay(min_delay), + _old_sample(CpuUsage::sample()), + _util() {} + + CpuUsage::PerCategory<double> get_util() { + if (steady_clock::now() >= (_old_sample.first + _min_delay)) { + auto new_sample = CpuUsage::sample(); + auto dt = to_s(new_sample.first - _old_sample.first); + for (size_t i = 0; i < _util.size(); ++i) { + _util[i] = to_s(new_sample.second[i] - _old_sample.second[i]) / dt; + } + _old_sample = new_sample; + } + return _util; + } +}; + } // namespace diff --git a/vespalib/src/vespa/vespalib/util/nice.cpp b/vespalib/src/vespa/vespalib/util/nice.cpp new file mode 100644 index 00000000000..cefaaa0347b --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/nice.cpp @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nice.h" + +#include <unistd.h> +#include <algorithm> + +namespace vespalib { + +namespace { + +void set_nice_value(double how_nice) { + if (how_nice > 0.0) { +#ifndef __APPLE__ + int now = nice(0); + int max = 19; + int max_inc = (max - now); + nice(std::min(max_inc, int(how_nice * (max_inc + 1)))); +#endif + } +} + +} + +Runnable::init_fun_t be_nice(Runnable::init_fun_t init, double how_nice) { + return [init,how_nice](Runnable &target) { + set_nice_value(how_nice); + return init(target); + }; +} + +} // namespace diff --git a/vespalib/src/vespa/vespalib/util/nice.h b/vespalib/src/vespa/vespalib/util/nice.h new file mode 100644 index 00000000000..7c3873337eb --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/nice.h @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "runnable.h" + +namespace vespalib { + +// Wraps an init function inside another init function that adjusts +// the niceness of the thread being started. The 'how_nice' parameter +// is a value from 0.0 (not nice at all) to 1.0 (super nice). It will +// be mapped into an actual nice value in a linear fashion based on +// the nice value space that is still available. + +Runnable::init_fun_t be_nice(Runnable::init_fun_t init, double how_nice); + +} diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.h b/vespalib/src/vespa/vespalib/util/rcuvector.h index dd4fa660279..00d050fa8d1 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.h +++ b/vespalib/src/vespa/vespalib/util/rcuvector.h @@ -122,6 +122,7 @@ public: void reset(); void shrink(size_t newSize) __attribute__((noinline)); void replaceVector(ArrayType replacement); + ArrayType create_replacement_vector() const { return _data.create(); } }; template <typename T> diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.hpp b/vespalib/src/vespa/vespalib/util/rcuvector.hpp index 3c455149dfd..7c70539da0e 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.hpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.hpp @@ -42,7 +42,7 @@ template <typename T> void RcuVectorBase<T>::reset() { // Assumes no readers at this moment - ArrayType().swap(_data); + _data.reset(); _data.reserve(16); } @@ -52,7 +52,7 @@ RcuVectorBase<T>::~RcuVectorBase() = default; template <typename T> void RcuVectorBase<T>::expand(size_t newCapacity) { - ArrayType tmpData; + auto tmpData = create_replacement_vector(); tmpData.reserve(newCapacity); for (const T & v : _data) { tmpData.push_back_fast(v); @@ -91,7 +91,7 @@ RcuVectorBase<T>::shrink(size_t newSize) return; } if (!_data.try_unreserve(wantedCapacity)) { - ArrayType tmpData; + auto tmpData = create_replacement_vector(); tmpData.reserve(wantedCapacity); tmpData.resize(newSize); for (uint32_t i = 0; i < newSize; ++i) { diff --git a/vespalib/src/vespa/vespalib/util/runnable.cpp b/vespalib/src/vespa/vespalib/util/runnable.cpp index c67c4696d88..3b75efe93f6 100644 --- a/vespalib/src/vespa/vespalib/util/runnable.cpp +++ b/vespalib/src/vespa/vespalib/util/runnable.cpp @@ -4,4 +4,11 @@ namespace vespalib { +int +Runnable::default_init_function(Runnable &target) +{ + target.run(); + return 1; +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/runnable.h b/vespalib/src/vespa/vespalib/util/runnable.h index 43144ebc2cd..bc89a9decac 100644 --- a/vespalib/src/vespa/vespalib/util/runnable.h +++ b/vespalib/src/vespa/vespalib/util/runnable.h @@ -23,6 +23,7 @@ namespace vespalib { struct Runnable { using UP = std::unique_ptr<Runnable>; using init_fun_t = std::function<int(Runnable&)>; + static int default_init_function(Runnable &target); /** * Entry point called by the running thread @@ -36,4 +37,3 @@ struct Runnable { }; } // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp new file mode 100644 index 00000000000..f5a1c117cc8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp @@ -0,0 +1,434 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "shared_operation_throttler.h" +#include <atomic> +#include <condition_variable> +#include <cassert> +#include <functional> +#include <mutex> + +namespace vespalib { + +namespace { + +class NoLimitsOperationThrottler final : public SharedOperationThrottler { +public: + NoLimitsOperationThrottler() + : SharedOperationThrottler(), + _refs(0u) + { + } + ~NoLimitsOperationThrottler() override { + assert(_refs.load(std::memory_order_acquire) == 0u); + } + Token blocking_acquire_one() noexcept override { + ++_refs; + return Token(this, TokenCtorTag{}); + } + Token blocking_acquire_one(vespalib::duration) noexcept override { + ++_refs; + return Token(this, TokenCtorTag{}); + } + Token try_acquire_one() noexcept override { + ++_refs; + return Token(this, TokenCtorTag{}); + } + uint32_t current_window_size() const noexcept override { return 0; } + uint32_t waiting_threads() const noexcept override { return 0; } + void reconfigure_dynamic_throttling(const DynamicThrottleParams&) noexcept override { /* no-op */ } +private: + void release_one() noexcept override { --_refs; } + std::atomic<uint32_t> _refs; +}; + +/** + * Effectively a 1-1 transplant of the MessageBus DynamicThrottlePolicy, but + * without an underlying StaticThrottlePolicy and with no need for individual + * MessageBus Message/Reply objects. + * + * Please keep the underlying algorithm in sync with the Java implementation, + * as that is considered the source of truth. For descriptions of the various + * parameters, also see the Java code: + * messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java + */ +class DynamicThrottlePolicy { + SharedOperationThrottler::DynamicThrottleParams _active_params; + std::function<steady_time()> _time_provider; + uint32_t _num_sent; + uint32_t _num_ok; + double _resize_rate; + uint64_t _resize_time; + uint64_t _time_of_last_message; + uint64_t _idle_time_period; + double _efficiency_threshold; + double _window_size_increment; + double _window_size; + double _max_window_size; + double _min_window_size; + double _decrement_factor; + double _window_size_backoff; + double _weight; + double _local_max_throughput; +public: + DynamicThrottlePolicy(const SharedOperationThrottler::DynamicThrottleParams& params, + std::function<steady_time()> time_provider); + + void set_window_size_increment(double window_size_increment) noexcept; + void set_window_size_backoff(double window_size_backoff) noexcept; + void set_resize_rate(double resize_rate) noexcept; + void set_max_window_size(double max_size) noexcept; + + void set_min_window_size(double min_size) noexcept; + void set_window_size_decrement_factor(double decrement_factor) noexcept; + + void configure(const SharedOperationThrottler::DynamicThrottleParams& params) noexcept; + + [[nodiscard]] uint32_t current_window_size() const noexcept { + return static_cast<uint32_t>(_window_size); + } + [[nodiscard]] bool has_spare_capacity(uint32_t pending_count) noexcept; + void process_request() noexcept; + void process_response(bool success) noexcept; + +private: + void internal_unconditional_configure(const SharedOperationThrottler::DynamicThrottleParams& params) noexcept; + + [[nodiscard]] uint64_t current_time_as_millis() noexcept { + return count_ms(_time_provider().time_since_epoch()); + } +}; + +DynamicThrottlePolicy::DynamicThrottlePolicy(const SharedOperationThrottler::DynamicThrottleParams& params, + std::function<steady_time()> time_provider) + : _active_params(params), + _time_provider(std::move(time_provider)), + _num_sent(0), + _num_ok(0), + _resize_rate(3.0), + _resize_time(0), + _time_of_last_message(current_time_as_millis()), + _idle_time_period(60000), + _efficiency_threshold(1), + _window_size_increment(20), + _window_size(_window_size_increment), + _max_window_size(INT_MAX), + _min_window_size(_window_size_increment), + _decrement_factor(2.0), + _window_size_backoff(0.9), + _weight(1), + _local_max_throughput(0) +{ + internal_unconditional_configure(_active_params); +} + +void +DynamicThrottlePolicy::set_window_size_increment(double window_size_increment) noexcept +{ + _window_size_increment = window_size_increment; + _window_size = std::max(_window_size, _window_size_increment); +} + +void +DynamicThrottlePolicy::set_window_size_backoff(double window_size_backoff) noexcept +{ + _window_size_backoff = std::max(0.0, std::min(1.0, window_size_backoff)); +} + +void +DynamicThrottlePolicy::set_resize_rate(double resize_rate) noexcept +{ + _resize_rate = std::max(2.0, resize_rate); +} + +void +DynamicThrottlePolicy::set_max_window_size(double max_size) noexcept +{ + _max_window_size = max_size; +} + +void +DynamicThrottlePolicy::set_min_window_size(double min_size) noexcept +{ + _min_window_size = min_size; + _window_size = std::max(_min_window_size, _window_size_increment); +} + +void +DynamicThrottlePolicy::set_window_size_decrement_factor(double decrement_factor) noexcept +{ + _decrement_factor = decrement_factor; +} + +void +DynamicThrottlePolicy::internal_unconditional_configure(const SharedOperationThrottler::DynamicThrottleParams& params) noexcept +{ + // We use setters for convenience, since setting one parameter may imply setting others, + // based on it, and there's frequently min/max capping of values. + set_window_size_increment(params.window_size_increment); + set_min_window_size(params.min_window_size); + set_max_window_size(params.max_window_size); + set_resize_rate(params.resize_rate); + set_window_size_decrement_factor(params.window_size_decrement_factor); + set_window_size_backoff(params.window_size_backoff); +} + +void +DynamicThrottlePolicy::configure(const SharedOperationThrottler::DynamicThrottleParams& params) noexcept +{ + // To avoid any noise where setting parameters on the throttler may implicitly reduce the + // current window size (even though this isn't _currently_ the case), don't invoke any internal + // reconfiguration code unless the parameters have actually changed. + if (params != _active_params) { + internal_unconditional_configure(params); + _active_params = params; + } +} + +bool +DynamicThrottlePolicy::has_spare_capacity(uint32_t pending_count) noexcept +{ + const uint64_t time = current_time_as_millis(); + if ((time - _time_of_last_message) > _idle_time_period) { + _window_size = std::max(_min_window_size, std::min(_window_size, pending_count + _window_size_increment)); + } + _time_of_last_message = time; + const auto window_size_floored = static_cast<uint32_t>(_window_size); + // Use floating point window sizes, so the algorithm sees the difference between 1.1 and 1.9 window size. + const bool carry = _num_sent < ((_window_size * _resize_rate) * (_window_size - window_size_floored)); + return pending_count < (window_size_floored + (carry ? 1 : 0)); +} + +void +DynamicThrottlePolicy::process_request() noexcept +{ + if (++_num_sent < (_window_size * _resize_rate)) { + return; + } + + const uint64_t time = current_time_as_millis(); + const double elapsed = time - _resize_time; + _resize_time = time; + + const double throughput = _num_ok / elapsed; + _num_sent = 0; + _num_ok = 0; + + if (throughput > _local_max_throughput) { + _local_max_throughput = throughput; + _window_size += _weight * _window_size_increment; + } else { + // scale up/down throughput for comparing to window size + double period = 1; + while ((throughput * (period / _window_size)) < 2) { + period *= 10; + } + while ((throughput * (period / _window_size)) > 2) { + period *= 0.1; + } + const double efficiency = throughput * (period / _window_size); + + if (efficiency < _efficiency_threshold) { + _window_size = std::min(_window_size * _window_size_backoff, + _window_size - _decrement_factor * _window_size_increment); + _local_max_throughput = 0; + } else { + _window_size += _weight * _window_size_increment; + } + } + _window_size = std::max(_min_window_size, _window_size); + _window_size = std::min(_max_window_size, _window_size); +} + +void +DynamicThrottlePolicy::process_response(bool success) noexcept +{ + if (success) { + ++_num_ok; + } +} + +class DynamicOperationThrottler final : public SharedOperationThrottler { + mutable std::mutex _mutex; + std::condition_variable _cond; + DynamicThrottlePolicy _throttle_policy; + uint32_t _pending_ops; + uint32_t _waiting_threads; +public: + DynamicOperationThrottler(const DynamicThrottleParams& params, + std::function<steady_time()> time_provider); + ~DynamicOperationThrottler() override; + + Token blocking_acquire_one() noexcept override; + Token blocking_acquire_one(vespalib::duration timeout) noexcept override; + Token try_acquire_one() noexcept override; + uint32_t current_window_size() const noexcept override; + uint32_t waiting_threads() const noexcept override; + void reconfigure_dynamic_throttling(const DynamicThrottleParams& params) noexcept override; +private: + void release_one() noexcept override; + // Non-const since actually checking the send window of a dynamic throttler might change + // it if enough time has passed. + [[nodiscard]] bool has_spare_capacity_in_active_window() noexcept; + void add_one_to_active_window_size() noexcept; + void subtract_one_from_active_window_size() noexcept; +}; + +DynamicOperationThrottler::DynamicOperationThrottler(const DynamicThrottleParams& params, + std::function<steady_time()> time_provider) + : _mutex(), + _cond(), + _throttle_policy(params, std::move(time_provider)), + _pending_ops(0), + _waiting_threads(0) +{ +} + +DynamicOperationThrottler::~DynamicOperationThrottler() +{ + assert(_pending_ops == 0u); +} + +bool +DynamicOperationThrottler::has_spare_capacity_in_active_window() noexcept +{ + return _throttle_policy.has_spare_capacity(_pending_ops); +} + +void +DynamicOperationThrottler::add_one_to_active_window_size() noexcept +{ + _throttle_policy.process_request(); + ++_pending_ops; +} + +void +DynamicOperationThrottler::subtract_one_from_active_window_size() noexcept +{ + _throttle_policy.process_response(true); // TODO support failure push-back + assert(_pending_ops > 0); + --_pending_ops; +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::blocking_acquire_one() noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + ++_waiting_threads; + _cond.wait(lock, [&] { + return has_spare_capacity_in_active_window(); + }); + --_waiting_threads; + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::blocking_acquire_one(vespalib::duration timeout) noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + ++_waiting_threads; + const bool accepted = _cond.wait_for(lock, timeout, [&] { + return has_spare_capacity_in_active_window(); + }); + --_waiting_threads; + if (!accepted) { + return Token(); + } + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::try_acquire_one() noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + return Token(); + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +void +DynamicOperationThrottler::release_one() noexcept +{ + std::unique_lock lock(_mutex); + subtract_one_from_active_window_size(); + // Only wake up a waiting thread if doing so would possibly result in success. + if ((_waiting_threads > 0) && has_spare_capacity_in_active_window()) { + lock.unlock(); + _cond.notify_one(); + } +} + +uint32_t +DynamicOperationThrottler::current_window_size() const noexcept +{ + std::unique_lock lock(_mutex); + return _throttle_policy.current_window_size(); +} + +uint32_t +DynamicOperationThrottler::waiting_threads() const noexcept +{ + std::unique_lock lock(_mutex); + return _waiting_threads; +} + +void +DynamicOperationThrottler::reconfigure_dynamic_throttling(const DynamicThrottleParams& params) noexcept +{ + std::unique_lock lock(_mutex); + _throttle_policy.configure(params); +} + +} // anonymous namespace + +std::unique_ptr<SharedOperationThrottler> +SharedOperationThrottler::make_unlimited_throttler() +{ + return std::make_unique<NoLimitsOperationThrottler>(); +} + +std::unique_ptr<SharedOperationThrottler> +SharedOperationThrottler::make_dynamic_throttler(const DynamicThrottleParams& params) +{ + return std::make_unique<DynamicOperationThrottler>(params, []() noexcept { return steady_clock::now(); }); +} + +std::unique_ptr<SharedOperationThrottler> +SharedOperationThrottler::make_dynamic_throttler(const DynamicThrottleParams& params, + std::function<steady_time()> time_provider) +{ + return std::make_unique<DynamicOperationThrottler>(params, std::move(time_provider)); +} + +DynamicOperationThrottler::Token::~Token() +{ + if (_throttler) { + _throttler->release_one(); + } +} + +void +DynamicOperationThrottler::Token::reset() noexcept +{ + if (_throttler) { + _throttler->release_one(); + _throttler = nullptr; + } +} + +DynamicOperationThrottler::Token& +DynamicOperationThrottler::Token::operator=(Token&& rhs) noexcept +{ + reset(); + _throttler = rhs._throttler; + rhs._throttler = nullptr; + return *this; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h new file mode 100644 index 00000000000..030935339ed --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "time.h" +#include <functional> +#include <memory> +#include <optional> +#include <limits.h> + +namespace vespalib { + +/** + * Operation throttler that is intended to provide global throttling of + * async operations across multiple threads. A throttler wraps a logical + * max pending window size of in-flight operations. Depending on the + * throttler implementation, the window size may expand and shrink dynamically. + * Exactly how and when this happens is unspecified. + * + * Offers both polling and (timed, non-timed) blocking calls for acquiring + * a throttle token. If the returned token is valid, the caller may proceed + * to invoke the asynchronous operation. + * + * The window slot taken up by a valid throttle token is implicitly freed up + * when the token is destroyed. + * + * All operations on the throttler are thread safe. + */ +class SharedOperationThrottler { +protected: + struct TokenCtorTag {}; // Make available to subclasses for token construction. +public: + class Token { + SharedOperationThrottler* _throttler; + public: + constexpr Token(SharedOperationThrottler* throttler, TokenCtorTag) noexcept : _throttler(throttler) {} + constexpr Token() noexcept : _throttler(nullptr) {} + constexpr Token(Token&& rhs) noexcept + : _throttler(rhs._throttler) + { + rhs._throttler = nullptr; + } + Token& operator=(Token&& rhs) noexcept; + ~Token(); + + Token(const Token&) = delete; + Token& operator=(const Token&) = delete; + + [[nodiscard]] constexpr bool valid() const noexcept { return (_throttler != nullptr); } + void reset() noexcept; + }; + + virtual ~SharedOperationThrottler() = default; + + // Acquire a valid throttling token, uninterruptedly blocking until one can be obtained. + [[nodiscard]] virtual Token blocking_acquire_one() noexcept = 0; + // Attempt to acquire a valid throttling token, waiting up to `timeout` for one to be + // available. If the timeout is exceeded without any tokens becoming available, an + // invalid token will be returned. + [[nodiscard]] virtual Token blocking_acquire_one(vespalib::duration timeout) noexcept = 0; + // Attempt to acquire a valid throttling token if one is immediately available. + // An invalid token will be returned if none is available. Never blocks (other than + // when contending for the internal throttler mutex). + [[nodiscard]] virtual Token try_acquire_one() noexcept = 0; + + // May return 0, in which case the window size is unlimited. + [[nodiscard]] virtual uint32_t current_window_size() const noexcept = 0; + + // Exposed for unit testing only. + [[nodiscard]] virtual uint32_t waiting_threads() const noexcept = 0; + + struct DynamicThrottleParams { + uint32_t window_size_increment = 20; + uint32_t min_window_size = 20; + uint32_t max_window_size = INT_MAX; // signed max to be 1-1 compatible with Java defaults + double resize_rate = 3.0; + double window_size_decrement_factor = 1.2; + double window_size_backoff = 0.95; + + bool operator==(const DynamicThrottleParams&) const noexcept = default; + bool operator!=(const DynamicThrottleParams&) const noexcept = default; + }; + + // No-op if underlying throttler does not use a dynamic policy, or if the supplied + // parameters are equal to the current configuration. + // FIXME leaky abstraction alert! + virtual void reconfigure_dynamic_throttling(const DynamicThrottleParams& params) noexcept = 0; + + // Creates a throttler that does exactly zero throttling (but also has zero overhead and locking) + static std::unique_ptr<SharedOperationThrottler> make_unlimited_throttler(); + + // Creates a throttler that uses a DynamicThrottlePolicy under the hood + static std::unique_ptr<SharedOperationThrottler> make_dynamic_throttler(const DynamicThrottleParams& params); + static std::unique_ptr<SharedOperationThrottler> make_dynamic_throttler(const DynamicThrottleParams& params, + std::function<steady_time()> time_provider); +private: + // Exclusively called from a valid Token. Thread safe. + virtual void release_one() noexcept = 0; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.h b/vespalib/src/vespa/vespalib/util/shared_string_repo.h index ec65b942d88..7ba59937c7c 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.h +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.h @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/identity.h> +#include <vespa/vespalib/stllike/allocator.h> #include <vespa/vespalib/stllike/hashtable.hpp> #include <xxhash.h> #include <mutex> @@ -95,6 +96,7 @@ private: return (--_ref_cnt == 0); } }; + using EntryVector = std::vector<Entry, allocator_large<Entry>>; struct Key { uint32_t idx; uint32_t hash; @@ -104,8 +106,8 @@ private: uint32_t operator()(const AltKey &key) const { return key.hash; } }; struct Equal { - const std::vector<Entry> &entries; - Equal(const std::vector<Entry> &entries_in) : entries(entries_in) {} + const EntryVector &entries; + Equal(const EntryVector &entries_in) : entries(entries_in) {} Equal(const Equal &rhs) = default; bool operator()(const Key &a, const Key &b) const { return (a.idx == b.idx); } bool operator()(const Key &a, const AltKey &b) const { return ((a.hash == b.hash) && (entries[a.idx].str() == b.str)); } @@ -113,10 +115,10 @@ private: using HashType = hashtable<Key,Key,Hash,Equal,Identity,hashtable_base::and_modulator>; private: - mutable SpinLock _lock; - std::vector<Entry> _entries; - uint32_t _free; - HashType _hash; + mutable SpinLock _lock; + EntryVector _entries; + uint32_t _free; + HashType _hash; void make_entries(size_t hint); @@ -291,7 +293,7 @@ public: // A collection of string handles with ownership class Handles { private: - std::vector<string_id> _handles; + StringIdVector _handles; public: Handles(); Handles(Handles &&rhs); @@ -309,7 +311,7 @@ public: string_id id = _repo.copy(handle); _handles.push_back(id); } - const std::vector<string_id> &view() const { return _handles; } + const StringIdVector &view() const { return _handles; } }; }; diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp index ab83d4e05fd..99ab298864f 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp @@ -60,9 +60,10 @@ Signal::Signal() noexcept {} Signal::~Signal() = default; -SimpleThreadBundle::Pool::Pool(size_t bundleSize) +SimpleThreadBundle::Pool::Pool(size_t bundleSize, init_fun_t init_fun) : _lock(), _bundleSize(bundleSize), + _init_fun(init_fun), _bundles() { } @@ -86,7 +87,7 @@ SimpleThreadBundle::Pool::obtain() return ret; } } - return std::make_unique<SimpleThreadBundle>(_bundleSize); + return std::make_unique<SimpleThreadBundle>(_bundleSize, _init_fun); } void @@ -99,7 +100,7 @@ SimpleThreadBundle::Pool::release(SimpleThreadBundle::UP bundle) //----------------------------------------------------------------------------- -SimpleThreadBundle::SimpleThreadBundle(size_t size_in, Strategy strategy) +SimpleThreadBundle::SimpleThreadBundle(size_t size_in, Runnable::init_fun_t init_fun, Strategy strategy) : _work(), _signals(), _workers(), @@ -134,7 +135,7 @@ SimpleThreadBundle::SimpleThreadBundle(size_t size_in, Strategy strategy) _hook = std::move(hook); } else { size_t signal_idx = (strategy == USE_BROADCAST) ? 0 : (i - 1); - _workers.push_back(std::make_unique<Worker>(_signals[signal_idx], std::move(hook))); + _workers.push_back(std::make_unique<Worker>(_signals[signal_idx], init_fun, std::move(hook))); } } } @@ -175,19 +176,19 @@ SimpleThreadBundle::run(const std::vector<Runnable*> &targets) latch.await(); } -SimpleThreadBundle::Worker::Worker(Signal &s, Runnable::UP h) - : thread(*this, simple_thread_bundle_executor), - signal(s), - hook(std::move(h)) +SimpleThreadBundle::Worker::Worker(Signal &s, Runnable::init_fun_t init_fun, Runnable::UP h) + : thread(*this, std::move(init_fun)), + signal(s), + hook(std::move(h)) { thread.start(); } + void SimpleThreadBundle::Worker::run() { for (size_t gen = 0; signal.wait(gen) > 0; ) { - hook->run(); -} - + hook->run(); + } } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h index d9a29ee7bef..b7434d09ac3 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h @@ -86,8 +86,9 @@ struct Signal { class SimpleThreadBundle : public ThreadBundle { public: - typedef fixed_thread_bundle::Work Work; - typedef fixed_thread_bundle::Signal Signal; + using Work = fixed_thread_bundle::Work; + using Signal = fixed_thread_bundle::Signal; + using init_fun_t = Runnable::init_fun_t; typedef std::unique_ptr<SimpleThreadBundle> UP; enum Strategy { USE_SIGNAL_LIST, USE_SIGNAL_TREE, USE_BROADCAST }; @@ -97,10 +98,12 @@ public: private: std::mutex _lock; size_t _bundleSize; + init_fun_t _init_fun; std::vector<SimpleThreadBundle*> _bundles; public: - Pool(size_t bundleSize); + Pool(size_t bundleSize, init_fun_t init_fun); + Pool(size_t bundleSize) : Pool(bundleSize, Runnable::default_init_function) {} ~Pool(); SimpleThreadBundle::UP obtain(); void release(SimpleThreadBundle::UP bundle); @@ -112,7 +115,7 @@ private: Thread thread; Signal &signal; Runnable::UP hook; - Worker(Signal &s, Runnable::UP h); + Worker(Signal &s, init_fun_t init_fun, Runnable::UP h); void run() override; }; @@ -122,7 +125,9 @@ private: Runnable::UP _hook; public: - SimpleThreadBundle(size_t size, Strategy strategy = USE_SIGNAL_LIST); + SimpleThreadBundle(size_t size, init_fun_t init_fun, Strategy strategy = USE_SIGNAL_LIST); + SimpleThreadBundle(size_t size, Strategy strategy = USE_SIGNAL_LIST) + : SimpleThreadBundle(size, Runnable::default_init_function, strategy) {} ~SimpleThreadBundle(); size_t size() const override; void run(const std::vector<Runnable*> &targets) override; diff --git a/vespalib/src/vespa/vespalib/util/spin_lock.h b/vespalib/src/vespa/vespalib/util/spin_lock.h index abc2b89106f..3af7bc0fd55 100644 --- a/vespalib/src/vespa/vespalib/util/spin_lock.h +++ b/vespalib/src/vespa/vespalib/util/spin_lock.h @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + #include <atomic> #include <thread> diff --git a/vespalib/src/vespa/vespalib/util/string_id.h b/vespalib/src/vespa/vespalib/util/string_id.h index 7a72feee64a..7fec1da0bb8 100644 --- a/vespalib/src/vespa/vespalib/util/string_id.h +++ b/vespalib/src/vespa/vespalib/util/string_id.h @@ -2,7 +2,8 @@ #pragma once -#include <cstdint> +#include <vespa/vespalib/stllike/allocator.h> +#include <vector> namespace vespalib { @@ -38,4 +39,6 @@ public: constexpr bool operator!=(const string_id &rhs) const noexcept { return (_id != rhs._id); } }; +using StringIdVector = std::vector<string_id, vespalib::allocator_large<string_id>>; + } diff --git a/vespalog/pom.xml b/vespalog/pom.xml index 08d73816e63..94be1302e30 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -48,6 +48,18 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <!-- TODO Vespa 8: remove configuration. + Included to allow 'removal' warnings for classes in its own module --> + <configuration> + <compilerArgs> + <arg>-Xlint:all,-serial,-try,-processing,-removal</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkMode>once</forkMode> @@ -59,10 +71,6 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <configuration> <updateReleaseInfo>true</updateReleaseInfo> diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 208cf1588bf..478e5e471fb 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -5,6 +5,7 @@ #include <vespa/log/internal.h> #include <vespa/log/component.h> +#include <optional> #include <unistd.h> #include <dirent.h> #include <sys/stat.h> @@ -14,9 +15,9 @@ LOG_SETUP("vespa-logctl"); using namespace ns_log; -static void modifyLevels(const char *file, char *component, char *levels, +static void modifyLevels(const char *file, const char *component, const char *levels, bool shouldCreateFile, bool shouldCreateEntry); -static void readLevels(const char *file, char *component); +static void readLevels(const char *file, const char *component); static void @@ -148,7 +149,7 @@ main(int argc, char **argv) strlist_t services; char nullComponent[] = "default"; - char *component = nullComponent; + std::string component(nullComponent); if (doAllFiles) { services = findAllFiles(dir); @@ -166,28 +167,26 @@ main(int argc, char **argv) fprintf(stderr, "ERROR: Missing service argument!\n"); return EXIT_FAILURE; } - char *service = strdup(argv[optind]); + std::string service(argv[optind]); ++optind; - char *delim = strchr(service, ':'); - if (delim) { - *delim = 0; - services.push_back(service); - *delim = '.'; - component = delim; + auto delim_pos = service.find(':'); + if (delim_pos != std::string::npos) { + services.push_back(service.substr(0, delim_pos)); + component = '.' + service.substr(delim_pos + 1); } else { services.push_back(service); } } char defLevels[] = "all=on,debug=off,spam=off"; - char *levels = NULL; + std::optional<std::string> levels; if (doResetLevels) { levels = defLevels; } else { if (argc > optind) { - levels = strdup(argv[optind]); + levels = argv[optind]; ++optind; } } @@ -210,10 +209,10 @@ main(int argc, char **argv) // fprintf(stderr, "Log control file %s:\n", file); try { - if (levels) { - modifyLevels(file, component, levels, shouldCreateFile, shouldCreateEntry); + if (levels.has_value()) { + modifyLevels(file, component.c_str(), levels.value().c_str(), shouldCreateFile, shouldCreateEntry); } else { - readLevels(file, component); + readLevels(file, component.c_str()); } hadSuccess = true; } catch (InvalidLogException& x) { @@ -230,7 +229,7 @@ main(int argc, char **argv) } static void -modifyLevels(const char *file, char *componentPattern, char *levels, +modifyLevels(const char *file, const char *componentPattern, const char *levels, bool shouldCreateFile, bool shouldCreateEntry) { ControlFile cf(file, shouldCreateFile @@ -250,7 +249,7 @@ modifyLevels(const char *file, char *componentPattern, char *levels, } static void -readLevels(const char *file, char *componentPattern) +readLevels(const char *file, const char *componentPattern) { ControlFile cf(file, ControlFile::READONLY); Component *c; diff --git a/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java index 58e93c96ec9..b90c63b185e 100644 --- a/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java @@ -1,18 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; import java.util.logging.Level; /** * a levelcontroller that just implements a simple default * (possibly controlled by a system property or environment) - **/ + * @deprecated Should only be used internally in the log library + */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) class DefaultLevelController implements LevelController { private String levelstring; private Level levelLimit = LogLevel.EVENT; DefaultLevelController(String env) { - if (LogUtil.empty(env)) { + if (LogUtils.empty(env)) { env = "all -debug -spam"; } diff --git a/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java index 313fbbe074b..b4752c23b73 100644 --- a/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java @@ -6,7 +6,10 @@ import java.io.*; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public class FileLogTarget implements LogTarget { private final File file; private FileOutputStream fileOutputStream; diff --git a/vespalog/src/main/java/com/yahoo/log/LevelController.java b/vespalog/src/main/java/com/yahoo/log/LevelController.java index 2a549c0dd4f..c710d5c60eb 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelController.java @@ -10,7 +10,9 @@ import java.util.logging.Level; * * @author arnej27959 * + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) public interface LevelController { /** diff --git a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java index 372b3d44e9e..ab70ecf2c96 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java @@ -7,7 +7,10 @@ package com.yahoo.log; * * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public interface LevelControllerRepo { /** * Return the level controller for a given component. diff --git a/vespalog/src/main/java/com/yahoo/log/LogLevel.java b/vespalog/src/main/java/com/yahoo/log/LogLevel.java index 602a0603bef..3129762e60e 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogLevel.java +++ b/vespalog/src/main/java/com/yahoo/log/LogLevel.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; -import java.util.Collections; import java.util.HashMap; -import java.util.Map; import java.util.LinkedHashMap; +import java.util.Map; import java.util.logging.Level; @@ -31,8 +30,11 @@ import java.util.logging.Level; * * @author Bjorn Borud * @author arnej27959 + * + * @deprecated Use {@link java.util.logging.Level} instead. */ - +// TODO Vespa 9: move to non-PublicApi package +@Deprecated(since = "7") public class LogLevel extends Level { /** A map from the name of the log level to the instance */ private static final Map<String, Level> nameToLevel; diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessage.java b/vespalog/src/main/java/com/yahoo/log/LogMessage.java index f48ae97850a..878e041231c 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessage.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessage.java @@ -108,8 +108,9 @@ public class LogMessage if (! m.matches()) { throw new InvalidLogFormatException(msg); } - + @SuppressWarnings("deprecation") Level msgLevel = LogLevel.parse(m.group(6)); + Instant timestamp = parseTimestamp(m.group(1)); String threadProcess = m.group(3); @@ -161,6 +162,7 @@ public class LogMessage * it will return <code>null</code>. * */ + @SuppressWarnings("deprecation") public Event getEvent () throws MalformedEventException { if ((level == LogLevel.EVENT) && (event == null)) { try { @@ -188,7 +190,7 @@ public class LogMessage + component.length() + level.toString().length() + payload.length() - + 1) + + 7) .append(timeStr).append("\t") .append(host).append("\t") .append(threadProcess).append("\t") diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java index 7132dbfbc7f..069a47b1efe 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java @@ -12,8 +12,7 @@ import java.util.Comparator; * This is due to only looking at the timestamp, so two different messages with * the same timestamp would appear "equal" to this comparator. * - * @author vlarsen - * + * @author Vidar Larsen */ public class LogMessageTimeComparator implements Comparator<LogMessage>, Serializable { private static final long serialVersionUID = 1L; diff --git a/vespalog/src/main/java/com/yahoo/log/LogSetup.java b/vespalog/src/main/java/com/yahoo/log/LogSetup.java index 932c993b19e..67fb7f71a2d 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogSetup.java +++ b/vespalog/src/main/java/com/yahoo/log/LogSetup.java @@ -1,12 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Enumeration; -import java.util.logging.*; import java.util.Timer; +import java.util.logging.FileHandler; +import java.util.logging.Filter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.LogRecord; +import java.util.logging.Logger; /** * Sets up Vespa logging. Call a setup method to set up this. @@ -14,10 +22,16 @@ import java.util.Timer; * @author Bjorn Borud * @author arnej27959 */ +@SuppressWarnings("removal") public class LogSetup { private static Timer taskRunner = new Timer(true); - /** A global task thread */ + + /** + * A global task thread + * @deprecated Just construct a java.util.Timer instead + **/ + @Deprecated(since = "7", forRemoval = true) public static Timer getTaskRunner() { return taskRunner; } /** The log handler used by this */ @@ -25,6 +39,7 @@ public class LogSetup { private static ZooKeeperFilter zooKeeperFilter = null; + /** Clear all handlers registered in java.util.logging framework */ public static void clearHandlers () { Enumeration<String> names = LogManager.getLogManager().getLoggerNames(); while (names.hasMoreElements()) { @@ -155,14 +170,18 @@ public class LogSetup { Logger.getLogger("").addHandler(logHandler); } - /** Returns the log handler set up by this class */ + /** + * Returns the log handler set up by this class + * @deprecated Should only be used internally in the log library + */ + @Deprecated(since = "7", forRemoval = true) public static VespaLogHandler getLogHandler() { return logHandler; } - /** Returns the log handler set up by this class */ + /** perform cleanup */ public static void cleanup() { - if (zooKeeperFilter != null) + if (zooKeeperFilter != null) zooKeeperFilter.close(); } diff --git a/vespalog/src/main/java/com/yahoo/log/LogTarget.java b/vespalog/src/main/java/com/yahoo/log/LogTarget.java index 5cc5fd06137..ef5974259c8 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/LogTarget.java @@ -6,7 +6,9 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) public interface LogTarget { /** * Opens an output stream for the target. If already open, the stream should be reopened. diff --git a/vespalog/src/main/java/com/yahoo/log/LogUtil.java b/vespalog/src/main/java/com/yahoo/log/LogUtil.java index 3b726d9acf8..2424aa9060f 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogUtil.java +++ b/vespalog/src/main/java/com/yahoo/log/LogUtil.java @@ -4,7 +4,9 @@ package com.yahoo.log; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) class LogUtil { static boolean empty(String s) { return (s == null || s.equals("")); diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java index b602de25c7f..b7e18c83997 100644 --- a/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java @@ -9,7 +9,10 @@ import java.util.logging.Level; /** * a level controller that does lookup in a file via a memory-mapped * buffer for realtime logging control. - **/ + * @deprecated Should only be used internally in the log library + */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) class MappedLevelController implements LevelController { private static final int ONVAL = 0x20204f4e; // equals " ON" in file private static final int OFFVAL = 0x204f4646; // equals " OFF" in file diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java index 1e074445b44..97c8aa8f70f 100644 --- a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java @@ -12,7 +12,10 @@ import java.util.Map; * * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) class MappedLevelControllerRepo { private final Map<String, LevelController> levelControllerMap = new HashMap<>(); private final MappedByteBuffer mapBuf; diff --git a/vespalog/src/main/java/com/yahoo/log/RejectFilter.java b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java index c018645327a..b328073b20b 100644 --- a/vespalog/src/main/java/com/yahoo/log/RejectFilter.java +++ b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java @@ -9,7 +9,9 @@ import java.util.List; * * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) public class RejectFilter { private final List<String> rejectedMessages = new ArrayList<>(); diff --git a/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java index bc91d7e76b4..b1c245642ff 100644 --- a/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java @@ -6,7 +6,10 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public class StderrLogTarget implements LogTarget { @Override diff --git a/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java index 6a1cd9ac672..e036e534e00 100644 --- a/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java @@ -6,7 +6,10 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public class StdoutLogTarget implements LogTarget { @Override diff --git a/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java index 827c1c373f8..61e278ef93a 100644 --- a/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java +++ b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java @@ -7,7 +7,9 @@ import java.io.OutputStream; /** * @author Simon Thoresen Hult * @since 5.1.14 + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) class UncloseableOutputStream extends OutputStream { private final OutputStream out; diff --git a/vespalog/src/main/java/com/yahoo/log/Util.java b/vespalog/src/main/java/com/yahoo/log/Util.java index bcbe5bbae4a..7a7ae5221c4 100644 --- a/vespalog/src/main/java/com/yahoo/log/Util.java +++ b/vespalog/src/main/java/com/yahoo/log/Util.java @@ -9,7 +9,9 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; * @author arnej27959 * @author bjorncs * + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) public class Util { public static String getHostName () { diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java index 19e898155f4..0ce3668223d 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java @@ -1,18 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; import java.time.Instant; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Vespa log formatting utility methods. - * Contains some code based on Util.java in Cloudname https://github.com/Cloudname/cloudname + * Contains some code based on LogUtils.java in Cloudname https://github.com/Cloudname/cloudname * written by Bjørn Borud, licensed under the Apache 2.0 license. - * + * * @author arnej27959 * @author bjorncs + * + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public class VespaFormat { private static final Pattern special = Pattern.compile("[\r\\\n\\\t\\\\]+"); @@ -25,8 +30,8 @@ public class VespaFormat { private static final String processID; static { - hostname = Util.getHostName(); - processID = Util.getPID(); + hostname = LogUtils.getHostName(); + processID = LogUtils.getPID(); } /** diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java index db4e131c530..caa68cf3019 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java @@ -3,6 +3,7 @@ package com.yahoo.log; import com.yahoo.log.event.Event; +import com.yahoo.log.impl.LogUtils; import java.io.PrintWriter; import java.io.StringWriter; @@ -18,7 +19,9 @@ import java.util.regex.Pattern; * @author Bjorn Borud * @author arnej27959 * + * @deprecated Should only be used internally in the log library */ +@Deprecated(since = "7", forRemoval = true) public class VespaFormatter extends SimpleFormatter { private static final Pattern backSlash = Pattern.compile("\\\\"); @@ -34,8 +37,8 @@ public class VespaFormatter extends SimpleFormatter { public static final String serviceNameUnsetValue = "-"; static { - hostname = Util.getHostName(); - processID = Util.getPID(); + hostname = LogUtils.getHostName(); + processID = LogUtils.getPID(); } private String serviceName; diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java index b1a3305daae..0163a68bf4c 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -17,7 +17,10 @@ import static java.nio.charset.StandardCharsets.US_ASCII; /** * @author Ulf Lilleengen * @since 5.1 + * @deprecated Should only be used internally in the log library */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) public class VespaLevelControllerRepo implements LevelControllerRepo { private RandomAccessFile ctlFile; diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java index c0dac86dd26..d41ded6a9b4 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java @@ -10,6 +10,8 @@ import java.util.logging.StreamHandler; * @author Bjorn Borud * @author arnej27959 */ +@SuppressWarnings("removal") +@Deprecated(since = "7", forRemoval = true) class VespaLogHandler extends StreamHandler { private final LogTarget logTarget; diff --git a/vespalog/src/main/java/com/yahoo/log/event/Collection.java b/vespalog/src/main/java/com/yahoo/log/event/Collection.java index 5556878fd11..331fdf33d9f 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Collection.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Collection.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Collection extends Event { public Collection () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Count.java b/vespalog/src/main/java/com/yahoo/log/event/Count.java index c7c48130a47..284d23e6999 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Count.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Count.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Count extends Event { public Count () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java b/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java index a65a38095a0..fc52c577bda 100755 --- a/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java +++ b/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; +@Deprecated(forRemoval = true, since = "7") public class CountGroup extends Event { public CountGroup () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Crash.java b/vespalog/src/main/java/com/yahoo/log/event/Crash.java index 3cb6b6c0232..bcfaed4cce7 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Crash.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Crash.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Crash extends Event { public Crash () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Event.java b/vespalog/src/main/java/com/yahoo/log/event/Event.java index 8ec8c528db7..288fd1f56dc 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Event.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Event.java @@ -28,7 +28,10 @@ import java.util.regex.Pattern; * them through the logging API. * * @author Bjorn Borud + * + * @deprecated Only for internal Vespa usage */ +@Deprecated(forRemoval = true, since = "7") public abstract class Event implements Serializable { private static Logger log = Logger.getLogger(Event.class.getName()); @@ -386,7 +389,9 @@ public abstract class Event implements Serializable { * the prettiest way to do it... */ private static final void log(Logger logger, Object param) { + @SuppressWarnings("deprecation") LogRecord r = new LogRecord(LogLevel.EVENT, null); + r.setParameters(new Object[] {param}); r.setLoggerName(logger.getName()); logger.log(r); diff --git a/vespalog/src/main/java/com/yahoo/log/event/Histogram.java b/vespalog/src/main/java/com/yahoo/log/event/Histogram.java index f118c0f0bab..08875063a4e 100755 --- a/vespalog/src/main/java/com/yahoo/log/event/Histogram.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Histogram.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; +@Deprecated(forRemoval = true, since = "7") public class Histogram extends Event { public Histogram () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java index a9f9774fd9f..6283832ff6d 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java +++ b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; +@Deprecated(forRemoval = true, since = "7") public class MalformedEventException extends Exception { public MalformedEventException (Throwable cause) { super(cause); diff --git a/vespalog/src/main/java/com/yahoo/log/event/Progress.java b/vespalog/src/main/java/com/yahoo/log/event/Progress.java index f63b7981ddf..2e54e72cb17 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Progress.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Progress.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Progress extends Event { public Progress () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java index bf63ab4c4c6..517c28f5f2e 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Reloaded extends Event { public Reloaded () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java index c7683f6276f..ae537b84316 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Reloading extends Event { public Reloading () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Started.java b/vespalog/src/main/java/com/yahoo/log/event/Started.java index 41b6f8f369a..c3d90174893 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Started.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Started.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Started extends Event { public Started () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Starting.java b/vespalog/src/main/java/com/yahoo/log/event/Starting.java index aa4141eb788..4f4af50a729 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Starting.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Starting.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Starting extends Event { public Starting () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/State.java b/vespalog/src/main/java/com/yahoo/log/event/State.java index 6b9479a6518..106f4031fee 100755 --- a/vespalog/src/main/java/com/yahoo/log/event/State.java +++ b/vespalog/src/main/java/com/yahoo/log/event/State.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; +@Deprecated(forRemoval = true, since = "7") public class State extends Event { public State () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java index e40d67ed790..c6e95d877c4 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Stopped extends Event { public Stopped () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java index 9c1f534b6a7..daec0123e80 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Stopping extends Event { public Stopping () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java index 12305f1059b..b82cfebbaaa 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Unknown extends Event { public Unknown() { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Value.java b/vespalog/src/main/java/com/yahoo/log/event/Value.java index f549b9f4f07..d0d1f6424c7 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Value.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Value.java @@ -5,6 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ +@Deprecated(forRemoval = true, since = "7") public class Value extends Event { public Value () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java b/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java index d164e4c15fd..49835ca8ed4 100755 --- a/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java +++ b/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; +@Deprecated(forRemoval = true, since = "7") public class ValueGroup extends Event { public ValueGroup () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/package-info.java b/vespalog/src/main/java/com/yahoo/log/event/package-info.java index 158f89b065b..f2cb20e3ac8 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/package-info.java +++ b/vespalog/src/main/java/com/yahoo/log/event/package-info.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi +@PublicApi // TODO Vespa 8: remove package com.yahoo.log.event; import com.yahoo.api.annotations.PublicApi; diff --git a/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java b/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java new file mode 100644 index 00000000000..8c182625c85 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.impl; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * @author Ulf Lilleengen + * @author Bjorn Borud + * @author arnej27959 + * @author bjorncs + * TODO remove "public" keyword, should be package private + */ +public class LogUtils { + public static boolean empty(String s) { + return (s == null || s.equals("")); + } + public static String getHostName () { + return getDefaults().vespaHostname(); + } + public static String getPID() { + return Long.toString(ProcessHandle.current().pid()); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java b/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java index 34d8ec33093..5bcf2b76fdf 100644 --- a/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java +++ b/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java @@ -16,6 +16,7 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen * @since 5.1 */ +@SuppressWarnings("removal") public class FileLogTargetTest { @Test(expected = RuntimeException.class) public void requireThatExceptionIsThrowIfFileNotFound() throws IOException { diff --git a/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java index fe34c219e19..b377bd72f3a 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java @@ -14,6 +14,7 @@ import static org.junit.Assert.*; * * @author Bjorn Borud */ +@SuppressWarnings("deprecation") public class LogLevelTestCase { /** diff --git a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java index 6b0912a78d5..354892e7d37 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; + import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -23,6 +25,7 @@ import static org.junit.Assert.assertTrue; * * @author Bjorn Borud */ +@SuppressWarnings({"deprecation", "removal"}) public class LogSetupTestCase { // For testing zookeeper log records protected static LogRecord zookeeperLogRecord; @@ -51,8 +54,8 @@ public class LogSetupTestCase { curatorLogRecord.setLoggerName("org.apache.curator.utils.DefaultTracerDriver"); curatorLogRecord.setInstant(Instant.ofEpochMilli(1107011348029L)); - hostname = Util.getHostName(); - pid = Util.getPID(); + hostname = LogUtils.getHostName(); + pid = LogUtils.getPID(); zookeeperLogRecordString = "1107011348.029000\t" + hostname diff --git a/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java b/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java index 96717e9ac88..4a5c59a18cb 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java +++ b/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertFalse; * @author Ulf Lilleengen * @since 5.1 */ +@SuppressWarnings("removal") public class LogUtilTest { @Test public void testEmpty() { diff --git a/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java b/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java index 311359a37a0..3bce09801a1 100644 --- a/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java +++ b/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen * @since 5.1 */ +@SuppressWarnings("removal") public class RejectFilterTest { @Test public void testBasicPatternMatching() { diff --git a/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java b/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java index 95889a01e67..4b0e6622c1e 100644 --- a/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java @@ -10,6 +10,7 @@ import java.io.OutputStream; /** * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") public class UncloseableOutputStreamTestCase { @Test diff --git a/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java index 096672d8858..75294f67fb3 100644 --- a/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java @@ -8,6 +8,7 @@ import static org.junit.Assert.*; /** * @author Bjorn Borud */ +@SuppressWarnings("removal") public class UtilTestCase { /** diff --git a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java index 913bcafdd44..95a06ab2535 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; + import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -17,6 +19,7 @@ import static org.junit.Assert.fail; /** * @author Bjorn Borud */ +@SuppressWarnings({"deprecation", "removal"}) public class VespaFormatterTestCase { private String hostname; @@ -33,8 +36,8 @@ public class VespaFormatterTestCase { @Before public void setUp () { - hostname = Util.getHostName(); - pid = Util.getPID(); + hostname = LogUtils.getHostName(); + pid = LogUtils.getPID(); testRecord1 = new LogRecord(Level.INFO, "this is a test"); testRecord1.setInstant(Instant.ofEpochMilli(1098709021843L)); diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java index 264938fcd24..83db5a027a4 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen * @since 5.1 */ +@SuppressWarnings({"deprecation", "removal"}) public class VespaLevelControllerRepoTest { static int findControlString(RandomAccessFile f, String s) { diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java index 9b2af866f65..77d7fa66bb6 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -30,6 +32,7 @@ import static org.junit.Assert.fail; /** * @author Bjorn Borud */ +@SuppressWarnings("removal") public class VespaLogHandlerTestCase { private final static String hostname; private final static String pid; @@ -47,8 +50,8 @@ public class VespaLogHandlerTestCase { private static final String record4String; static { - hostname = Util.getHostName(); - pid = Util.getPID(); + hostname = LogUtils.getHostName(); + pid = LogUtils.getPID(); record1 = new LogRecord(Level.INFO, "This is a test"); record1.setInstant(ofEpochSecond(1100011348L, 29_123_543)); diff --git a/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java index b3890f860c9..045c7cd874f 100644 --- a/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@SuppressWarnings("removal") public class EventTestCase { Count countEvent; diff --git a/vespalog/src/test/java/com/yahoo/log/impl/LogUtilsTest.java b/vespalog/src/test/java/com/yahoo/log/impl/LogUtilsTest.java new file mode 100644 index 00000000000..76998395e12 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/impl/LogUtilsTest.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.impl; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +/** + * @author Ulf Lilleengen + * @author Bjorn Borud + */ +public class LogUtilsTest { + + @Test + public void testEmpty() { + assertTrue(LogUtils.empty(null)); + assertTrue(LogUtils.empty("")); + assertFalse(LogUtils.empty("f")); + assertFalse(LogUtils.empty("fo")); + assertFalse(LogUtils.empty("foo")); + } + + /** + * Just make sure the static getHostName() method returns something + * that looks half sensible. + */ + @Test + public void testSimple () { + String name = LogUtils.getHostName(); + assertNotNull(name); + assertFalse(name.equals("")); + } + +} diff --git a/vespalog/src/vespa/log/component.cpp b/vespalog/src/vespa/log/component.cpp index fdb50a83245..009a69ad0c5 100644 --- a/vespalog/src/vespa/log/component.cpp +++ b/vespalog/src/vespa/log/component.cpp @@ -45,14 +45,15 @@ Component::matches(const char *pattern) } void -Component::modifyLevels(char *levels) +Component::modifyLevels(const char *levels) { // levels is a comma-separated list of level={on|off} pairs. // the levels string can always be converted to a // AND bitmask -- for all levels to be removed // and an OR bitmask -- for all levels to be added - char *s = levels; + std::string levels_copy(levels); + char *s = &levels_copy[0]; LOG(spam, "Will modify levels for '%.*s' according to \"%s\"", (int)strcspn(_name, " :\n"), _name, levels); diff --git a/vespalog/src/vespa/log/component.h b/vespalog/src/vespa/log/component.h index bcb4a4f6e4f..aa45c871d2b 100644 --- a/vespalog/src/vespa/log/component.h +++ b/vespalog/src/vespa/log/component.h @@ -13,7 +13,7 @@ class Component { public: bool matches(const char *pattern); - void modifyLevels(char *levels); + void modifyLevels(const char *levels); void display(); const char *endPointer() const { return _charLevels + Logger::NUM_LOGLEVELS*sizeof(int); } explicit Component(char *); diff --git a/vespamalloc/CMakeLists.txt b/vespamalloc/CMakeLists.txt index 395e7ec6d5a..df8e38653bb 100644 --- a/vespamalloc/CMakeLists.txt +++ b/vespamalloc/CMakeLists.txt @@ -1,4 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin" AND + NOT DEFINED VESPA_USE_SANITIZER) add_compile_options(-fvisibility=hidden) add_definitions(-DPARANOID_LEVEL=0) @@ -25,4 +27,7 @@ vespa_define_module( ) vespa_install_script(bin/parsememorydump.pl vespa-malloc-parse-memorydump.pl bin) +else() +install(DIRECTORY DESTINATION lib64/vespa) +endif() install(FILES etc/vespamalloc.conf PERMISSIONS OWNER_READ GROUP_READ WORLD_READ DESTINATION etc) diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt index cbb851e878d..6c9b4a01e59 100644 --- a/vespamalloc/src/tests/allocfree/CMakeLists.txt +++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt @@ -8,10 +8,6 @@ vespa_add_executable(vespamalloc_allocfree_shared_test_app allocfree.cpp producerconsumer.cpp ) -vespa_add_executable(vespamalloc_realloc_test_app - SOURCES - realloc.cpp -) vespa_add_executable(vespamalloc_linklist_test_app SOURCES linklist.cpp @@ -24,5 +20,5 @@ vespa_add_executable(vespamalloc_linklist_test_app ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK - DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app + DEPENDS vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app vespamalloc vespamallocd) diff --git a/vespamalloc/src/tests/allocfree/allocfree_test.sh b/vespamalloc/src/tests/allocfree/allocfree_test.sh index fc9899a7c6c..aca710b0df6 100755 --- a/vespamalloc/src/tests/allocfree/allocfree_test.sh +++ b/vespamalloc/src/tests/allocfree/allocfree_test.sh @@ -7,8 +7,6 @@ TIME=/usr/bin/time VESPA_MALLOC_SO=../../../src/vespamalloc/libvespamalloc.so VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamallocd.so -LD_PRELOAD=$VESPA_MALLOC_SO ./vespamalloc_realloc_test_app -LD_PRELOAD=$VESPA_MALLOC_SO_D ./vespamalloc_realloc_test_app $TIME ./vespamalloc_linklist_test_app 3 LD_PRELOAD=$VESPA_MALLOC_SO $TIME ./vespamalloc_allocfree_shared_test_app 3 LD_PRELOAD=$VESPA_MALLOC_SO_D $TIME ./vespamalloc_allocfree_shared_test_app 3 diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index ea6eab44555..1bffb76b2ee 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -20,18 +20,18 @@ template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> class MemBlockT : public vespamalloc::CommonT<MinSizeClassC> { public: - typedef vespamalloc::StackEntry<vespamalloc::StackReturnEntry> Stack; + typedef vespamalloc::StackEntry Stack; enum { MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC) }; - MemBlockT() : _ptr(NULL) { } + MemBlockT() : _ptr(nullptr) { } MemBlockT(void * p) : _ptr(p) { } MemBlockT(void * p, size_t /*sz*/) : _ptr(p) { } void *ptr() { return _ptr; } const void *ptr() const { return _ptr; } - bool validAlloc() const { return _ptr != NULL; } - bool validFree() const { return _ptr != NULL; } + bool validAlloc() const { return _ptr != nullptr; } + bool validFree() const { return _ptr != nullptr; } void setExact(size_t ) { } void alloc(bool ) { } void threadId(int ) { } @@ -40,7 +40,7 @@ public: bool allocated() const { return false; } int threadId() const { return 0; } void info(FILE *, unsigned level=0) const { (void) level; } - Stack * callStack() { return NULL; } + Stack * callStack() { return nullptr; } size_t callStackLen() const { return 0; } static size_t adjustSize(size_t sz) { return sz; } @@ -138,7 +138,7 @@ int Test::Main() { ASSERT_TRUE((l >= &globalList[0]) && (l < &globalList[NumBlocks])); } List *n = List::linkOut(sharedList); - ASSERT_TRUE(n == NULL); + ASSERT_TRUE(n == nullptr); List::HeadPtr tmp(sharedList.load()); tmp._tag = 1; @@ -156,12 +156,12 @@ int Test::Main() { LinkInOutAndIn pc1(sharedList, 16, false); LinkInOutAndIn pc2(sharedList, 16, true); - ASSERT_TRUE(pool.NewThread(&c1, NULL) != NULL); - ASSERT_TRUE(pool.NewThread(&c2, NULL) != NULL); - ASSERT_TRUE(pool.NewThread(&p1, NULL) != NULL); - ASSERT_TRUE(pool.NewThread(&p2, NULL) != NULL); - ASSERT_TRUE(pool.NewThread(&pc1, NULL) != NULL); - ASSERT_TRUE(pool.NewThread(&pc2, NULL) != NULL); + ASSERT_TRUE(pool.NewThread(&c1, nullptr) != nullptr); + ASSERT_TRUE(pool.NewThread(&c2, nullptr) != nullptr); + ASSERT_TRUE(pool.NewThread(&p1, nullptr) != nullptr); + ASSERT_TRUE(pool.NewThread(&p2, nullptr) != nullptr); + ASSERT_TRUE(pool.NewThread(&pc1, nullptr) != nullptr); + ASSERT_TRUE(pool.NewThread(&pc2, nullptr) != nullptr); for (; duration > 0; --duration) { LOG(info, "%d seconds left...", duration); @@ -182,6 +182,6 @@ int Test::Main() { ASSERT_TRUE((l >= &globalList[0]) && (l < &globalList[NumBlocks])); } n = List::linkOut(sharedList); - ASSERT_TRUE(n == NULL); + ASSERT_TRUE(n == nullptr); TEST_DONE(); } diff --git a/vespamalloc/src/tests/allocfree/realloc.cpp b/vespamalloc/src/tests/allocfree/realloc.cpp deleted file mode 100644 index 739664989c8..00000000000 --- a/vespamalloc/src/tests/allocfree/realloc.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> - -TEST_SETUP(Test); - -char *ptr_add(char *ptr, size_t offset) __attribute__((noinline)); -char *ptr_add(char *ptr, size_t offset) { return ptr + offset; } - -int Test::Main() { - char * v = static_cast<char *>(malloc(0x400001)); - char * nv = static_cast<char *>(realloc(v, 0x500001)); - ASSERT_TRUE(v == nv); - v = static_cast<char *>(realloc(nv, 0x600001)); - ASSERT_TRUE(v != nv); - free(v); - - char *t = static_cast<char *>(malloc(70)); - free (ptr_add(t, 7)); - t = static_cast<char *>(malloc(0x400001)); - free (ptr_add(t, 7)); - return 0; -} diff --git a/vespamalloc/src/tests/stacktrace/stacktrace.cpp b/vespamalloc/src/tests/stacktrace/stacktrace.cpp index b28a9653d27..40d77b20e27 100644 --- a/vespamalloc/src/tests/stacktrace/stacktrace.cpp +++ b/vespamalloc/src/tests/stacktrace/stacktrace.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <cstdlib> -#include <cstdio> #include <pthread.h> #include <dlfcn.h> #include <cassert> @@ -29,7 +28,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.keepcost == 0); assert(info.ordblks == 0); assert(info.smblks == 0); - assert(info.uordblks == 0); + assert(info.uordblks > 0); assert(info.usmblks == 0); #else struct mallinfo info = mallinfo(); @@ -43,7 +42,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.keepcost == 0); assert(info.ordblks == 0); assert(info.smblks == 0); - assert(info.uordblks == 0); + assert(info.uordblks > 0); assert(info.usmblks == 0); #endif } diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index 71e9ce272f9..dd7c92a4dac 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -2,6 +2,10 @@ vespa_add_executable(vespamalloc_testatomic_app TEST SOURCES testatomic.cpp + ../../vespamalloc/malloc/mmappool.cpp + ../../vespamalloc/malloc/common.cpp + DEPENDS + vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} ) diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp index 5230869145d..dfa67f2aa7c 100644 --- a/vespamalloc/src/tests/test1/new_test.cpp +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> #include <malloc.h> #include <dlfcn.h> @@ -121,27 +122,76 @@ TEST("verify reallocarray") { } #endif -void verify_vespamalloc_usable_size() { - struct AllocInfo { size_t requested; size_t usable;}; - AllocInfo allocInfo[] = {{0x7, 0x20}, {0x27, 0x40}, {0x47, 0x80}, {0x87, 0x100}, {0x107, 0x200}, {0x207, 0x400}, - {0x407, 0x800}, {0x807, 0x1000}, {0x1007, 0x2000}, {0x2007, 0x4000}, {0x4007, 0x8000}, - {0x8007, 0x10000}, {0x10007, 0x20000}, {0x20007, 0x40000}, {0x40007, 0x80000}, {0x80007, 0x100000}, - {0x100007, 0x200000}, {0x200007, 0x400000}, {0x400007, 0x600000}}; - for (const AllocInfo & info : allocInfo) { +namespace { + +void +verify_vespamalloc_usable_size() { + struct AllocInfo { + size_t requested; + size_t usable; + }; + AllocInfo allocInfo[] = {{0x7, 0x20}, + {0x27, 0x40}, + {0x47, 0x80}, + {0x87, 0x100}, + {0x107, 0x200}, + {0x207, 0x400}, + {0x407, 0x800}, + {0x807, 0x1000}, + {0x1007, 0x2000}, + {0x2007, 0x4000}, + {0x4007, 0x8000}, + {0x8007, 0x10000}, + {0x10007, 0x20000}, + {0x20007, 0x40000}, + {0x40007, 0x80000}, + {0x80007, 0x100000}, + {0x100007, 0x200000}, + {0x200007, 0x400000}, + {0x400007, 0x600000}}; + for (const AllocInfo &info: allocInfo) { std::unique_ptr<char[]> buf = std::make_unique<char[]>(info.requested); size_t usable_size = malloc_usable_size(buf.get()); EXPECT_EQUAL(info.usable, usable_size); } } +enum class MallocLibrary { + UNKNOWN, VESPA_MALLOC, VESPA_MALLOC_D +}; + +MallocLibrary +detectLibrary() { + if (dlsym(RTLD_NEXT, "is_vespamallocd") != nullptr) { + // Debug variants will never have more memory available as there is pre/postamble for error detection. + return MallocLibrary::VESPA_MALLOC_D; + } else if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) { + return MallocLibrary::VESPA_MALLOC; + } + return MallocLibrary::UNKNOWN; +} + +MallocLibrary _env = detectLibrary(); + +size_t +count_mismatches(const char * v, char c, size_t count) { + size_t errors = 0; + for (size_t i(0); i < count; i++) { + if (v[i] != c) errors++; + } + return errors; +} + +} + TEST("verify malloc_usable_size is sane") { constexpr size_t SZ = 33; std::unique_ptr<char[]> buf = std::make_unique<char[]>(SZ); size_t usable_size = malloc_usable_size(buf.get()); - if (dlsym(RTLD_NEXT, "is_vespamallocd") != nullptr) { + if (_env == MallocLibrary::VESPA_MALLOC_D) { // Debug variants will never have more memory available as there is pre/postamble for error detection. EXPECT_EQUAL(SZ, usable_size); - } else if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) { + } else if (_env == MallocLibrary::VESPA_MALLOC) { // Normal production vespamalloc will round up EXPECT_EQUAL(64u, usable_size); verify_vespamalloc_usable_size(); @@ -151,5 +201,53 @@ TEST("verify malloc_usable_size is sane") { } } +TEST("verify mallopt") { + if (_env == MallocLibrary::UNKNOWN) return; + EXPECT_EQUAL(0, mallopt(M_MMAP_MAX, 0x1000000)); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x1000000)); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); +} + +TEST("verify mmap_limit") { + if (_env == MallocLibrary::UNKNOWN) return; + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 0x100000)); + auto small = std::make_unique<char[]>(16_Ki); + auto large_1 = std::make_unique<char[]>(1200_Ki); + EXPECT_GREATER(size_t(labs(small.get() - large_1.get())), 1_Ti); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); + auto large_2 = std::make_unique<char[]>(1200_Ki); + EXPECT_LESS(size_t(labs(small.get() - large_2.get())), 1_Ti); +} + +void +verifyReallocLarge(char * initial, bool expect_vespamalloc_optimization) { + const size_t INITIAL_SIZE = 0x400001; + const size_t SECOND_SIZE = 0x500001; + const size_t THIRD_SIZE = 0x600001; + char *v = static_cast<char *>(realloc(initial, INITIAL_SIZE)); + memset(v, 0x5b, INITIAL_SIZE); + char *nv = static_cast<char *>(realloc(v, SECOND_SIZE)); + if (expect_vespamalloc_optimization) { + ASSERT_TRUE(v == nv); + } + EXPECT_EQUAL(0u, count_mismatches(nv, 0x5b, INITIAL_SIZE)); + memset(nv, 0xbe, SECOND_SIZE); + v = static_cast<char *>(realloc(nv, THIRD_SIZE)); + if (expect_vespamalloc_optimization) { + ASSERT_TRUE(v != nv); + } + EXPECT_EQUAL(0u, count_mismatches(v, 0xbe, SECOND_SIZE)); + free(v); +} +TEST("test realloc large buffers") { + verifyReallocLarge(nullptr, _env != MallocLibrary::UNKNOWN); + verifyReallocLarge(static_cast<char *>(malloc(2000)), _env != MallocLibrary::UNKNOWN); + if (_env == MallocLibrary::UNKNOWN) return; + + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Mi)); + verifyReallocLarge(nullptr, false); + verifyReallocLarge(static_cast<char *>(malloc(2000)), false); + EXPECT_EQUAL(1, mallopt(M_MMAP_THRESHOLD, 1_Gi)); +} TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp index 1f0b2acaeef..5ce52330f77 100644 --- a/vespamalloc/src/tests/test1/testatomic.cpp +++ b/vespamalloc/src/tests/test1/testatomic.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vespamalloc/malloc/allocchunk.h> +#include <vespamalloc/malloc/mmappool.h> TEST("verify lock freeness of atomics"){ { @@ -20,4 +21,34 @@ TEST("verify lock freeness of atomics"){ } +TEST("test explicit mmap/munmap") { + vespamalloc::MMapPool mmapPool; + EXPECT_EQUAL(0u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); + + void * mmap1 = mmapPool.mmap(0xe000); + EXPECT_EQUAL(1u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); + mmapPool.unmap(mmap1); + EXPECT_EQUAL(0u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); + mmap1 = mmapPool.mmap(0xe000); + EXPECT_EQUAL(1u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); + + void * mmap2 = mmapPool.mmap(0x1e000); + EXPECT_EQUAL(2u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0x2c000u, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); + EXPECT_EQUAL(0x1e000u, mmapPool.get_size(mmap2)); + mmapPool.unmap(mmap1); + EXPECT_EQUAL(1u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0x1e000u, mmapPool.getMmappedBytes()); + mmapPool.unmap(mmap2); + EXPECT_EQUAL(0u, mmapPool.getNumMappings()); + EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespamalloc/src/tests/test2/testgraph.cpp b/vespamalloc/src/tests/test2/testgraph.cpp index 9aa3020f0be..7a79756f7fb 100644 --- a/vespamalloc/src/tests/test2/testgraph.cpp +++ b/vespamalloc/src/tests/test2/testgraph.cpp @@ -23,7 +23,7 @@ public: asciistream os; os << ' ' << node; _string += os.c_str(); - if (node.callers() == NULL) { + if (node.callers() == nullptr) { printf("%s\n", _string.c_str()); } } @@ -75,7 +75,7 @@ void testaggregator() { callGraph.addStack(s4, 3); Aggregator agg; DumpGraph<CallGraphT::Node> dump(&agg, "{ ", " }"); - callGraph.traverseDepth(dump);; + callGraph.traverseDepth(dump); asciistream ost; ost << agg; printf("%s\n", ost.c_str()); diff --git a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt index cfa8018e25f..985cd9948ad 100644 --- a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt @@ -4,6 +4,8 @@ vespa_add_library(vespamalloc_malloc OBJECT malloc.cpp allocchunk.cpp common.cpp + freelist.cpp + mmappool.cpp threadproxy.cpp memblock.cpp datasegment.cpp @@ -17,10 +19,12 @@ vespa_add_library(vespamalloc_mallocd OBJECT mallocd.cpp allocchunk.cpp common.cpp + freelist.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_d.cpp - datasegmentd.cpp + datasegment.cpp globalpoold.cpp threadpoold.cpp threadlistd.cpp @@ -31,10 +35,12 @@ vespa_add_library(vespamalloc_mallocdst16 OBJECT mallocdst16.cpp allocchunk.cpp common.cpp + freelist.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_dst.cpp - datasegmentdst.cpp + datasegment.cpp globalpooldst.cpp threadpooldst.cpp threadlistdst.cpp @@ -46,10 +52,12 @@ vespa_add_library(vespamalloc_mallocdst16_nl OBJECT mallocdst16_nl.cpp allocchunk.cpp common.cpp + freelist.cpp + mmappool.cpp threadproxy.cpp memblockboundscheck.cpp memblockboundscheck_dst.cpp - datasegmentdst.cpp + datasegment.cpp globalpooldst.cpp threadpooldst.cpp threadlistdst.cpp diff --git a/vespamalloc/src/vespamalloc/malloc/common.cpp b/vespamalloc/src/vespamalloc/malloc/common.cpp index eab023d12fa..4d97aa13b05 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.cpp +++ b/vespamalloc/src/vespamalloc/malloc/common.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "common.h" +#include <vespamalloc/util/callstack.h> #include <pthread.h> namespace vespamalloc { @@ -30,7 +31,7 @@ void Mutex::quit() void Mutex::init() { if (!_use && ! _stopRecursion) { - pthread_mutex_init(&_mutex, NULL); + pthread_mutex_init(&_mutex, nullptr); _use = true; } } @@ -43,6 +44,34 @@ Guard::Guard(Mutex & m) : MallocRecurseOnSuspend(true); } +FILE * _G_logFile = stderr; +size_t _G_bigBlockLimit = 0x80000000; + +void +logStackTrace() { + StackEntry st[32]; + size_t count = StackEntry::fillStack(st, NELEMS(st)); + st[4].info(_G_logFile); + fprintf(_G_logFile, "\n"); + for(size_t i=1; (i < count) && (i < NELEMS(st)); i++) { + const auto & s = st[i]; + if (s.valid()) { + s.info(_G_logFile); + fprintf(_G_logFile, " from "); + } + } + fprintf(_G_logFile, "\n"); +} + +void +logBigBlock(const void *ptr, size_t exact, size_t adjusted, size_t gross) +{ + size_t sz(exact); + if (std::max(std::max(sz, adjusted), gross) > _G_bigBlockLimit) { + fprintf(_G_logFile, "validating %p(%ld, %ld, %ld) ", ptr, sz, adjusted, gross); + logStackTrace(); + } +} } diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index 421575ac587..892df72def4 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -55,7 +55,7 @@ static constexpr uint32_t NUM_THREADS = 16384; using OSMemory = MmapMemory; using SizeClassT = int; -constexpr size_t ALWAYS_REUSE_LIMIT = 0x200000ul; +constexpr size_t ALWAYS_REUSE_LIMIT = 0x100000ul; inline constexpr int msbIdx(uint64_t v) { return (sizeof(v)*8 - 1) - __builtin_clzl(v); @@ -67,11 +67,11 @@ class CommonT public: static constexpr size_t MAX_ALIGN = 0x200000ul; enum {MinClassSize = MinClassSizeC}; - static inline constexpr SizeClassT sizeClass(size_t sz) { + static constexpr SizeClassT sizeClass(size_t sz) noexcept { SizeClassT tmp(msbIdx(sz - 1) - (MinClassSizeC - 1)); return (sz <= (1 << MinClassSizeC )) ? 0 : tmp; } - static inline constexpr size_t classSize(SizeClassT sc) { return (size_t(1) << (sc + MinClassSizeC)); } + static constexpr size_t classSize(SizeClassT sc) noexcept { return (size_t(1) << (sc + MinClassSizeC)); } }; inline void crash() { *((volatile unsigned *) nullptr) = 0; } @@ -122,6 +122,11 @@ public: }; void info(); +void logBigBlock(const void * ptr, size_t exact, size_t adjusted, size_t gross) __attribute__((noinline)); +void logStackTrace() __attribute__((noinline)); + +extern FILE * _G_logFile; +extern size_t _G_bigBlockLimit; } diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp index a8ff04793e7..4c815476dab 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp @@ -1,9 +1,332 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/datasegment.hpp> -#include <vespamalloc/malloc/memblock.h> -namespace vespamalloc { +#include "datasegment.h" -template class DataSegment<MemBlock>; +namespace vespamalloc::segment { + +DataSegment::~DataSegment() = default; + +#define INIT_LOG_LIMIT 0x400000000ul // 16G + +DataSegment::DataSegment(const IHelper & helper) : + _osMemory(BlockSize), + _bigSegmentLogLevel(0), + _bigIncrement (0x4000000), + _allocs2Show (8), + _unmapSize(0x100000), + _nextLogLimit(INIT_LOG_LIMIT), + _partialExtension(0), + _helper(helper), + _mutex(), + _freeList(_blockList), + _unMappedList(_blockList) +{ + size_t wanted(0x1000000000ul); //64G + void * everything = _osMemory.reserve(wanted); + if (everything) { + for (BlockIdT i = blockId(everything), m = blockId(everything) + (wanted / BlockSize); i < m; i++) { + if (i > BlockCount) { + abort(); + } + _blockList[i].sizeClass(UNUSED_BLOCK); + _blockList[i].freeChainLength(m-i); + } + _freeList.add(blockId(everything)); + } + _nextLogLimit = std::max(size_t(end()) + _nextLogLimit, _nextLogLimit); +} + +size_t +DataSegment::freeSize() const { + return _freeList.numFreeBlocks() * BlockSize; +} + +void * DataSegment::getBlock(size_t & oldBlockSize, SizeClassT sc) +{ + const size_t minBlockSize = std::max(BlockSize, _osMemory.getMinBlockSize()); + oldBlockSize = ((oldBlockSize + (minBlockSize-1))/minBlockSize)*minBlockSize; + BlockIdT numBlocks((oldBlockSize + (BlockSize - 1)) / BlockSize); + size_t blockSize = BlockSize * numBlocks; + void * newBlock; + { + Guard sync(_mutex); + newBlock = _freeList.sub(numBlocks); + if ( newBlock == nullptr ) { + newBlock = _unMappedList.sub(numBlocks); + if ( newBlock == nullptr ) { + BlockIdT nextBlock = blockId(end()); + BlockIdT startBlock = _freeList.lastBlock(nextBlock); + if (startBlock) { + size_t adjustedBlockSize = blockSize - BlockSize*(nextBlock-startBlock); + newBlock = _osMemory.get(adjustedBlockSize); + if (newBlock != nullptr) { + assert (newBlock == fromBlockId(nextBlock)); + _freeList.removeLastBlock(); + newBlock = fromBlockId(startBlock); + _partialExtension++; + } + } else { + newBlock = _osMemory.get(blockSize); + } + } else { + bool result(_osMemory.reclaim(newBlock, blockSize)); + assert (result); + (void) result; + } + } else { + DEBUG(fprintf(stderr, "Reuse segment %p(%d, %d)\n", newBlock, sc, numBlocks)); + } + } + if (newBlock == (void *) -1) { + newBlock = nullptr; + blockSize = 0; + } else if (newBlock == nullptr) { + blockSize = 0; + } else { + assert(blockId(newBlock)+numBlocks < BlockCount); + // assumes _osMemory.get will always return a value that does not make + // "i" overflow the _blockList array; this will break when hitting the + // 2T address space boundary. + for (BlockIdT i = blockId(newBlock), m = blockId(newBlock) + numBlocks; i < m; i++) { + _blockList[i].sizeClass(sc); + _blockList[i].freeChainLength(m-i); + _blockList[i].realNumBlocks(m-i); + } + } + oldBlockSize = blockSize; + if (newBlock == nullptr) { + static int recurse = 0; + if (recurse++ == 0) { + perror("Failed extending datasegment: "); + assert(false); + } + return nullptr; + } + checkAndLogBigSegment(); + return newBlock; +} + +void DataSegment::checkAndLogBigSegment() +{ + if (size_t(end()) >= _nextLogLimit) { + fprintf(stderr, "Datasegment is growing ! Start:%p - End:%p : nextLogLimit = %lx\n", start(), end(), _nextLogLimit); + _nextLogLimit = ((size_t(end()) + _bigIncrement)/_bigIncrement)*_bigIncrement; + static int recurse = 0; + if (recurse++ == 0) { + if (_bigSegmentLogLevel > 0) { + _helper.dumpInfo(_bigSegmentLogLevel); + } + } + recurse--; + } +} + +void DataSegment::returnBlock(void *ptr) +{ + BlockIdT bId(blockId(ptr)); + SizeClassT sc = _blockList[bId].sizeClass(); + size_t bsz = _helper.classSize(sc); + if (bsz >= BlockSize) { + BlockIdT numBlocks = bsz / BlockSize; + if (numBlocks > _blockList[bId].realNumBlocks()) { + numBlocks = _blockList[bId].realNumBlocks(); + } + assert(_blockList[bId].freeChainLength() >= numBlocks); + if ((_unmapSize < bsz) && _osMemory.release(ptr, numBlocks*BlockSize)) { + for(BlockIdT i=0; i < numBlocks; i++) { + BlockT & b = _blockList[bId + i]; + b.sizeClass(UNMAPPED_BLOCK); + b.freeChainLength(numBlocks - i); + } + { + Guard sync(_mutex); + _unMappedList.add(bId); + } + } else { + for(BlockIdT i=0; i < numBlocks; i++) { + BlockT & b = _blockList[bId + i]; + b.sizeClass(FREE_BLOCK); + b.freeChainLength(numBlocks - i); + } + { + Guard sync(_mutex); + _freeList.add(bId); + } + } + } +} + +namespace { + +std::vector<uint32_t> +createHistogram(bool allThreads, uint32_t maxThreads) { + if (allThreads) { + return std::vector<uint32_t>(maxThreads, 0); + } + return std::vector<uint32_t>(); +} + +} + +size_t DataSegment::infoThread(FILE * os, int level, uint32_t thread, SizeClassT sct, uint32_t maxThreadId) const +{ + using CallGraphLT = CallGraph<StackEntry, 0x10000, Index>; + bool allThreads(thread == 0); + size_t usedCount(0); + size_t checkedCount(0); + size_t allocatedCount(0); + size_t notAccounted(0); + size_t invalidCallStacks(0); + std::unique_ptr<CallGraphLT> callGraph = std::make_unique<CallGraphLT>(); + std::vector<uint32_t> threadHistogram = createHistogram(allThreads, maxThreadId); + for (size_t i=0; i < NELEMS(_blockList); ) { + const BlockT & b = _blockList[i]; + SizeClassT sc = b.sizeClass(); + if (sc == sct) { + size_t sz = _helper.classSize(sc); + size_t numB(b.freeChainLength()); + for(char *m((char *)(fromBlockId(i))), *em((char*)(fromBlockId(i+numB))); (m + sz) <= em; m += sz) { + (void) m; + (void) em; + auto mem = _helper.createMemblockInfo(m); + checkedCount++; + if (mem->allocated()) { + allocatedCount++; + if (allThreads || (mem->threadId() == thread)) { + usedCount++; + if (mem->threadId() < threadHistogram.size()) { + threadHistogram[mem->threadId()]++; + } + if (usedCount < _allocs2Show) { + mem->info(os, level); + } + if (mem->callStackLen() && mem->callStack()[0].valid()) { + size_t csl(mem->callStackLen()); + for (size_t j(0); j < csl; j++) { + if ( ! mem->callStack()[j].valid()) { + csl = j; + } + } + if ( ! callGraph->addStack(mem->callStack(), csl)) { + notAccounted++; + } + } else { + if (mem->callStackLen()) { + invalidCallStacks++; + } + } + } + } + } + i += numB; + } else { + i++; + } + } + if (checkedCount == 0) return 0; + + fprintf(os, "\nCallTree SC %d(Checked=%ld, GlobalAlloc=%ld(%ld%%)," "By%sAlloc=%ld(%2.2f%%) NotAccountedDue2FullGraph=%ld InvalidCallStacks=%ld:\n", + sct, checkedCount, allocatedCount, allocatedCount*100/checkedCount, + allThreads ? "Us" : "Me", + usedCount, static_cast<double>(usedCount*100)/checkedCount, notAccounted, invalidCallStacks); + if ( ! callGraph->empty()) { + Aggregator agg; + DumpGraph<typename CallGraphLT::Node> dump(&agg, "{ ", " }"); + callGraph->traverseDepth(dump);; + asciistream ost; + ost << agg; + fprintf(os, "%s\n", ost.c_str()); + } + if ( !threadHistogram.empty()) { + uint32_t nonZeroCount(0); + for (uint32_t i(0); i < threadHistogram.size(); i++) { + if (threadHistogram[i] > 0) { + nonZeroCount++; + } + } + using Pair = std::pair<uint32_t, uint32_t>; + std::vector<Pair> orderedHisto; + orderedHisto.reserve(nonZeroCount); + for (uint32_t i(0); i < threadHistogram.size(); i++) { + if (threadHistogram[i] > 0) { + orderedHisto.emplace_back(i, threadHistogram[i]); + } + } + std::sort(orderedHisto.begin(), orderedHisto.end(), [](const Pair & a, const Pair & b) { return a.second > b.second;}); + fprintf(os, "ThreadHistogram SC %d: [", sct); + + bool first(true); + for (const Pair & entry : orderedHisto) { + if ( !first) { + fprintf(os, ", "); + } + fprintf(os, "{%u, %u}", entry.first, entry.second); + first = false; + } + fprintf(os, " ]"); + } + return usedCount; +} + +void DataSegment::info(FILE * os, size_t level) +{ + fprintf(os, "Start at %p, End at %p(%p) size(%ld) partialExtension(%ld) NextLogLimit(%lx) logLevel(%ld)\n", + _osMemory.getStart(), _osMemory.getEnd(), sbrk(0), dataSize(), _partialExtension, _nextLogLimit, level); + size_t numAllocatedBlocks(0); + size_t numFreeBlocks = _freeList.numFreeBlocks(); + _freeList.info(os); + _unMappedList.info(os); + if (level >= 1) { +#ifdef PRINT_ALOT + SizeClassT oldSc(-17); + size_t oldChainLength(0); +#endif + size_t scTable[32+NUM_ADMIN_CLASSES]; + memset(scTable, 0, sizeof(scTable)); + for (size_t i=0; (i < NELEMS(_blockList)) && ((i*BlockSize) < dataSize()); i++) { + BlockT & b = _blockList[i]; +#ifdef PRINT_ALOT + if ((b.sizeClass() != oldSc) + || ((oldChainLength < (b.freeChainLength()+1)) + && b.freeChainLength())) + { + scTable[b.sizeClass()+NUM_ADMIN_CLASSES] += b.freeChainLength(); + oldSc = b.sizeClass(); + if (level & 0x2) { + fprintf(os, "Block %d at address %p with chainLength %d " + "freeCount %d sizeClass %d and size %d\n", + i, fromBlockId(i), b.freeChainLength(), b.freeCount(), + b.sizeClass(), classSize(b.sizeClass())); + } + } + oldChainLength = b.freeChainLength(); +#else + scTable[b.sizeClass()+NUM_ADMIN_CLASSES]++; +#endif + } + size_t numAdminBlocks(0); + for(size_t i=0; i < NUM_ADMIN_CLASSES; i++) { + if (scTable[i] != 0ul) { + numAllocatedBlocks += scTable[i]; + numAdminBlocks += scTable[i]; + fprintf(os, "SizeClass %2ld(%s) has %5ld blocks with %10lu bytes\n", + i-NUM_ADMIN_CLASSES, getAdminClassName(i-NUM_ADMIN_CLASSES), scTable[i], scTable[i]*BlockSize); + } + } + for(size_t i=NUM_ADMIN_CLASSES; i < NELEMS(scTable); i++) { + if (scTable[i] != 0ul) { + numAllocatedBlocks += scTable[i]; + fprintf(os, "SizeClass %2ld has %5ld blocks with %10lu bytes\n", + i-NUM_ADMIN_CLASSES, scTable[i], scTable[i]*BlockSize); + } + } + size_t total(dataSize()/BlockSize); + fprintf(os, "Usage: Total=%ld(100%%), admin=%ld(%ld%%), unused=%ld(%ld%%), allocated=%ld(%ld%%)\n", + total*BlockSize, + numAdminBlocks*BlockSize, numAdminBlocks*100/total, + numFreeBlocks*BlockSize, numFreeBlocks*100/total, + (numAllocatedBlocks-numAdminBlocks)*BlockSize, (numAllocatedBlocks-numAdminBlocks)*100/total); + } +} } diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h index dc81178150a..a9f5e0046db 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.h +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h @@ -1,37 +1,53 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <climits> -#include <memory> -#include <vespamalloc/malloc/common.h> +#include "common.h" +#include "freelist.h" #include <vespamalloc/util/traceutil.h> #include <vespamalloc/util/stream.h> -namespace vespamalloc { +namespace vespamalloc::segment { + +class IMemBlockInfo { +public: + virtual ~IMemBlockInfo() = default; + virtual bool allocated() const = 0; + virtual uint32_t threadId() const = 0; + virtual void info(FILE * os, int level) const = 0; + virtual uint32_t callStackLen() const = 0; + virtual const StackEntry * callStack() const = 0; +}; +class IHelper { +public: + virtual ~IHelper() = default; + virtual size_t classSize(SizeClassT sc) const = 0; + virtual void dumpInfo(int level) const = 0; + virtual std::unique_ptr<IMemBlockInfo> createMemblockInfo(void * ptr) const = 0; +}; -template<typename MemBlockPtrT> class DataSegment { public: - typedef unsigned FreeCountT; - enum { UNMAPPED_BLOCK=-4, UNUSED_BLOCK=-3, FREE_BLOCK=-2, SYSTEM_BLOCK=-1, NUM_ADMIN_CLASSES=4 }; - DataSegment() __attribute__((noinline)); + DataSegment(const DataSegment & rhs) = delete; + DataSegment & operator = (const DataSegment & rhs) = delete; + explicit DataSegment(const IHelper & helper) __attribute__((noinline)); ~DataSegment() __attribute__((noinline)); void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline)); void returnBlock(void *ptr) __attribute__((noinline)); SizeClassT sizeClass(const void * ptr) const { return _blockList[blockId(ptr)].sizeClass(); } - size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize(); } + bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; } + template<typename MemBlockPtrT> + size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize<MemBlockPtrT>(); } const void * start() const { return _osMemory.getStart(); } const void * end() const { return _osMemory.getEnd(); } static SizeClassT adjustedSizeClass(size_t sz) { return (sz >> 16) + 0x400; } static size_t adjustedClassSize(SizeClassT sc) { return (sc > 0x400) ? (sc - 0x400) << 16 : sc; } size_t dataSize() const { return (const char*)end() - (const char*)start(); } - size_t textSize() const { return size_t(start()); } + size_t freeSize() const; size_t infoThread(FILE * os, int level, uint32_t thread, SizeClassT sct, uint32_t maxThreadId=0) const __attribute__((noinline)); void info(FILE * os, size_t level) __attribute__((noinline)); - void setupLog(size_t bigMemLogLevel, size_t bigLimit, size_t bigIncrement, size_t allocs2Show) - { + void setupLog(size_t bigMemLogLevel, size_t bigLimit, size_t bigIncrement, size_t allocs2Show) { _bigSegmentLogLevel = bigMemLogLevel; if ((size_t(end()) < _nextLogLimit) || (size_t(end()) < (size_t(start()) + bigLimit))) { _nextLogLimit = size_t(start()) + bigLimit; @@ -41,87 +57,26 @@ public: checkAndLogBigSegment(); } void enableThreadSupport() { _mutex.init(); } - static size_t blockId(const void * ptr) { - return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize; - } - static void * fromBlockId(size_t id) { - return reinterpret_cast<void *>(id*BlockSize + Memory::getMinPreferredStartAddress()); - } -private: - const char * getAdminClassName(int id) { - switch (id) { - case UNMAPPED_BLOCK: return "UNMAPPED"; - case UNUSED_BLOCK: return "UNUSED"; - case FREE_BLOCK: return "FREE"; - case SYSTEM_BLOCK: return "SYSTEM"; - default: return "UNKNOWN"; - } - } - DataSegment(const DataSegment & rhs); - DataSegment & operator = (const DataSegment & rhs); - enum { BlockSize=0x200000, BlockCount=0x80000 }; //1T - - class BlockT - { - public: - BlockT(SizeClassT szClass = UNUSED_BLOCK, FreeCountT numBlocks = 0) - : _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks) - { } - SizeClassT sizeClass() const { return _sizeClass; } - FreeCountT realNumBlocks() const { return _realNumBlocks; } - FreeCountT freeChainLength() const { return _freeChainLength; } - void sizeClass(SizeClassT sc) { _sizeClass = sc; } - void realNumBlocks(FreeCountT fc) { _realNumBlocks = fc; } - void freeChainLength(FreeCountT fc) { _freeChainLength = fc; } - size_t getMaxSize() const { - return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass), - size_t(_realNumBlocks) * BlockSize)); - } - private: - SizeClassT _sizeClass; - /// Number of blocks free from here and on. For memory reuse, big blocks only. - FreeCountT _freeChainLength; - /// Real number of blocks used. Used to avoid rounding for big blocks. - FreeCountT _realNumBlocks; - }; - - template <int MaxCount> - class FreeListT { - public: - FreeListT(BlockT * blockList) __attribute__((noinline)); - void add(size_t startIndex) __attribute__((noinline)); - void * sub(size_t numBlocks) __attribute__((noinline)); - size_t lastBlock(size_t nextBlock) __attribute__((noinline)); - void removeLastBlock() { - if (_count > 0) { - _count--; - } - } - size_t info(FILE * os, int level) __attribute__((noinline)); - private: - void * linkOut(size_t findex, size_t left) __attribute__((noinline)); - BlockT *_blockList; - size_t _count; - size_t _freeStartIndex[MaxCount]; - }; +private: void checkAndLogBigSegment() __attribute__((noinline)); typedef BlockT BlockList[BlockCount]; typedef FreeListT<BlockCount/2> FreeList; - OSMemory _osMemory; - size_t _bigSegmentLogLevel; - size_t _bigIncrement; - size_t _allocs2Show; - size_t _unmapSize; + OSMemory _osMemory; + size_t _bigSegmentLogLevel; + size_t _bigIncrement; + size_t _allocs2Show; + size_t _unmapSize; + size_t _nextLogLimit; + size_t _partialExtension; + const IHelper &_helper; - size_t _nextLogLimit; - size_t _partialExtension; - Mutex _mutex; - BlockList _blockList; - FreeList _freeList; - FreeList _unMappedList; + Mutex _mutex; + BlockList _blockList; + FreeList _freeList; + FreeList _unMappedList; }; } diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp b/vespamalloc/src/vespamalloc/malloc/datasegment.hpp deleted file mode 100644 index 75ca07a5cf4..00000000000 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp +++ /dev/null @@ -1,466 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespamalloc/malloc/datasegment.h> - -namespace vespamalloc { - -template<typename MemBlockPtrT> -DataSegment<MemBlockPtrT>::~DataSegment() = default; - -#define INIT_LOG_LIMIT 0x400000000ul // 16G - -template<typename MemBlockPtrT> -DataSegment<MemBlockPtrT>::DataSegment() : - _osMemory(BlockSize), - _bigSegmentLogLevel(0), - _bigIncrement (0x4000000), - _allocs2Show (8), - _unmapSize(0x100000), - _nextLogLimit(INIT_LOG_LIMIT), - _partialExtension(0), - _mutex(), - _freeList(_blockList), - _unMappedList(_blockList) -{ - size_t wanted(0x1000000000ul); //64G - void * everything = _osMemory.reserve(wanted); - if (everything) { - for (size_t i = blockId(everything), m = blockId(everything)+(wanted/BlockSize); i < m; i++) { - if (i > BlockCount) { - abort(); - } - _blockList[i].sizeClass(UNUSED_BLOCK); - _blockList[i].freeChainLength(m-i); - } - _freeList.add(blockId(everything)); - } - _nextLogLimit = std::max(size_t(end()) + _nextLogLimit, _nextLogLimit); -} - -template<typename MemBlockPtrT> -void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc) -{ - const size_t minBlockSize = std::max(size_t(BlockSize), _osMemory.getMinBlockSize()); - oldBlockSize = ((oldBlockSize + (minBlockSize-1))/minBlockSize)*minBlockSize; - size_t numBlocks((oldBlockSize + (BlockSize-1))/BlockSize); - size_t blockSize = BlockSize * numBlocks; - void * newBlock(nullptr); - { - Guard sync(_mutex); - newBlock = _freeList.sub(numBlocks); - if ( newBlock == nullptr ) { - newBlock = _unMappedList.sub(numBlocks); - if ( newBlock == nullptr ) { - size_t nextBlock(blockId(end())); - size_t startBlock = _freeList.lastBlock(nextBlock); - if (startBlock) { - size_t adjustedBlockSize = blockSize - BlockSize*(nextBlock-startBlock); - newBlock = _osMemory.get(adjustedBlockSize); - if (newBlock != nullptr) { - assert (newBlock == fromBlockId(nextBlock)); - _freeList.removeLastBlock(); - newBlock = fromBlockId(startBlock); - _partialExtension++; - } - } else { - newBlock = _osMemory.get(blockSize); - } - } else { - bool result(_osMemory.reclaim(newBlock, blockSize)); - assert (result); - (void) result; - } - } else { - DEBUG(fprintf(stderr, "Reuse segment %p(%d, %d)\n", newBlock, sc, numBlocks)); - } - } - if (newBlock == (void *) -1) { - newBlock = nullptr; - blockSize = 0; - } else if (newBlock == nullptr) { - blockSize = 0; - } else { - assert(blockId(newBlock)+numBlocks < BlockCount); - // assumes _osMemory.get will always return a value that does not make - // "i" overflow the _blockList array; this will break when hitting the - // 2T address space boundary. - for (size_t i = blockId(newBlock), m = blockId(newBlock)+numBlocks; i < m; i++) { - _blockList[i].sizeClass(sc); - _blockList[i].freeChainLength(m-i); - _blockList[i].realNumBlocks(m-i); - } - } - oldBlockSize = blockSize; - if (newBlock == nullptr) { - static int recurse = 0; - if (recurse++ == 0) { - perror("Failed extending datasegment: "); - assert(false); - } - return nullptr; - } - checkAndLogBigSegment(); - return newBlock; -} - -template<typename MemBlockPtrT> -void DataSegment<MemBlockPtrT>::checkAndLogBigSegment() -{ - if (size_t(end()) >= _nextLogLimit) { - fprintf(stderr, "Datasegment is growing ! Start:%p - End:%p : nextLogLimit = %lx\n", start(), end(), _nextLogLimit); - _nextLogLimit = ((size_t(end()) + _bigIncrement)/_bigIncrement)*_bigIncrement; - static int recurse = 0; - if (recurse++ == 0) { - if (_bigSegmentLogLevel > 0) { - MemBlockPtrT::dumpInfo(_bigSegmentLogLevel); - } - } - recurse--; - } -} - -template<typename MemBlockPtrT> -void DataSegment<MemBlockPtrT>::returnBlock(void *ptr) -{ - size_t bId(blockId(ptr)); - SizeClassT sc = _blockList[bId].sizeClass(); - size_t bsz = MemBlockPtrT::classSize(sc); - if (bsz >= BlockSize) { - size_t numBlocks = bsz / BlockSize; - if (numBlocks > _blockList[bId].realNumBlocks()) { - numBlocks = _blockList[bId].realNumBlocks(); - } - assert(_blockList[bId].freeChainLength() >= numBlocks); - if ((_unmapSize < bsz) && _osMemory.release(ptr, numBlocks*BlockSize)) { - for(size_t i=0; i < numBlocks; i++) { - BlockT & b = _blockList[bId + i]; - b.sizeClass(UNMAPPED_BLOCK); - b.freeChainLength(numBlocks - i); - } - { - Guard sync(_mutex); - _unMappedList.add(bId); - } - } else { - for(size_t i=0; i < numBlocks; i++) { - BlockT & b = _blockList[bId + i]; - b.sizeClass(FREE_BLOCK); - b.freeChainLength(numBlocks - i); - } - { - Guard sync(_mutex); - _freeList.add(bId); - } - } - } -} - -namespace { - -std::vector<uint32_t> -createHistogram(bool allThreads, uint32_t maxThreads) { - if (allThreads) { - return std::vector<uint32_t>(maxThreads, 0); - } - return std::vector<uint32_t>(); -} - -} -template<typename MemBlockPtrT> -size_t DataSegment<MemBlockPtrT>::infoThread(FILE * os, int level, uint32_t thread, SizeClassT sct, uint32_t maxThreadId) const -{ - using CallGraphLT = CallGraph<typename MemBlockPtrT::Stack, 0x10000, Index>; - bool allThreads(thread == 0); - size_t usedCount(0); - size_t checkedCount(0); - size_t allocatedCount(0); - size_t notAccounted(0); - size_t invalidCallStacks(0); - std::unique_ptr<CallGraphLT> callGraph = std::make_unique<CallGraphLT>(); - std::vector<uint32_t> threadHistogram = createHistogram(allThreads, maxThreadId); - for (size_t i=0; i < NELEMS(_blockList); ) { - const BlockT & b = _blockList[i]; - SizeClassT sc = b.sizeClass(); - if (sc == sct) { - size_t sz(MemBlockPtrT::classSize(sc)); - size_t numB(b.freeChainLength()); - for(char *m((char *)(fromBlockId(i))), *em((char*)(fromBlockId(i+numB))); (m + sz) <= em; m += sz) { - MemBlockPtrT mem(m,0,false); - checkedCount++; - if (mem.allocated()) { - allocatedCount++; - if (allThreads || (mem.threadId() == thread)) { - usedCount++; - if (mem.threadId() < threadHistogram.size()) { - threadHistogram[mem.threadId()]++; - } - if (usedCount < _allocs2Show) { - mem.info(os, level); - } - if (mem.callStackLen() && mem.callStack()[0].valid()) { - size_t csl(mem.callStackLen()); - for (size_t j(0); j < csl; j++) { - if ( ! mem.callStack()[j].valid()) { - csl = j; - } - } - if ( ! callGraph->addStack(mem.callStack(), csl)) { - notAccounted++; - } - } else { - if (mem.callStackLen()) { - invalidCallStacks++; - } - } - } - } - } - i += numB; - } else { - i++; - } - } - if (checkedCount == 0) { - return 0; - } - - fprintf(os, "\nCallTree SC %d(Checked=%ld, GlobalAlloc=%ld(%ld%%)," "By%sAlloc=%ld(%2.2f%%) NotAccountedDue2FullGraph=%ld InvalidCallStacks=%ld:\n", - sct, checkedCount, allocatedCount, checkedCount ? allocatedCount*100/checkedCount : 0, - allThreads ? "Us" : "Me", - usedCount, checkedCount ? static_cast<double>(usedCount*100)/checkedCount : 0.0, notAccounted, invalidCallStacks); - if ( ! callGraph->empty()) { - Aggregator agg; - DumpGraph<typename CallGraphLT::Node> dump(&agg, "{ ", " }"); - callGraph->traverseDepth(dump);; - asciistream ost; - ost << agg; - fprintf(os, "%s\n", ost.c_str()); - } - if ( !threadHistogram.empty()) { - uint32_t nonZeroCount(0); - for (uint32_t i(0); i < threadHistogram.size(); i++) { - if (threadHistogram[i] > 0) { - nonZeroCount++; - } - } - using Pair = std::pair<uint32_t, uint32_t>; - std::vector<Pair> orderedHisto; - orderedHisto.reserve(nonZeroCount); - for (uint32_t i(0); i < threadHistogram.size(); i++) { - if (threadHistogram[i] > 0) { - orderedHisto.emplace_back(i, threadHistogram[i]); - } - } - std::sort(orderedHisto.begin(), orderedHisto.end(), [](const Pair & a, const Pair & b) { return a.second > b.second;}); - fprintf(os, "ThreadHistogram SC %d: [", sct); - - bool first(true); - for (const Pair & entry : orderedHisto) { - if ( !first) { - fprintf(os, ", "); - } - fprintf(os, "{%u, %u}", entry.first, entry.second); - first = false; - } - fprintf(os, " ]"); - } - return usedCount; -} - -template<typename MemBlockPtrT> -void DataSegment<MemBlockPtrT>::info(FILE * os, size_t level) -{ - fprintf(os, "Start at %p, End at %p(%p) size(%ld) partialExtension(%ld) NextLogLimit(%lx) logLevel(%ld)\n", - _osMemory.getStart(), _osMemory.getEnd(), sbrk(0), dataSize(), _partialExtension, _nextLogLimit, level); - size_t numFreeBlocks(0), numAllocatedBlocks(0); - { - // Guard sync(_mutex); - numFreeBlocks = _freeList.info(os, level); - _unMappedList.info(os, level); - } - if (level >= 1) { -#ifdef PRINT_ALOT - SizeClassT oldSc(-17); - size_t oldChainLength(0); -#endif - size_t scTable[32+NUM_ADMIN_CLASSES]; - memset(scTable, 0, sizeof(scTable)); - for (size_t i=0; (i < NELEMS(_blockList)) && ((i*BlockSize) < dataSize()); i++) { - BlockT & b = _blockList[i]; -#ifdef PRINT_ALOT - if ((b.sizeClass() != oldSc) - || ((oldChainLength < (b.freeChainLength()+1)) - && b.freeChainLength())) - { - scTable[b.sizeClass()+NUM_ADMIN_CLASSES] += b.freeChainLength(); - oldSc = b.sizeClass(); - if (level & 0x2) { - fprintf(os, "Block %d at address %p with chainLength %d " - "freeCount %d sizeClass %d and size %d\n", - i, fromBlockId(i), b.freeChainLength(), b.freeCount(), - b.sizeClass(), classSize(b.sizeClass())); - } - } - oldChainLength = b.freeChainLength(); -#else - scTable[b.sizeClass()+NUM_ADMIN_CLASSES]++; -#endif - } - size_t numAdminBlocks(0); - for(size_t i=0; i < NUM_ADMIN_CLASSES; i++) { - if (scTable[i] != 0ul) { - numAllocatedBlocks += scTable[i]; - numAdminBlocks += scTable[i]; - fprintf(os, "SizeClass %2ld(%s) has %5ld blocks with %10lu bytes\n", - i-NUM_ADMIN_CLASSES, getAdminClassName(i-NUM_ADMIN_CLASSES), scTable[i], scTable[i]*BlockSize); - } - } - for(size_t i=NUM_ADMIN_CLASSES; i < NELEMS(scTable); i++) { - if (scTable[i] != 0ul) { - numAllocatedBlocks += scTable[i]; - fprintf(os, "SizeClass %2ld has %5ld blocks with %10lu bytes\n", - i-NUM_ADMIN_CLASSES, scTable[i], scTable[i]*BlockSize); - } - } - size_t total(dataSize()/BlockSize); - fprintf(os, "Usage: Total=%ld(100%%), admin=%ld(%ld%%), unused=%ld(%ld%%), allocated=%ld(%ld%%)\n", - total*BlockSize, - numAdminBlocks*BlockSize, numAdminBlocks*100/total, - numFreeBlocks*BlockSize, numFreeBlocks*100/total, - (numAllocatedBlocks-numAdminBlocks)*BlockSize, (numAllocatedBlocks-numAdminBlocks)*100/total); - } -} - -template<typename MemBlockPtrT> -template <int MaxCount> -DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::FreeListT(BlockT * blockList) : - _blockList(blockList), - _count(0) -{ - for (size_t i = 0; i < NELEMS(_freeStartIndex); i++) { - _freeStartIndex[i] = -1; - } -} - -template<typename MemBlockPtrT> -template <int MaxCount> -void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex) -{ - size_t i(0); - size_t numBlocks(_blockList[startIndex].freeChainLength()); - for (i=0; (i < _count) && (_freeStartIndex[i] < startIndex); i++) { } - size_t prevIndex(0), nextIndex(0); - BlockT * prev(nullptr), * next(nullptr); - if (i > 0) { - prevIndex = _freeStartIndex[i-1]; - prev = & _blockList[prevIndex]; - } - if (i < _count) { - nextIndex = _freeStartIndex[i]; - next = & _blockList[nextIndex]; - } - - if (prev && (prevIndex + prev->freeChainLength() == startIndex)) { - // Join with freeChain ahead. - prev->freeChainLength(prev->freeChainLength() + numBlocks); - startIndex = prevIndex; - } else if (next && (startIndex + numBlocks == nextIndex)) { - // Join with freeChain that follows. - _freeStartIndex[i] = startIndex; - nextIndex = startIndex; - size_t oldNextCount = next->freeChainLength(); - next = & _blockList[startIndex]; - next->freeChainLength(oldNextCount + numBlocks); - } else { - // Insert. - for(size_t j=0; j < (_count-i); j++) { - _freeStartIndex[_count-j] = _freeStartIndex[_count-j-1]; - } - _count++; - _freeStartIndex[i] = startIndex; - } - - if (prev && next && (prevIndex + prev->freeChainLength() == nextIndex)) { - prev->freeChainLength(prev->freeChainLength() + next->freeChainLength()); - _count--; - for(size_t j=i; j < _count; j++) { - _freeStartIndex[j] = _freeStartIndex[j+1]; - } - _freeStartIndex[_count] = -1; - } -} - -template<typename MemBlockPtrT> -template <int MaxCount> -void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(size_t numBlocks) -{ - void * block(nullptr); - size_t bestFitIndex(_count); - int bestLeft(INT_MAX); - for(size_t i=0; i < _count; i++) { - size_t index(_freeStartIndex[i]); - BlockT & b = _blockList[index]; - int left = b.freeChainLength() - numBlocks; - if ((left >= 0) && (left < bestLeft)) { - bestLeft = left; - bestFitIndex = i; - } - } - if (bestLeft != INT_MAX) { - block = linkOut(bestFitIndex, bestLeft); - } - return block; -} - -template<typename MemBlockPtrT> -template <int MaxCount> -size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBlock) -{ - size_t lastIndex(0); - if (_count > 0) { - size_t index(_freeStartIndex[_count-1]); - BlockT & b = _blockList[index]; - if (index + b.freeChainLength() == nextBlock) { - lastIndex = index; - } - } - return lastIndex; -} - -template<typename MemBlockPtrT> -template <int MaxCount> -size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os, int UNUSED(level)) -{ - size_t freeBlockCount(0); - for (size_t i=0; i < _count; i++) { - size_t index(_freeStartIndex[i]); - const BlockT & b = _blockList[index]; - freeBlockCount += b.freeChainLength(); - fprintf(os, "Free #%3ld block #%5ld chainlength %5d size %10lu\n", - i, index, b.freeChainLength(), size_t(b.freeChainLength())*BlockSize); - } - return freeBlockCount; -} - -template<typename MemBlockPtrT> -template <int MaxCount> -void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::linkOut(size_t findex, size_t left) -{ - size_t index(_freeStartIndex[findex]); - BlockT & b = _blockList[index]; - size_t startIndex = index + left; - void *block = fromBlockId(startIndex); - if (left > 0) { - b.freeChainLength(left); - } else { - _count--; - for(size_t j=findex; j < (_count); j++) { - _freeStartIndex[j] = _freeStartIndex[j+1]; - } - _freeStartIndex[_count] = -1; - } - return block; -} - -} diff --git a/vespamalloc/src/vespamalloc/malloc/datasegmentd.cpp b/vespamalloc/src/vespamalloc/malloc/datasegmentd.cpp deleted file mode 100644 index c285f03fcf3..00000000000 --- a/vespamalloc/src/vespamalloc/malloc/datasegmentd.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/datasegment.hpp> -#include <vespamalloc/malloc/memblockboundscheck_d.h> - -namespace vespamalloc { - -template class DataSegment<MemBlockBoundsCheck>; - -} diff --git a/vespamalloc/src/vespamalloc/malloc/datasegmentdst.cpp b/vespamalloc/src/vespamalloc/malloc/datasegmentdst.cpp deleted file mode 100644 index cb5b9c0981c..00000000000 --- a/vespamalloc/src/vespamalloc/malloc/datasegmentdst.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespamalloc/malloc/datasegment.hpp> -#include <vespamalloc/malloc/memblockboundscheck_dst.h> - -namespace vespamalloc { - -template class DataSegment<MemBlockBoundsCheck>; - -} diff --git a/vespamalloc/src/vespamalloc/malloc/freelist.cpp b/vespamalloc/src/vespamalloc/malloc/freelist.cpp new file mode 100644 index 00000000000..8c9c4875207 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/freelist.cpp @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "freelist.hpp" + +namespace vespamalloc::segment { + +template class FreeListT<0x40000>; + +} diff --git a/vespamalloc/src/vespamalloc/malloc/freelist.h b/vespamalloc/src/vespamalloc/malloc/freelist.h new file mode 100644 index 00000000000..977252b0a03 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/freelist.h @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "common.h" + +namespace vespamalloc::segment { + +using BlockIdT = uint32_t; +enum { UNMAPPED_BLOCK=-4, UNUSED_BLOCK=-3, FREE_BLOCK=-2, SYSTEM_BLOCK=-1, NUM_ADMIN_CLASSES=4 }; + +inline +const char * +getAdminClassName(int id) { + switch (id) { + case UNMAPPED_BLOCK: return "UNMAPPED"; + case UNUSED_BLOCK: return "UNUSED"; + case FREE_BLOCK: return "FREE"; + case SYSTEM_BLOCK: return "SYSTEM"; + default: return "UNKNOWN"; + } +} + +// Allow for 1T heap +static constexpr size_t BlockSize = 0x200000ul; +static constexpr BlockIdT BlockCount = 0x80000; + +inline +BlockIdT +blockId(const void * ptr) { + return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize; +} + +inline +void * +fromBlockId(size_t id) { + return reinterpret_cast<void *>(id*BlockSize + Memory::getMinPreferredStartAddress()); +} + +class BlockT +{ +public: + BlockT(SizeClassT szClass = UNUSED_BLOCK, BlockIdT numBlocks = 0) + : _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks) + { } + SizeClassT sizeClass() const { return _sizeClass; } + BlockIdT realNumBlocks() const { return _realNumBlocks; } + BlockIdT freeChainLength() const { return _freeChainLength; } + void sizeClass(SizeClassT sc) { _sizeClass = sc; } + void realNumBlocks(BlockIdT fc) { _realNumBlocks = fc; } + void freeChainLength(BlockIdT fc) { _freeChainLength = fc; } + template<typename MemBlockPtrT> + size_t getMaxSize() const { + return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass), + size_t(_realNumBlocks) * BlockSize)); + } +private: + SizeClassT _sizeClass; + /// Number of blocks free from here and on. For memory reuse, big blocks only. + BlockIdT _freeChainLength; + /// Real number of blocks used. Used to avoid rounding for big blocks. + BlockIdT _realNumBlocks; +}; + +template <int MaxCount> +class FreeListT { +public: + using Index = BlockIdT; + FreeListT(BlockT * blockList) __attribute__((noinline)); + FreeListT(const FreeListT &) = delete; + FreeListT & operator =(const FreeListT &) = delete; + FreeListT(FreeListT &&) = delete; + FreeListT & operator =(FreeListT &&) = delete; + ~FreeListT(); + void add(Index startIndex) __attribute__((noinline)); + void * sub(Index numBlocks) __attribute__((noinline)); + Index lastBlock(Index nextBlock) __attribute__((noinline)); + void removeLastBlock() { + if (_count > 0) { + _count--; + } + } + Index numFreeBlocks() const; + void info(FILE * os) __attribute__((noinline)); +private: + void * linkOut(Index findex, Index left) __attribute__((noinline)); + BlockT *_blockList; + Index _count; + Index _freeStartIndex[MaxCount]; +}; + +} diff --git a/vespamalloc/src/vespamalloc/malloc/freelist.hpp b/vespamalloc/src/vespamalloc/malloc/freelist.hpp new file mode 100644 index 00000000000..0bc812b8ad3 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/freelist.hpp @@ -0,0 +1,152 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "freelist.h" + +namespace vespamalloc::segment { + + +template <int MaxCount> +FreeListT<MaxCount>::FreeListT(BlockT * blockList) : + _blockList(blockList), + _count(0) +{ + for (size_t i = 0; i < NELEMS(_freeStartIndex); i++) { + _freeStartIndex[i] = -1; + } +} + +template <int MaxCount> +FreeListT<MaxCount>::~FreeListT() = default; + +template <int MaxCount> +void +FreeListT<MaxCount>::add(Index startIndex) +{ + Index i(0); + Index numBlocks(_blockList[startIndex].freeChainLength()); + for (i=0; (i < _count) && (_freeStartIndex[i] < startIndex); i++) { } + Index prevIndex(0), nextIndex(0); + BlockT * prev(nullptr), * next(nullptr); + if (i > 0) { + prevIndex = _freeStartIndex[i-1]; + prev = & _blockList[prevIndex]; + } + if (i < _count) { + nextIndex = _freeStartIndex[i]; + next = & _blockList[nextIndex]; + } + + if (prev && (prevIndex + prev->freeChainLength() == startIndex)) { + // Join with freeChain ahead. + prev->freeChainLength(prev->freeChainLength() + numBlocks); + startIndex = prevIndex; + } else if (next && (startIndex + numBlocks == nextIndex)) { + // Join with freeChain that follows. + _freeStartIndex[i] = startIndex; + nextIndex = startIndex; + Index oldNextCount = next->freeChainLength(); + next = & _blockList[startIndex]; + next->freeChainLength(oldNextCount + numBlocks); + } else { + // Insert. + for(Index j=0; j < (_count-i); j++) { + _freeStartIndex[_count-j] = _freeStartIndex[_count-j-1]; + } + _count++; + _freeStartIndex[i] = startIndex; + } + + if (prev && next && (prevIndex + prev->freeChainLength() == nextIndex)) { + prev->freeChainLength(prev->freeChainLength() + next->freeChainLength()); + _count--; + for(Index j=i; j < _count; j++) { + _freeStartIndex[j] = _freeStartIndex[j+1]; + } + _freeStartIndex[_count] = -1; + } +} + +template <int MaxCount> +void * +FreeListT<MaxCount>::sub(Index numBlocks) +{ + void * block(nullptr); + size_t bestFitIndex(_count); + int bestLeft(INT_MAX); + for(size_t i=0; i < _count; i++) { + size_t index(_freeStartIndex[i]); + BlockT & b = _blockList[index]; + int left = b.freeChainLength() - numBlocks; + if ((left >= 0) && (left < bestLeft)) { + bestLeft = left; + bestFitIndex = i; + } + } + if (bestLeft != INT_MAX) { + block = linkOut(bestFitIndex, bestLeft); + } + return block; +} + +template <int MaxCount> +uint32_t +FreeListT<MaxCount>::lastBlock(Index nextBlock) +{ + Index lastIndex(0); + if (_count > 0) { + Index index(_freeStartIndex[_count-1]); + BlockT & b = _blockList[index]; + if (index + b.freeChainLength() == nextBlock) { + lastIndex = index; + } + } + return lastIndex; +} + +template <int MaxCount> +void +FreeListT<MaxCount>::info(FILE * os) +{ + for (Index i=0; i < _count; i++) { + Index index(_freeStartIndex[i]); + const BlockT & b = _blockList[index]; + fprintf(os, "Free #%3d block #%5d chainlength %5d size %10lu\n", + i, index, b.freeChainLength(), b.freeChainLength()*BlockSize); + } +} + +template <int MaxCount> +uint32_t +FreeListT<MaxCount>::numFreeBlocks() const +{ + Index freeBlockCount(0); + for (Index i=0; i < _count; i++) { + Index index(_freeStartIndex[i]); + const BlockT & b = _blockList[index]; + freeBlockCount += b.freeChainLength(); + } + return freeBlockCount; +} + +template <int MaxCount> +void * +FreeListT<MaxCount>::linkOut(Index findex, Index left) +{ + size_t index(_freeStartIndex[findex]); + BlockT & b = _blockList[index]; + Index startIndex = index + left; + void *block = fromBlockId(startIndex); + if (left > 0) { + b.freeChainLength(left); + } else { + _count--; + for(Index j=findex; j < (_count); j++) { + _freeStartIndex[j] = _freeStartIndex[j+1]; + } + _freeStartIndex[_count] = -1; + } + return block; +} + +} diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.h b/vespamalloc/src/vespamalloc/malloc/globalpool.h index a6b398bc1f1..807d1498633 100644 --- a/vespamalloc/src/vespamalloc/malloc/globalpool.h +++ b/vespamalloc/src/vespamalloc/malloc/globalpool.h @@ -13,9 +13,10 @@ namespace vespamalloc { template <typename MemBlockPtrT> class AllocPoolT { + using DataSegment = segment::DataSegment; public: typedef AFList<MemBlockPtrT> ChunkSList; - AllocPoolT(DataSegment<MemBlockPtrT> & ds); + AllocPoolT(DataSegment & ds); ~AllocPoolT(); ChunkSList *getFree(SizeClassT sc, size_t minBlocks); @@ -24,7 +25,7 @@ public: ChunkSList *exactAlloc(size_t exactSize, SizeClassT sc, ChunkSList * csl) __attribute__((noinline)); ChunkSList *returnMemory(SizeClassT sc, ChunkSList * csl) __attribute__((noinline)); - DataSegment<MemBlockPtrT> & dataSegment() { return _dataSegment; } + DataSegment & dataSegment() { return _dataSegment; } void enableThreadSupport() __attribute__((noinline)); static void setParams(size_t threadCacheLimit); @@ -71,15 +72,15 @@ private: } }; - Mutex _mutex; - ChunkSList * _chunkPool; - AllocFree _scList[NUM_SIZE_CLASSES]; - DataSegment<MemBlockPtrT> & _dataSegment; - std::atomic<size_t> _getChunks; - std::atomic<size_t> _getChunksSum; - std::atomic<size_t> _allocChunkList; - Stat _stat[NUM_SIZE_CLASSES]; - static size_t _threadCacheLimit __attribute__((visibility("hidden"))); + Mutex _mutex; + ChunkSList * _chunkPool; + AllocFree _scList[NUM_SIZE_CLASSES]; + DataSegment & _dataSegment; + std::atomic<size_t> _getChunks; + std::atomic<size_t> _getChunksSum; + std::atomic<size_t> _allocChunkList; + Stat _stat[NUM_SIZE_CLASSES]; + static size_t _threadCacheLimit __attribute__((visibility("hidden"))); }; } diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp index 65e9af2fe52..71694b9a653 100644 --- a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp +++ b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp @@ -11,7 +11,7 @@ template <typename MemBlockPtrT> size_t AllocPoolT<MemBlockPtrT>::_threadCacheLimit __attribute__((visibility("hidden"))) = 0x10000; template <typename MemBlockPtrT> -AllocPoolT<MemBlockPtrT>::AllocPoolT(DataSegment<MemBlockPtrT> & ds) +AllocPoolT<MemBlockPtrT>::AllocPoolT(DataSegment & ds) : _chunkPool(nullptr), _scList(), _dataSegment(ds), @@ -127,7 +127,7 @@ AllocPoolT<MemBlockPtrT>::exactAlloc(size_t exactSize, SizeClassT sc, csl->add(mem); ChunkSList * ncsl = csl; USE_STAT2(_stat[sc]._exactAlloc.fetch_add(1, std::memory_order_relaxed)); - mem.logBigBlock(exactSize, mem.adjustSize(exactSize), MemBlockPtrT::classSize(sc)); + logBigBlock(mem.ptr(), exactSize, mem.adjustSize(exactSize), MemBlockPtrT::classSize(sc)); PARANOID_CHECK1( if (ncsl->empty() || (ncsl->count() > ChunkSList::NumBlocks)) { *(int*)0 = 0; } ); return ncsl; } @@ -143,7 +143,7 @@ AllocPoolT<MemBlockPtrT>::returnMemory(SizeClassT sc, typename AllocPoolT<MemBlo for(; !csl->empty(); ) { MemBlockPtrT mem; csl->sub(mem); - mem.logBigBlock(mem.size(), mem.adjustSize(mem.size()), MemBlockPtrT::classSize(sc)); + logBigBlock(mem.ptr(), mem.size(), mem.adjustSize(mem.size()), MemBlockPtrT::classSize(sc)); _dataSegment.returnBlock(mem.rawPtr()); } completelyEmpty = csl; @@ -234,7 +234,7 @@ AllocPoolT<MemBlockPtrT>::allocChunkList(const Guard & guard) { (void) guard; size_t blockSize(sizeof(ChunkSList)*0x2000); - void * block = _dataSegment.getBlock(blockSize, _dataSegment.SYSTEM_BLOCK); + void * block = _dataSegment.getBlock(blockSize, segment::SYSTEM_BLOCK); ChunkSList * newList(nullptr); if (block != nullptr) { size_t chunksInBlock(blockSize/sizeof(ChunkSList)); diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.cpp b/vespamalloc/src/vespamalloc/malloc/malloc.cpp index 75c06c04859..36912e80768 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.cpp +++ b/vespamalloc/src/vespamalloc/malloc/malloc.cpp @@ -26,7 +26,7 @@ template <> void MemBlock:: dumpInfo(size_t level) { - _GmemP->info(_logFile, level); + _GmemP->info(_G_logFile, level); } } diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h index 8e0a39e944b..24bfda0b840 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.h +++ b/vespamalloc/src/vespamalloc/malloc/malloc.h @@ -11,9 +11,23 @@ namespace vespamalloc { +template <typename MemBlockPtrT> +class MemBlockInfoT final : public segment::IMemBlockInfo { +public: + MemBlockInfoT(void *ptr) : _mem(ptr, 0, false) { } + bool allocated() const override { return _mem.allocated(); } + uint32_t threadId() const override { return _mem.threadId(); } + void info(FILE * os, int level) const override { _mem.info(os, level); } + uint32_t callStackLen() const override { return _mem.callStackLen(); } + const StackEntry * callStack() const override { return _mem.callStack(); } +private: + MemBlockPtrT _mem; +}; + template <typename MemBlockPtrT, typename ThreadListT> -class MemoryManager : public IAllocator +class MemoryManager : public IAllocator, public segment::IHelper { + using DataSegment = segment::DataSegment; public: MemoryManager(size_t logLimitAtStart); ~MemoryManager() override; @@ -24,26 +38,45 @@ public: MemBlockPtrT::Stack::setStopAddress(returnAddressStop); } size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); } + size_t classSize(SizeClassT sc) const override { return MemBlockPtrT::classSize(sc); } + void dumpInfo(int level) const override { MemBlockPtrT::dumpInfo(level); } + std::unique_ptr<segment::IMemBlockInfo> + createMemblockInfo(void * ptr) const override { + return std::make_unique<MemBlockInfoT<MemBlockPtrT>>(ptr); + } + int mallopt(int param, int value); void *malloc(size_t sz); void *malloc(size_t sz, std::align_val_t); void *realloc(void *oldPtr, size_t sz); void free(void *ptr) { - freeSC(ptr, _segment.sizeClass(ptr)); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, _segment.sizeClass(ptr)); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } void free(void *ptr, size_t sz) { - freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } void free(void *ptr, size_t sz, std::align_val_t alignment) { - freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + if (_segment.containsPtr(ptr)) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + } else { + _mmapPool.unmap(MemBlockPtrT(ptr).rawPtr()); + } } size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } size_t usable_size(void *ptr) const { - return MemBlockPtrT::usable_size(ptr, _segment.getMaxSize(ptr)); + return MemBlockPtrT::usable_size(ptr, _segment.getMaxSize<MemBlockPtrT>(ptr)); } - void *calloc(size_t nelm, size_t esz) { + void * calloc(size_t nelm, size_t esz) { void * ptr = malloc(nelm * esz); if (ptr) { memset(ptr, 0, nelm * esz); @@ -63,25 +96,28 @@ public: _threadList.setParams(threadCacheLimit); _allocPool.setParams(threadCacheLimit); } - const DataSegment<MemBlockPtrT> & dataSegment() const { return _segment; } + const DataSegment & dataSegment() const { return _segment; } + const MMapPool & mmapPool() const { return _mmapPool; } private: void freeSC(void *ptr, SizeClassT sc); - void crash() __attribute__((noinline));; - typedef AllocPoolT<MemBlockPtrT> AllocPool; - typedef typename ThreadListT::ThreadPool ThreadPool; - size_t _prAllocLimit; - DataSegment<MemBlockPtrT> _segment; - AllocPool _allocPool; - ThreadListT _threadList; + void crash() __attribute__((noinline)); + using AllocPool = AllocPoolT<MemBlockPtrT>; + using ThreadPool = typename ThreadListT::ThreadPool; + size_t _prAllocLimit; + DataSegment _segment; + AllocPool _allocPool; + MMapPool _mmapPool; + ThreadListT _threadList; }; template <typename MemBlockPtrT, typename ThreadListT> MemoryManager<MemBlockPtrT, ThreadListT>::MemoryManager(size_t logLimitAtStart) : IAllocator(), _prAllocLimit(logLimitAtStart), - _segment(), + _segment(*this), _allocPool(_segment), - _threadList(_allocPool) + _mmapPool(), + _threadList(_allocPool, _mmapPool) { setAllocatorForThreads(this); initThisThread(); @@ -122,18 +158,7 @@ template <typename MemBlockPtrT, typename ThreadListT> void MemoryManager<MemBlockPtrT, ThreadListT>::crash() { fprintf(stderr, "vespamalloc detected unrecoverable error.\n"); -#if 0 - if (_invalidMemLogLevel > 0) { - static size_t numRecurse=0; - if (numRecurse++ == 0) { - MemBlockPtrT::dumpInfo(_invalidMemLogLevel); - } - numRecurse--; - } - sleep(1); -#else abort(); -#endif } template <typename MemBlockPtrT, typename ThreadListT> @@ -149,6 +174,11 @@ void MemoryManager<MemBlockPtrT, ThreadListT>::info(FILE * os, size_t level) } template <typename MemBlockPtrT, typename ThreadListT> +int MemoryManager<MemBlockPtrT, ThreadListT>::mallopt(int param, int value) { + return _threadList.getCurrent().mallopt(param, value); +} + +template <typename MemBlockPtrT, typename ThreadListT> void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz) { MemBlockPtrT mem; @@ -205,35 +235,37 @@ void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc) template <typename MemBlockPtrT, typename ThreadListT> void * MemoryManager<MemBlockPtrT, ThreadListT>::realloc(void *oldPtr, size_t sz) { - void *ptr(NULL); - if (oldPtr) { - MemBlockPtrT mem(oldPtr); - mem.readjustAlignment(_segment); - if (! mem.validAlloc()) { - fprintf(stderr, "Someone has tamper with my pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size()); - crash(); - } - SizeClassT sc(_segment.sizeClass(oldPtr)); - if (sc >= 0) { - size_t oldSz(_segment.getMaxSize(oldPtr)); - if (sz > oldSz) { - ptr = malloc(sz); - if (ptr) { - memcpy(ptr, oldPtr, oldSz); - free(oldPtr); - } - } else { - mem.setExact(sz); - ptr = oldPtr; - } - } else { + if (oldPtr == nullptr) return malloc(sz); + if ( ! _segment.containsPtr(oldPtr)) { + void * ptr = malloc(sz); + size_t oldBlockSize = _mmapPool.get_size(MemBlockPtrT(oldPtr).rawPtr()); + memcpy(ptr, oldPtr, MemBlockPtrT::unAdjustSize(oldBlockSize)); + _mmapPool.unmap(MemBlockPtrT(oldPtr).rawPtr()); + return ptr; + } + + MemBlockPtrT mem(oldPtr); + mem.readjustAlignment(_segment); + if (! mem.validAlloc()) { + fprintf(stderr, "Someone has tampered with the pre/post signatures of my memoryblock %p(%ld).\n", mem.ptr(), mem.size()); + crash(); + } + + SizeClassT sc(_segment.sizeClass(oldPtr)); + void * ptr; + if (sc >= 0) { + size_t oldSz(_segment.getMaxSize<MemBlockPtrT>(oldPtr)); + if (sz > oldSz) { ptr = malloc(sz); - if (ptr) { - memcpy(ptr, oldPtr, sz); - } + memcpy(ptr, oldPtr, oldSz); + free(oldPtr); + } else { + mem.setExact(sz); + ptr = oldPtr; } } else { ptr = malloc(sz); + memcpy(ptr, oldPtr, sz); } PARANOID_CHECK2( { MemBlockPtrT mem(ptr); mem.readjustAlignment(_segment); if (! mem.validAlloc()) { crash(); } }); return ptr; diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.h b/vespamalloc/src/vespamalloc/malloc/memblock.h index 7a7ee7c71f0..f7b5923ecff 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblock.h +++ b/vespamalloc/src/vespamalloc/malloc/memblock.h @@ -3,7 +3,7 @@ #include <vespamalloc/util/callstack.h> #include <vespamalloc/malloc/common.h> -#include <stdio.h> +#include <cstdio> namespace vespamalloc { @@ -12,7 +12,7 @@ class MemBlockT : public CommonT<MinSizeClassC> { public: using Parent = CommonT<MinSizeClassC>; - using Stack = StackEntry<StackReturnEntry>; + using Stack = StackEntry; enum { MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC) @@ -40,17 +40,14 @@ public: bool allocated() const { return false; } uint32_t threadId() const { return 0; } void info(FILE *, unsigned level=0) const { (void) level; } - Stack * callStack() { return nullptr; } + const Stack * callStack() const { return nullptr; } size_t callStackLen() const { return 0; } void fillMemory(size_t) { } - void logBigBlock(size_t exact, size_t adjusted, size_t gross) const __attribute__((noinline)); static size_t adjustSize(size_t sz) { return sz; } static size_t adjustSize(size_t sz, std::align_val_t) { return sz; } static size_t unAdjustSize(size_t sz) { return sz; } static void dumpInfo(size_t level); - static void dumpFile(FILE * fp) { _logFile = fp; } - static void bigBlockLimit(size_t lim); static void setFill(uint8_t ) { } static bool verifySizeClass(int sc) { (void) sc; return true; } static size_t getMinSizeForAlignment(size_t align, size_t sz) { @@ -60,11 +57,9 @@ public: } private: void * _ptr; - static FILE *_logFile; - static size_t _bigBlockLimit; }; -typedef MemBlockT<5, 20> MemBlock; +using MemBlock = MemBlockT<5, 20>; template <> void MemBlock::dumpInfo(size_t level); extern template class MemBlockT<5, 20>; diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.hpp b/vespamalloc/src/vespamalloc/malloc/memblock.hpp index ffd27d2dfbc..c4bef6de99d 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblock.hpp +++ b/vespamalloc/src/vespamalloc/malloc/memblock.hpp @@ -5,40 +5,5 @@ namespace vespamalloc { -template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> -void -MemBlockT<MinSizeClassC, MaxSizeClassMultiAllocC>::logBigBlock(size_t exact, size_t adjusted, size_t gross) const -{ - size_t sz(exact); - if (std::max(std::max(sz, adjusted), gross) > _bigBlockLimit) { - Stack st[32]; - size_t count = Stack::fillStack(st, NELEMS(st)); - fprintf(_logFile, "validating %p(%ld, %ld, %ld)", - ptr(), sz, adjusted, gross); - st[3].info(_logFile); - fprintf(_logFile, "\n"); - for(size_t i=1; (i < count) && (i < NELEMS(st)); i++) { - const Stack & s = st[i]; - if (s.valid()) { - s.info(_logFile); - fprintf(_logFile, " from "); - } - } - fprintf(_logFile, "\n"); - } -} - -template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> -void -MemBlockT<MinSizeClassC, MaxSizeClassMultiAllocC>::bigBlockLimit(size_t lim) -{ - _bigBlockLimit = lim; -} - -template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> -FILE * MemBlockT<MinSizeClassC, MaxSizeClassMultiAllocC>::_logFile = stderr; -template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> -size_t MemBlockT<MinSizeClassC, MaxSizeClassMultiAllocC>::_bigBlockLimit = 0x80000000; - } diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h index 8c734f6dbd3..b465a4e834c 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h @@ -9,7 +9,7 @@ namespace vespamalloc { class MemBlockBoundsCheckBaseTBase : public CommonT<5> { public: - typedef StackEntry<StackReturnEntry> Stack; + using Stack = StackEntry; void * rawPtr() { return _ptr; } void *ptr() { char *p((char*)_ptr); @@ -24,7 +24,7 @@ public: bool allocated() const { return (static_cast<uint32_t*>(_ptr)[3] == ALLOC_MAGIC); } size_t size() const { return static_cast<const uint32_t *>(_ptr)[0]; } size_t alignment() const { return static_cast<const uint32_t *>(_ptr)[1]; } - uint32_t threadId() const { return static_cast<uint32_t *>(_ptr)[2]; } + uint32_t threadId() const { return static_cast<uint32_t *>(_ptr)[2]; } Stack * callStack() { return reinterpret_cast<Stack *>((char *)_ptr + size() + alignment()); } const Stack * callStack() const { return reinterpret_cast<const Stack *>((const char *)_ptr + size() + alignment()); } void fillMemory(size_t sz) { diff --git a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h index 242dd5d9fcc..88229aca77a 100644 --- a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h +++ b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h @@ -194,14 +194,14 @@ template <typename T, typename S> void MemoryWatcher<T, S>::activateOptions() { activateLogFile(_params[Params::logfile].value()); - T::dumpFile(_logFile); + _G_logFile = _logFile; this->setupSegmentLog(_params[Params::bigsegment_loglevel].valueAsLong(), _params[Params::bigsegment_limit].valueAsLong(), _params[Params::bigsegment_increment].valueAsLong(), _params[Params::allocs2show].valueAsLong()); this->setupLog(_params[Params::pralloc_loglimit].valueAsLong()); this->setParams(_params[Params::threadcachelimit].valueAsLong()); - T::bigBlockLimit(_params[Params::bigblocklimit].valueAsLong()); + _G_bigBlockLimit = _params[Params::bigblocklimit].valueAsLong(); T::setFill(_params[Params::fillvalue].valueAsLong()); } diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp new file mode 100644 index 00000000000..34232018671 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -0,0 +1,112 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespamalloc/malloc/mmappool.h> +#include <vespamalloc/malloc/common.h> +#include <cassert> +#include <sys/mman.h> + +namespace vespamalloc { + +MMapPool::MMapPool() + : _page_size(getpagesize()), + _huge_flags((getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0), + _count(0), + _mutex(), + _mappings() +{ + +} + +MMapPool::~MMapPool() { + assert(_mappings.empty()); +} + +size_t +MMapPool::getNumMappings() const { + std::lock_guard guard(_mutex); + return _mappings.size(); +} + +size_t +MMapPool::getMmappedBytes() const { + std::lock_guard guard(_mutex); + size_t sum(0); + std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); + return sum; +} + +void * +MMapPool::mmap(size_t sz) { + void * buf(nullptr); + assert((sz & (_page_size - 1)) == 0); + if (sz > 0) { + const int flags(MAP_ANON | MAP_PRIVATE); + const int prot(PROT_READ | PROT_WRITE); + size_t mmapId = _count.fetch_add(1); + if (sz >= _G_bigBlockLimit) { + fprintf(_G_logFile, "mmap %ld of size %ld from : ", mmapId, sz); + logStackTrace(); + } + buf = ::mmap(nullptr, sz, prot, flags | _huge_flags, -1, 0); + if (buf == MAP_FAILED) { + if (!_has_hugepage_failure_just_happened) { + _has_hugepage_failure_just_happened = true; + } + buf = ::mmap(nullptr, sz, prot, flags, -1, 0); + if (buf == MAP_FAILED) { + fprintf(_G_logFile, "Failed mmaping anonymous of size %ld errno(%d) from : ", sz, errno); + logStackTrace(); + abort(); + } + } else { + if (_has_hugepage_failure_just_happened) { + _has_hugepage_failure_just_happened = false; + } + } +#ifdef __linux__ + if (sz >= _G_bigBlockLimit) { + if (madvise(buf, sz, MADV_DONTDUMP) != 0) { + std::error_code ec(errno, std::system_category()); + fprintf(_G_logFile, "Failed madvise(%p, %ld, MADV_DONTDUMP) = '%s'\n", buf, sz, + ec.message().c_str()); + } + } +#endif + std::lock_guard guard(_mutex); + auto [it, inserted] = _mappings.insert(std::make_pair(buf, MMapInfo(mmapId, sz))); + assert(inserted); + if (sz >= _G_bigBlockLimit) { + size_t sum(0); + std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); + fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), sum); + } + } + return buf; +} + +void +MMapPool::unmap(void * ptr) { + size_t sz; + { + std::lock_guard guard(_mutex); + auto found = _mappings.find(ptr); + if (found == _mappings.end()) { + fprintf(_G_logFile, "Not able to unmap %p as it is not registered: ", ptr); + logStackTrace(); + abort(); + } + sz = found->second._sz; + _mappings.erase(found); + } + int munmap_ok = ::munmap(ptr, sz); + assert(munmap_ok == 0); +} + +size_t +MMapPool::get_size(void * ptr) const { + std::lock_guard guard(_mutex); + auto found = _mappings.find(ptr); + assert(found != _mappings.end()); + return found->second._sz; +} + +} diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h new file mode 100644 index 00000000000..aa8679c8171 --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <atomic> +#include <unordered_map> + +namespace vespamalloc { + +class MMapPool { +public: + MMapPool(); + MMapPool(const MMapPool &) = delete; + MMapPool & operator =(const MMapPool &) = delete; + ~MMapPool(); + void * mmap(size_t sz); + void unmap(void *); + size_t get_size(void *) const; + size_t getNumMappings() const; + size_t getMmappedBytes() const; +private: + struct MMapInfo { + MMapInfo(size_t id, size_t sz) : _id(id), _sz(sz) { } + size_t _id; + size_t _sz; + }; + const size_t _page_size; + const int _huge_flags; + std::atomic<size_t> _count; + std::atomic<bool> _has_hugepage_failure_just_happened; + mutable std::mutex _mutex; + std::unordered_map<const void *, MMapInfo> _mappings; +}; + +} diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 69d95ef5cdc..6650f107ca9 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -4,7 +4,7 @@ #include <dlfcn.h> #include <errno.h> #include <new> -#include <stdlib.h> +#include <cstdlib> #include <malloc.h> class CreateAllocator @@ -113,12 +113,12 @@ struct mallinfo2 mallinfo2() __THROW { info.arena = vespamalloc::_GmemP->dataSegment().dataSize(); info.ordblks = 0; info.smblks = 0; - info.hblks = 0; - info.hblkhd = 0; + info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); + info.hblks = vespamalloc::_GmemP->mmapPool().getMmappedBytes(); info.usmblks = 0; info.fsmblks = 0; - info.uordblks = 0; - info.fordblks = 0; + info.fordblks = vespamalloc::_GmemP->dataSegment().freeSize(); + info.uordblks = info.arena + info.hblks - info.fordblks; info.keepcost = 0; return info; } @@ -129,17 +129,22 @@ struct mallinfo mallinfo() __THROW { info.arena = (vespamalloc::_GmemP->dataSegment().dataSize() >> 20); // Note reporting in 1M blocks info.ordblks = 0; info.smblks = 0; - info.hblks = 0; - info.hblkhd = 0; + info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); + info.hblks = (vespamalloc::_GmemP->mmapPool().getMmappedBytes() >> 20); info.usmblks = 0; info.fsmblks = 0; - info.uordblks = 0; - info.fordblks = 0; + info.fordblks = (vespamalloc::_GmemP->dataSegment().freeSize() >> 20); + info.uordblks = info.arena + info.hblks - info.fordblks; info.keepcost = 0; return info; } #endif +int mallopt(int param, int value) throw() __attribute((visibility("default"))); +int mallopt(int param, int value) throw() { + return vespamalloc::createAllocator()->mallopt(param, value); +} + void * malloc(size_t sz) __attribute((visibility("default"))); void * malloc(size_t sz) { return vespamalloc::createAllocator()->malloc(sz); diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.h b/vespamalloc/src/vespamalloc/malloc/threadlist.h index c95760dc015..ca3a58483c9 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.h +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.h @@ -17,7 +17,9 @@ class ThreadListT public: using ThreadPool = ThreadPoolT<MemBlockPtrT, ThreadStatT >; using AllocPool = AllocPoolT<MemBlockPtrT>; - ThreadListT(AllocPool & pool); + ThreadListT(AllocPool & allocPool, MMapPool & mmapPool); + ThreadListT(const ThreadListT & tl) = delete; + ThreadListT & operator = (const ThreadListT & tl) = delete; ~ThreadListT(); void setParams(size_t threadCacheLimit) { ThreadPool::setParams(threadCacheLimit); @@ -33,13 +35,12 @@ public: void info(FILE * os, size_t level=0); size_t getMaxNumThreads() const { return NELEMS(_threadVector); } private: - ThreadListT(const ThreadListT & tl); - ThreadListT & operator = (const ThreadListT & tl); std::atomic_flag _isThreaded; std::atomic<uint32_t> _threadCount; std::atomic<uint32_t> _threadCountAccum; ThreadPool _threadVector[NUM_THREADS]; AllocPoolT<MemBlockPtrT> & _allocPool; + MMapPool & _mmapPool; static thread_local ThreadPool * _myPool TLS_LINKAGE; }; diff --git a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp index 8a2cb1de879..5f65e98d0ac 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadlist.hpp +++ b/vespamalloc/src/vespamalloc/malloc/threadlist.hpp @@ -6,14 +6,15 @@ namespace vespamalloc { template <typename MemBlockPtrT, typename ThreadStatT> -ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & pool) : +ThreadListT<MemBlockPtrT, ThreadStatT>::ThreadListT(AllocPool & allocPool, MMapPool & mmapPool) : _isThreaded(false), _threadCount(0), _threadCountAccum(0), - _allocPool(pool) + _allocPool(allocPool), + _mmapPool(mmapPool) { for (size_t i = 0; i < getMaxNumThreads(); i++) { - _threadVector[i].setPool(_allocPool); + _threadVector[i].setPool(_allocPool, _mmapPool); } } diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.h b/vespamalloc/src/vespamalloc/malloc/threadpool.h index ac4f5bb8551..f49e6cf24af 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadpool.h +++ b/vespamalloc/src/vespamalloc/malloc/threadpool.h @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "common.h" +#include "allocchunk.h" +#include "globalpool.h" +#include "mmappool.h" #include <atomic> -#include <vespamalloc/malloc/common.h> -#include <vespamalloc/malloc/allocchunk.h> -#include <vespamalloc/malloc/globalpool.h> namespace vespamalloc { @@ -12,17 +13,20 @@ template <typename MemBlockPtrT, typename ThreadStatT > class ThreadPoolT { public: - typedef AFList<MemBlockPtrT> ChunkSList; - typedef AllocPoolT<MemBlockPtrT> AllocPool; + using ChunkSList = AFList<MemBlockPtrT>; + using AllocPool = AllocPoolT<MemBlockPtrT>; + using DataSegment = segment::DataSegment; ThreadPoolT(); ~ThreadPoolT(); - void setPool(AllocPool & pool) { - _allocPool = & pool; + void setPool(AllocPool & allocPool, MMapPool & mmapPool) { + _allocPool = & allocPool; + _mmapPool = & mmapPool; } + int mallopt(int param, int value); void malloc(size_t sz, MemBlockPtrT & mem); void free(MemBlockPtrT mem, SizeClassT sc); - void info(FILE * os, size_t level, const DataSegment<MemBlockPtrT> & ds) const __attribute__((noinline)); + void info(FILE * os, size_t level, const DataSegment & ds) const __attribute__((noinline)); /** * Indicates if it represents an active thread. * @return true if this represents an active thread. @@ -65,6 +69,8 @@ private: static constexpr bool alwaysReuse(SizeClassT sc) { return sc > ALWAYS_REUSE_SC_LIMIT; } AllocPool * _allocPool; + MMapPool * _mmapPool; + size_t _mmapLimit; AllocFree _memList[NUM_SIZE_CLASSES]; ThreadStatT _stat[NUM_SIZE_CLASSES]; uint32_t _threadId; diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp index 7e86c3f691a..7d177c4c129 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadpool.hpp +++ b/vespamalloc/src/vespamalloc/malloc/threadpool.hpp @@ -2,14 +2,21 @@ #pragma once #include <vespamalloc/malloc/threadpool.h> +#include <malloc.h> namespace vespamalloc { +namespace { + constexpr size_t MMAP_LIMIT_MIN = 0x100000; // 1M + constexpr size_t MMAP_LIMIT_MAX = 0x40000000; // 1G +} + template <typename MemBlockPtrT, typename ThreadStatT> size_t ThreadPoolT<MemBlockPtrT, ThreadStatT>::_threadCacheLimit __attribute__((visibility("hidden"))) = 0x10000; template <typename MemBlockPtrT, typename ThreadStatT> -void ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const DataSegment<MemBlockPtrT> & ds) const { +void +ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const DataSegment & ds) const { if (level > 0) { for (size_t i=0; i < NELEMS(_stat); i++) { const ThreadStatT & s = _stat[i]; @@ -49,7 +56,8 @@ void ThreadPoolT<MemBlockPtrT, ThreadStatT>::info(FILE * os, size_t level, const } template <typename MemBlockPtrT, typename ThreadStatT > -void ThreadPoolT<MemBlockPtrT, ThreadStatT>:: +void +ThreadPoolT<MemBlockPtrT, ThreadStatT>:: mallocHelper(size_t exactSize, SizeClassT sc, typename ThreadPoolT<MemBlockPtrT, ThreadStatT>::AllocFree & af, @@ -70,13 +78,20 @@ mallocHelper(size_t exactSize, PARANOID_CHECK2( *(int *)2 = 2; ); } } else { - af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom); - _stat[sc].incExactAlloc(); - if (af._allocFrom) { - af._allocFrom->sub(mem); - PARANOID_CHECK2( if (!mem.ptr()) { *(int *)3 = 3; } ); + if (exactSize > _mmapLimit) { + mem = MemBlockPtrT(_mmapPool->mmap(MemBlockPtrT::classSize(sc)), MemBlockPtrT::classSize(sc)); + // The below settings are to allow the sanity checks conducted at the call site to succeed + mem.setExact(exactSize); + mem.free(); } else { - PARANOID_CHECK2( *(int *)4 = 4; ); + af._allocFrom = _allocPool->exactAlloc(exactSize, sc, af._allocFrom); + _stat[sc].incExactAlloc(); + if (af._allocFrom) { + af._allocFrom->sub(mem); + PARANOID_CHECK2(if (!mem.ptr()) { *(int *) 3 = 3; }); + } else { + PARANOID_CHECK2(*(int *) 4 = 4;); + } } } } @@ -85,6 +100,8 @@ mallocHelper(size_t exactSize, template <typename MemBlockPtrT, typename ThreadStatT > ThreadPoolT<MemBlockPtrT, ThreadStatT>::ThreadPoolT() : _allocPool(nullptr), + _mmapPool(nullptr), + _mmapLimit(MMAP_LIMIT_MAX), _threadId(0), _osThreadId(0) { @@ -94,6 +111,16 @@ template <typename MemBlockPtrT, typename ThreadStatT > ThreadPoolT<MemBlockPtrT, ThreadStatT>::~ThreadPoolT() = default; template <typename MemBlockPtrT, typename ThreadStatT > +int ThreadPoolT<MemBlockPtrT, ThreadStatT>::mallopt(int param, int value) { + size_t limit = value; + if (param == M_MMAP_THRESHOLD) { + _mmapLimit = std::min(MMAP_LIMIT_MAX, std::max(MMAP_LIMIT_MIN, limit)); + return 1; + } + return 0; +} + +template <typename MemBlockPtrT, typename ThreadStatT > void ThreadPoolT<MemBlockPtrT, ThreadStatT>::malloc(size_t sz, MemBlockPtrT & mem) { SizeClassT sc = MemBlockPtrT::sizeClass(sz); diff --git a/vespamalloc/src/vespamalloc/util/callgraph.h b/vespamalloc/src/vespamalloc/util/callgraph.h index 94df3bd5e92..2d66fc8b717 100644 --- a/vespamalloc/src/vespamalloc/util/callgraph.h +++ b/vespamalloc/src/vespamalloc/util/callgraph.h @@ -11,7 +11,7 @@ template<typename T, typename AddSub> class CallGraphNode { public: - CallGraphNode() : _callers(NULL), _next(NULL), _content(), _count(0) { } + CallGraphNode() : _callers(nullptr), _next(nullptr), _content(), _count(0) { } const CallGraphNode *next() const { return _next; } const CallGraphNode *callers() const { return _callers; } const T & content() const { return _content; } @@ -21,7 +21,7 @@ public: size_t count() const { return _count; } void content(const T & v) { _content = v; } template <typename Store> - bool addStack(T * stack, size_t nelem, Store & store); + bool addStack(const T * stack, size_t nelem, Store & store); template<typename Object> void traverseDepth(size_t depth, size_t width, Object func); template<typename Object> @@ -38,16 +38,16 @@ private: template<typename T, typename AddSub> template <typename Store> -bool CallGraphNode<T, AddSub>::addStack(T * stack, size_t nelem, Store & store) { +bool CallGraphNode<T, AddSub>::addStack(const T * stack, size_t nelem, Store & store) { bool retval(false); if (nelem == 0) { retval = true; } else if (_content == stack[0]) { _count++; if (nelem > 1) { - if (_callers == NULL) { + if (_callers == nullptr) { _callers = store.alloc(); - if (_callers != NULL) { + if (_callers != nullptr) { _callers->content(stack[1]); } } @@ -58,9 +58,9 @@ bool CallGraphNode<T, AddSub>::addStack(T * stack, size_t nelem, Store & store) retval = true; } } else { - if (_next == NULL) { + if (_next == nullptr) { _next = store.alloc(); - if (_next != NULL) { + if (_next != nullptr) { _next->content(stack[0]); } } @@ -102,7 +102,7 @@ class ArrayStore { public: ArrayStore() : _used(0) { } - T * alloc() { return (_used < MaxElem) ? &_array[_used++] : NULL; } + T * alloc() { return (_used < MaxElem) ? &_array[_used++] : nullptr; } AddSub size() const { return _used; } private: AddSub _used; @@ -113,19 +113,19 @@ template <typename Content, size_t MaxElems, typename AddSub> class CallGraph { public: - typedef CallGraphNode<Content, AddSub> Node; + using Node = CallGraphNode<Content, AddSub>; CallGraph() : - _root(NULL), - _nodeStore(new NodeStore()) + _root(nullptr), + _nodeStore(std::make_unique<NodeStore>()) { } CallGraph(Content root) : - _root(NULL), - _nodeStore(new NodeStore()) + _root(nullptr), + _nodeStore(std::make_unique<NodeStore>()) { checkOrSetRoot(root); } - bool addStack(Content * stack, size_t nelem) { + bool addStack(const Content * stack, size_t nelem) { checkOrSetRoot(stack[0]); return _root->addStack(stack, nelem, *_nodeStore); } @@ -143,14 +143,14 @@ private: CallGraph(const CallGraph &); CallGraph & operator = (const CallGraph &); bool checkOrSetRoot(const Content & root) { - if (_root == NULL) { + if (_root == nullptr) { _root = _nodeStore->alloc(); _root->content(root); } - return (_root != NULL); + return (_root != nullptr); } typedef ArrayStore<Node, MaxElems, AddSub> NodeStore; - Node * _root; + Node * _root; std::unique_ptr<NodeStore> _nodeStore; }; diff --git a/vespamalloc/src/vespamalloc/util/callstack.cpp b/vespamalloc/src/vespamalloc/util/callstack.cpp index 9be2aa8133e..629f6427ab3 100644 --- a/vespamalloc/src/vespamalloc/util/callstack.cpp +++ b/vespamalloc/src/vespamalloc/util/callstack.cpp @@ -5,7 +5,8 @@ namespace vespamalloc { -const char * dlAddr(const void * func) { +const char * +dlAddr(const void * func) { static const char * _unknown = "UNKNOWN"; const char * funcName = _unknown; Dl_info info; @@ -16,25 +17,29 @@ const char * dlAddr(const void * func) { return funcName; } -static void verifyAndCopy(const void * addr, char *v, size_t sz) -{ +namespace { +void +verifyAndCopy(const void *addr, char *v, size_t sz) { size_t pos(0); - const char * sym = dlAddr(addr); - for (;sym && (sym[pos] != '\0') && (pos < sz-1); pos++) { + const char *sym = dlAddr(addr); + for (; sym && (sym[pos] != '\0') && (pos < sz - 1); pos++) { char c(sym[pos]); v[pos] = isprint(c) ? c : '.'; } v[pos] = '\0'; } +} -void StackReturnEntry::info(FILE * os) const +void +StackReturnEntry::info(FILE * os) const { static char tmp[0x400]; verifyAndCopy(_return, tmp, sizeof(tmp)); fprintf(os, "%s(%p)", tmp, _return); } -asciistream & operator << (asciistream & os, const StackReturnEntry & v) +asciistream & +operator << (asciistream & os, const StackReturnEntry & v) { static char tmp[0x100]; static char t[0x200]; @@ -43,4 +48,30 @@ asciistream & operator << (asciistream & os, const StackReturnEntry & v) return os << t; } +const void * StackEntry::_stopAddr = nullptr; + +size_t +StackEntry::fillStack(StackEntry *stack, size_t nelems) +{ + void * retAddr[nelems]; + int sz = backtrace(retAddr, nelems); + if ((sz > 0) && (size_t(sz) <= nelems)) { + for(int i(1); i < sz; i++) { + StackEntry entry(retAddr[i], nullptr); + if (entry.valid()) { + stack[i-1] = entry; + } else { + sz = i; + } + } + sz -= 1; // Do not count self + } else { + sz = 0; + } + if (size_t(sz) < nelems) { + stack[sz] = StackEntry(); + } + return sz; +} + } diff --git a/vespamalloc/src/vespamalloc/util/callstack.h b/vespamalloc/src/vespamalloc/util/callstack.h index 89feb01c4b2..3a2986a0a89 100644 --- a/vespamalloc/src/vespamalloc/util/callstack.h +++ b/vespamalloc/src/vespamalloc/util/callstack.h @@ -13,8 +13,8 @@ const char * dlAddr(const void * addr); class StackReturnEntry { public: - StackReturnEntry(const void * returnAddress = NULL, - const void * stack=NULL) + StackReturnEntry(const void * returnAddress = nullptr, + const void * stack=nullptr) : _return(returnAddress) { (void) stack; @@ -23,7 +23,7 @@ public: return (size_t(_return) - size_t(b._return)); } void info(FILE * os) const; - bool valid() const { return _return != NULL; } + bool valid() const { return _return != nullptr; } bool valid(const void * stopAddr) const { return valid() && (_return != stopAddr); } bool valid(const void * stopAddrMin, const void * stopAddrMax) const { return valid() && ! ((stopAddrMin <= _return) && (_return < stopAddrMax)); } private: @@ -31,11 +31,10 @@ private: const void * _return; }; -template <typename StackRep> class StackEntry { public: - StackEntry(const void * returnAddress = NULL, - const void * stack = NULL) + StackEntry(const void * returnAddress = nullptr, + const void * stack = nullptr) : _stackRep(returnAddress, stack) { } bool operator == (const StackEntry & b) const { return cmp(b) == 0; } @@ -47,39 +46,12 @@ public: static void setStopAddress(const void * stopAddr) { _stopAddr = stopAddr; } private: int cmp(const StackEntry & b) const { return _stackRep.cmp(b._stackRep); } - friend asciistream & operator << (asciistream & os, const StackEntry<StackRep> & v) { + friend asciistream & operator << (asciistream & os, const StackEntry & v) { return os << v._stackRep; } - StackRep _stackRep; + StackReturnEntry _stackRep; static const void * _stopAddr; }; -template <typename StackRep> -const void * StackEntry<StackRep>::_stopAddr = NULL; - -template <typename StackRep> -size_t StackEntry<StackRep>::fillStack(StackEntry<StackRep> *stack, size_t nelems) -{ - void * retAddr[nelems]; - int sz = backtrace(retAddr, nelems); - if ((sz > 0) && (size_t(sz) <= nelems)) { - for(int i(1); i < sz; i++) { - StackEntry<StackRep> entry(retAddr[i], NULL); - if (entry.valid()) { - stack[i-1] = entry; - } else { - sz = i; - } - } - sz -= 1; // Do not count self - } else { - sz = 0; - } - if (size_t(sz) < nelems) { - stack[sz] = StackEntry<StackRep>(); - } - return sz; -} - } diff --git a/vespamalloc/src/vespamalloc/util/stream.cpp b/vespamalloc/src/vespamalloc/util/stream.cpp index 4bf2e9aed9e..4af6ab56aba 100644 --- a/vespamalloc/src/vespamalloc/util/stream.cpp +++ b/vespamalloc/src/vespamalloc/util/stream.cpp @@ -5,18 +5,50 @@ namespace vespamalloc { -asciistream::asciistream() : - _rPos(0), - _wPos(0), - _buffer(static_cast<char *>(malloc(1024))), - _sz(1024) +asciistream::asciistream() + : _rPos(0), + _wPos(0), + _buffer(static_cast<char *>(malloc(1024))), + _sz(1024) { } asciistream::~asciistream() { - free(_buffer); - _buffer = nullptr; + if (_buffer != nullptr) { + free(_buffer); + _buffer = nullptr; + } +} + +asciistream::asciistream(asciistream && rhs) noexcept + : _rPos(rhs._rPos), + _wPos(rhs._wPos), + _buffer(rhs._buffer), + _sz(rhs._sz) +{ + rhs._rPos = 0; + rhs._wPos = 0; + rhs._sz = 0; + rhs._buffer = nullptr; +} + +asciistream & +asciistream::operator = (asciistream && rhs) noexcept +{ + if (this != &rhs) { + if (_buffer) free(_buffer); + + _rPos = rhs._rPos; + _wPos = rhs._wPos; + _buffer = rhs._buffer; + _sz = rhs._sz; + rhs._rPos = 0; + rhs._wPos = 0; + rhs._sz = 0; + rhs._buffer = nullptr; + } + return *this; } asciistream::asciistream(const asciistream & rhs) : @@ -29,7 +61,8 @@ asciistream::asciistream(const asciistream & rhs) : _buffer[_wPos] = 0; } -asciistream & asciistream::operator = (const asciistream & rhs) +asciistream & +asciistream::operator = (const asciistream & rhs) { if (this != &rhs) { asciistream newStream(rhs); @@ -38,6 +71,7 @@ asciistream & asciistream::operator = (const asciistream & rhs) return *this; } + void asciistream::swap(asciistream & rhs) { std::swap(_rPos, rhs._rPos); diff --git a/vespamalloc/src/vespamalloc/util/stream.h b/vespamalloc/src/vespamalloc/util/stream.h index 5d9750be827..34af28486dd 100644 --- a/vespamalloc/src/vespamalloc/util/stream.h +++ b/vespamalloc/src/vespamalloc/util/stream.h @@ -11,6 +11,8 @@ class asciistream public: asciistream(); ~asciistream(); + asciistream(asciistream && rhs) noexcept; + asciistream & operator = (asciistream && ) noexcept; asciistream(const asciistream & rhs); asciistream & operator = (const asciistream & rhs); void swap(asciistream & rhs); diff --git a/vespamalloc/src/vespamalloc/util/traceutil.cpp b/vespamalloc/src/vespamalloc/util/traceutil.cpp index 49cde544732..36698125f97 100644 --- a/vespamalloc/src/vespamalloc/util/traceutil.cpp +++ b/vespamalloc/src/vespamalloc/util/traceutil.cpp @@ -4,13 +4,9 @@ namespace vespamalloc { -Aggregator::Aggregator() -{ -} +Aggregator::Aggregator() = default; -Aggregator::~Aggregator() -{ -} +Aggregator::~Aggregator() = default; struct CmpGraph { @@ -23,8 +19,8 @@ asciistream & operator << (asciistream & os, const Aggregator & v) { Aggregator::Map map(v._map); std::sort(map.begin(), map.end(), CmpGraph()); - for (Aggregator::Map::const_iterator it=map.begin(); it != map.end(); it++) { - os << it->first << " : " << it->second.c_str() << '\n'; + for (const auto & e : map) { + os << e.first << " : " << e.second.c_str() << '\n'; } return os; } diff --git a/vespamalloc/src/vespamalloc/util/traceutil.h b/vespamalloc/src/vespamalloc/util/traceutil.h index 15fb8e221c8..fe424d7259e 100644 --- a/vespamalloc/src/vespamalloc/util/traceutil.h +++ b/vespamalloc/src/vespamalloc/util/traceutil.h @@ -13,7 +13,7 @@ namespace vespamalloc { -typedef StackEntry<StackReturnEntry> StackElem; +using StackElem = StackEntry; typedef CallGraph<StackElem, 0x10000, Index> CallGraphT; class Aggregator @@ -21,7 +21,7 @@ class Aggregator public: Aggregator(); ~Aggregator(); - void push_back(size_t num, const string & s) { _map.push_back(Map::value_type(num, s)); } + void push_back(size_t num, const string & s) { _map.emplace_back(num, s); } friend asciistream & operator << (asciistream & os, const Aggregator & v); private: typedef std::vector< std::pair<size_t, string> > Map; @@ -57,9 +57,7 @@ DumpGraph<N>::DumpGraph(Aggregator * aggregator, const char * start, const char } template<typename N> -DumpGraph<N>::~DumpGraph() -{ -} +DumpGraph<N>::~DumpGraph() = default; template<typename N> void DumpGraph<N>::handle(const N & node) @@ -71,7 +69,7 @@ void DumpGraph<N>::handle(const N & node) asciistream os; os << ' ' << node; _string += os.c_str(); - if (node.callers() == NULL) { + if (node.callers() == nullptr) { _string += _endString; _aggregator->push_back(_min, _string); } diff --git a/vsm/src/vespa/vsm/config/vsmfields.def b/vsm/src/vespa/vsm/config/vsmfields.def index 21d7d72eb0b..5e943c9274d 100644 --- a/vsm/src/vespa/vsm/config/vsmfields.def +++ b/vsm/src/vespa/vsm/config/vsmfields.def @@ -12,7 +12,7 @@ searchall int default=1 fieldspec[].name string ## The search method for a given field. Note: same field in 2 different document types must match on type if not a random result might be expected. -fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE } default=AUTOUTF8 +fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, GEOPOS } default=AUTOUTF8 fieldspec[].arg1 string default="" ## Maximum number of chars to search per field. diff --git a/vsm/src/vespa/vsm/searcher/CMakeLists.txt b/vsm/src/vespa/vsm/searcher/CMakeLists.txt index 0fab39c5586..0a2a9ec21d2 100644 --- a/vsm/src/vespa/vsm/searcher/CMakeLists.txt +++ b/vsm/src/vespa/vsm/searcher/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(vsm_vsmsearcher OBJECT floatfieldsearcher.cpp ${SSE2_FILES} futf8strchrfieldsearcher.cpp + geo_pos_field_searcher.cpp intfieldsearcher.cpp strchrfieldsearcher.cpp utf8flexiblestringfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp index b48c5e7d493..c587f9eadfb 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp @@ -295,6 +295,7 @@ void FieldSearcher::IteratorHandler::onStructStart(const Content & c) { LOG(spam, "onStructStart: field value '%s'", c.getValue().toString().c_str()); + _searcher.onStructValue(static_cast<const document::StructFieldValue &>(c.getValue())); } diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/vsm/src/vespa/vsm/searcher/fieldsearcher.h index eb1f3ddd524..5c2ef8fec28 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h +++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.h @@ -99,6 +99,7 @@ private: void setCurrentElementId(int32_t weight) { _currentElementId = weight; } bool onSearch(const StorageDocument & doc); virtual void onValue(const document::FieldValue & fv) = 0; + virtual void onStructValue(const document::StructFieldValue &) { } FieldIdT _field; MatchType _matchType; unsigned _maxFieldLength; diff --git a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp b/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp new file mode 100644 index 00000000000..db93bda7778 --- /dev/null +++ b/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "geo_pos_field_searcher.h" +#include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/structfieldvalue.h> +#include <vespa/searchlib/common/geo_location_parser.h> +#include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/util/exception.h> + +#include <vespa/log/log.h> +LOG_SETUP(".vsm.searcher.geo_pos_field_searcher"); + +using search::streaming::QueryTerm; +using search::streaming::QueryTermList; +using search::common::GeoLocation; +using search::common::GeoLocationParser; + +namespace vsm { + +std::unique_ptr<FieldSearcher> GeoPosFieldSearcher::duplicate() const { + return std::make_unique<GeoPosFieldSearcher>(*this); +} + +GeoPosFieldSearcher::GeoPosFieldSearcher(FieldIdT fId) : + FieldSearcher(fId), + _geoPosTerm() +{} + +GeoPosFieldSearcher::~GeoPosFieldSearcher() {} + +void GeoPosFieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) { + _geoPosTerm.clear(); + FieldSearcher::prepare(qtl, buf); + for (const QueryTerm * qt : qtl) { + const vespalib::string & str = qt->getTermString(); + GeoLocationParser parser; + bool valid = parser.parseNoField(str); + if (! valid) { + vespalib::Issue::report("invalid position in term: %s", str.c_str()); + } + _geoPosTerm.emplace_back(parser.getGeoLocation()); + } +} + +void GeoPosFieldSearcher::onValue(const document::FieldValue & fv) { + LOG(spam, "ignore field value '%s'", fv.toString().c_str()); +} + +void GeoPosFieldSearcher::onStructValue(const document::StructFieldValue & fv) { + size_t num_terms = _geoPosTerm.size(); + for (size_t j = 0; j < num_terms; ++j) { + const GeoPosInfo & gpi = _geoPosTerm[j]; + if (gpi.valid() && gpi.cmp(fv)) { + addHit(*_qtl[j], 0); + } + } + ++_words; +} + +bool GeoPosFieldSearcher::GeoPosInfo::cmp(const document::StructFieldValue & sfv) const { + try { + auto xv = sfv.getValue("x"); + auto yv = sfv.getValue("y"); + if (xv && yv) { + int32_t x = xv->getAsInt(); + int32_t y = yv->getAsInt(); + GeoLocation::Point p{x,y}; + if (inside_limit(p)) { + return true; + } + } + } catch (const vespalib::Exception &e) { + vespalib::Issue::report("bad fieldvalue for GeoPosFieldSearcher: %s", e.getMessage().c_str()); + } + return false; +} + +} diff --git a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.h b/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.h new file mode 100644 index 00000000000..ef1c5b5a1c4 --- /dev/null +++ b/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "fieldsearcher.h" +#include <vespa/searchlib/common/geo_location.h> + +namespace vsm { + +class GeoPosFieldSearcher : public FieldSearcher { +public: + GeoPosFieldSearcher(FieldIdT fId=0); + ~GeoPosFieldSearcher(); + void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void onValue(const document::FieldValue & fv) override; + void onStructValue(const document::StructFieldValue & fv) override; + std::unique_ptr<FieldSearcher> duplicate() const override; +protected: + using GeoLocation = search::common::GeoLocation; + class GeoPosInfo : public GeoLocation { + public: + GeoPosInfo (GeoLocation loc) noexcept : GeoLocation(std::move(loc)) {} + bool cmp(const document::StructFieldValue & fv) const; + }; + typedef std::vector<GeoPosInfo> GeoPosInfoListT; + GeoPosInfoListT _geoPosTerm; +}; + +} diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index 8fe85a9023a..7043e63ec87 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -10,6 +10,7 @@ #include <vespa/vsm/searcher/intfieldsearcher.h> #include <vespa/vsm/searcher/boolfieldsearcher.h> #include <vespa/vsm/searcher/floatfieldsearcher.h> +#include <vespa/vsm/searcher/geo_pos_field_searcher.h> #include <vespa/vespalib/stllike/asciistream.h> #include <regex> @@ -105,6 +106,9 @@ FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & case VsmfieldsConfig::Fieldspec::Searchmethod::DOUBLE: _searcher = std::make_unique<DoubleFieldSearcher>(fid); break; + case VsmfieldsConfig::Fieldspec::Searchmethod::GEOPOS: + _searcher = std::make_unique<GeoPosFieldSearcher>(fid); + break; } if (_searcher) { setMatchType(_searcher, arg1); diff --git a/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp b/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp index 231be9b6f73..405f2292f5d 100644 --- a/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp +++ b/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp @@ -4,6 +4,8 @@ #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> +#include <vespa/document/datatype/positiondatatype.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.slimefieldwriter"); @@ -93,6 +95,35 @@ SlimeFieldWriter::traverseRecursive(const document::FieldValue & fv, Inserter &i } else if (clazz.inherits(document::StructuredFieldValue::classId)) { const document::StructuredFieldValue & sfv = static_cast<const document::StructuredFieldValue &>(fv); Cursor &o = inserter.insertObject(); + if (sfv.getDataType() == &document::PositionDataType::getInstance() + && search::docsummary::ResultConfig::wantedV8geoPositions()) + { + bool ok = true; + try { + int x = std::numeric_limits<int>::min(); + int y = std::numeric_limits<int>::min(); + for (const document::Field & entry : sfv) { + document::FieldValue::UP fval(sfv.getValue(entry)); + if (entry.getName() == "x") { + x = fval->getAsInt(); + } else if (entry.getName() == "y") { + y = fval->getAsInt(); + } else { + ok = false; + } + } + if (x == std::numeric_limits<int>::min()) ok = false; + if (y == std::numeric_limits<int>::min()) ok = false; + if (ok) { + o.setDouble("lat", double(y) / 1.0e6); + o.setDouble("lng", double(x) / 1.0e6); + return; + } + } catch (std::exception &e) { + (void)e; + // fallback to code below + } + } for (const document::Field & entry : sfv) { if (explorePath(entry.getName())) { _currPath.push_back(entry.getName()); diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index 784b5ea38ba..09594832c4a 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "vsm-adapter.h" +#include "vsm-adapter.hpp" #include "docsumconfig.h" #include "i_matching_elements_filler.h" #include <vespa/searchlib/common/matching_elements.h> @@ -24,7 +24,7 @@ GetDocsumsStateCallback::GetDocsumsStateCallback() : void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) { (void) env; - if (_summaryFeatures.get() != NULL) { // set the summary features to write to the docsum + if (_summaryFeatures) { // set the summary features to write to the docsum state->_summaryFeatures = _summaryFeatures; state->_summaryFeaturesCached = true; } @@ -33,7 +33,7 @@ void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState * state, IDocs void GetDocsumsStateCallback::FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) { (void) env; - if (_rankFeatures.get() != NULL) { // set the rank features to write to the docsum + if (_rankFeatures) { // set the rank features to write to the docsum state->_rankFeatures = _rankFeatures; } } @@ -119,10 +119,10 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) std::lock_guard guard(_lock); LOG(debug, "(re-)configure VSM (docsum tools)"); - std::shared_ptr<SummaryConfig> summary(snapshot.getConfig<SummaryConfig>().release()); - std::shared_ptr<SummarymapConfig> summaryMap(snapshot.getConfig<SummarymapConfig>().release()); - std::shared_ptr<VsmsummaryConfig> vsmSummary(snapshot.getConfig<VsmsummaryConfig>().release()); - std::shared_ptr<JuniperrcConfig> juniperrc(snapshot.getConfig<JuniperrcConfig>().release()); + std::shared_ptr<SummaryConfig> summary(snapshot.getConfig<SummaryConfig>()); + std::shared_ptr<SummarymapConfig> summaryMap(snapshot.getConfig<SummarymapConfig>()); + std::shared_ptr<VsmsummaryConfig> vsmSummary(snapshot.getConfig<VsmsummaryConfig>()); + std::shared_ptr<JuniperrcConfig> juniperrc(snapshot.getConfig<JuniperrcConfig>()); _fieldsCfg.set(snapshot.getConfig<VsmfieldsConfig>().release()); _fieldsCfg.latch(); @@ -175,6 +175,12 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) } } +VSMConfigSnapshot::VSMConfigSnapshot(const vespalib::string & configId, const config::ConfigSnapshot & snapshot) + : _configId(configId), + _snapshot(std::make_unique<config::ConfigSnapshot>(snapshot)) +{ } +VSMConfigSnapshot::~VSMConfigSnapshot() = default; + VSMAdapter::VSMAdapter(const vespalib::string & highlightindexes, const vespalib::string & configId, Fast_WordFolder & wordFolder) : _highlightindexes(highlightindexes), _configId(configId), diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index dacf01aec08..6484269353b 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/query/base.h> -#include <vespa/config/retriever/configsnapshot.h> #include <vespa/vsm/config/vsm-cfif.h> #include <vespa/config-summary.h> #include <vespa/config-summarymap.h> @@ -25,6 +24,7 @@ using vespa::config::search::SummaryConfig; using vespa::config::search::SummarymapConfig; using vespa::config::search::summary::JuniperrcConfig; +namespace config { class ConfigSnapshot; } namespace vsm { class IMatchingElementsFiller; @@ -97,17 +97,12 @@ typedef std::shared_ptr<DocsumTools> DocsumToolsPtr; class VSMConfigSnapshot { private: const vespalib::string _configId; - const config::ConfigSnapshot _snapshot; + std::unique_ptr<const config::ConfigSnapshot> _snapshot; public: - VSMConfigSnapshot(const vespalib::string & configId, const config::ConfigSnapshot & snapshot) - : _configId(configId), - _snapshot(snapshot) - { } + VSMConfigSnapshot(const vespalib::string & configId, const config::ConfigSnapshot & snapshot); + ~VSMConfigSnapshot(); template <typename ConfigType> - std::unique_ptr<ConfigType> getConfig() const - { - return _snapshot.getConfig<ConfigType>(_configId); - } + std::unique_ptr<ConfigType> getConfig() const; }; class VSMAdapter diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.hpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.hpp new file mode 100644 index 00000000000..f071dbb2015 --- /dev/null +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.hpp @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "vsm-adapter.h" +#include <vespa/config/retriever/configsnapshot.hpp> + +namespace vsm { + +template <typename ConfigType> +std::unique_ptr<ConfigType> +VSMConfigSnapshot::getConfig() const +{ + return _snapshot->getConfig<ConfigType>(_configId); +} + +} // namespace vsm + diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java b/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java index 282c357f277..516e3786f28 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java @@ -14,7 +14,9 @@ import static java.util.Objects.requireNonNull; * An immutable and ordered list of components * * @author Tony Vaagenes + * @deprecated Will be removed in Vespa 8 with no replacement. */ +@Deprecated public final class Chain<T> implements Iterable<T> { private final String id; diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java b/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java index cbc9897d3c9..cb79c05ddab 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java @@ -12,7 +12,9 @@ import java.util.Set; /** * @author Tony Vaagenes * @author gjoranv + * @deprecated Will be removed in Vespa 8 with no replacement. */ +@Deprecated public final class ChainBuilder<T> { private final String chainId; diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java b/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java index f29020321c8..6d8e14d696d 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java @@ -9,7 +9,9 @@ import java.util.List; /** * @author Tony Vaagenes + * @deprecated Will be removed in Vespa 8 with no replacement. */ +@Deprecated public class ChainCycleException extends RuntimeException { private final List<?> components; diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java b/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java index 5cee9235397..57f74190aff 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java @@ -12,7 +12,9 @@ import java.util.List; /** * @author Tony Vaagenes * @author gjoranv + * @deprecated Will be removed in Vespa 8 with no replacement. */ +@Deprecated public class Dependencies<T> { final Order<T> before; diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java b/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java index acab42a7090..0a481042af4 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java @@ -13,6 +13,7 @@ import java.util.Set; * * @author Tony Vaagenes */ +@Deprecated class DirectedGraph { private IdentityHashMap<Vertex, List<Vertex>> incommingEdges = new IdentityHashMap<>(); diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java b/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java index feece288bad..220696e6c5f 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java @@ -20,6 +20,7 @@ import java.util.TreeMap; * * @author Tony Vaagenes */ +@Deprecated class EnumeratedIdentitySet<T> implements Set<T> { private int counter = 0; diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java b/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java index e48a3fea197..5a506406ebf 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java @@ -4,6 +4,7 @@ package com.yahoo.yolean.chain; /** * @author Tony Vaagenes */ +@Deprecated interface Vertex { } diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java deleted file mode 100644 index 1d5b6a6cac7..00000000000 --- a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import static com.yahoo.yolean.chain.Dependencies.after; -import static com.yahoo.yolean.chain.Dependencies.before; -import static com.yahoo.yolean.chain.Dependencies.provides; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Tony Vaagenes - * @author gjoranv - */ -public class ChainBuilderTest { - - static class Filter { - - } - - static class FilterA extends Filter { - - } - - static class FilterB extends Filter { - - } - - static class FilterExtendsA extends FilterA { - - } - - static class FilterExtendsB extends FilterB { - - } - - @Provides("A") - static class ProvidesA extends Filter { - - } - - @Provides("B") - static class ProvidesB extends Filter { - - } - - @Before("A") - static class BeforeA extends Filter { - - } - - @After("A") - static class AfterA extends Filter { - - } - - @Before("*") - @Provides("BeforeAll") - static class BeforeAll extends Filter { - - } - - @After("*") - @Provides("AfterAll") - static class AfterAll extends Filter { - - } - - @Before({ "BeforeAll", "*" }) - static class BeforeBeforeAll extends Filter { - - } - - @After({ "AfterAll", "*" }) - static class AfterAfterAll extends Filter { - - } - - static class ExtendsProvidesA extends ProvidesA { - - } - - @Provides("ExtendsA") - static class ProvidesA_and_ProvidesExtendsA extends ProvidesA { - - } - - @Before("B") - static class BeforeA_and_BeforeB extends BeforeA { - - } - - @Test - public void build_empty_chain() { - Chain<Filter> chain = getChain().build(); - assertTrue(chain.isEmpty()); - assertEquals("myChain", chain.id()); - } - - boolean equalOrder(Iterator<Filter> a, Iterator<Filter> b) { - while (a.hasNext() && b.hasNext()) { - if ( ! a.next().equals(b.next())) return false; - } - return a.hasNext() == b.hasNext(); - } - - @Test - public void filters_without_dependencies_are_not_reordered() { - List<Filter> filters = new ArrayList<>(); - - ChainBuilder<Filter> chain = new ChainBuilder<>("myChain"); - for (int i = 0; i < 10; ++i) { - Filter filter = new Filter(); - filters.add(filter); - chain.add(filter); - } - - assertTrue(equalOrder(chain.build().iterator(), filters.iterator())); - } - - @Test(expected = ChainCycleException.class) - public void cycles_are_detected() { - Filter a = new Filter(); - Filter b = new Filter(); - - getChain().add(b, before(a)).add(a, before(b)).build(); - } - - @Test(expected = IllegalArgumentException.class) - public void adding_same_instance_twice_is_illegal() { - Filter a = new Filter(); - - getChain().add(a).add(a).build(); - } - - @Test - public void before_instance() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b).add(a, before(b)).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void after_instance() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b, after(a)).add(a).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void before_class() { - Filter a = new FilterA(); - Filter b = new FilterB(); - - Chain<Filter> chain = getChain(). - add(b).add(a, before(b.getClass())).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void after_class() { - Filter a = new FilterA(); - Filter b = new FilterB(); - - Chain<Filter> chain = getChain(). - add(b, after(a.getClass())).add(a).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void before_subclass() { - Filter a = new FilterA(); - Filter b = new FilterExtendsB(); - - Chain<Filter> chain = getChain(). - add(b).add(a, before(FilterB.class)).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void after_subclass() { - Filter a = new FilterExtendsA(); - Filter b = new FilterB(); - - Chain<Filter> chain = getChain(). - add(b, after(FilterA.class)).add(a).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void before_provided_name() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b, provides("B")).add(a, before("B")).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void after_provided_name() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b, after("A")).add(a, provides("A")).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void before_provided_name_in_annotations() { - Filter providesA = new ProvidesA(); - Filter beforeA = new BeforeA(); - - Chain<Filter> chain = getChain(). - add(providesA).add(beforeA).build(); - - assertEquals(new Chain<>("myChain", beforeA, providesA), chain); - } - - @Test - public void after_provided_name_in_annotations() { - Filter providesA = new ProvidesA(); - Filter afterA = new AfterA(); - - Chain<Filter> chain = getChain(). - add(afterA).add(providesA).build(); - - assertEquals(new Chain<>("myChain", providesA, afterA), chain); - } - - @Test - public void before_all() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b).add(a, before("*")).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void after_all() { - Filter a = new Filter(); - Filter b = new Filter(); - - Chain<Filter> chain = getChain(). - add(b, after("*")).add(a).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void before_all_annotation() { - Filter a = new Filter(); - Filter beforeAll = new BeforeAll(); - - Chain<Filter> chain = getChain(). - add(a).add(beforeAll).build(); - - assertEquals(new Chain<>("myChain", beforeAll, a), chain); - } - - @Test - public void after_all_annotation() { - Filter a = new Filter(); - Filter afterAll = new AfterAll(); - - Chain<Filter> chain = getChain(). - add(afterAll).add(a).build(); - - assertEquals(new Chain<>("myChain", a, afterAll), chain); - } - - @Test - public void before_all_annotated_component_can_be_before_another_component_that_is_also_before_all_annotated() { - Filter beforeAll = new BeforeAll(); - Filter beforeBeforeAll = new BeforeBeforeAll(); - - Chain<Filter> chain = getChain(). - add(beforeAll).add(beforeBeforeAll).build(); - - assertEquals(new Chain<>("myChain", beforeBeforeAll, beforeAll), chain); - } - - @Test - public void after_all_annotated_component_can_be_after_another_component_that_is_also_after_all_annotated() { - Filter afterAll = new AfterAll(); - Filter afterAfterAll = new AfterAfterAll(); - - Chain<Filter> chain = getChain(). - add(afterAfterAll).add(afterAll).build(); - - assertEquals(new Chain<>("myChain", afterAll, afterAfterAll), chain); - } - - @Test - public void component_that_is_not_annotated_can_be_before_a_before_all_annotated_component() { - Filter first = new Filter(); - Filter beforeAll = new BeforeAll(); - - Chain<Filter> chain = getChain(). - add(beforeAll).add(first, before("BeforeAll")).build(); - - assertEquals(new Chain<>("myChain", first, beforeAll), chain); - } - - @Test - public void component_that_is_not_annotated_can_be_after_an_after_all_annotated_component() { - Filter last = new Filter(); - Filter afterAll = new AfterAll(); - - Chain<Filter> chain = getChain(). - add(last, after("AfterAll")).add(afterAll).build(); - - assertEquals(new Chain<>("myChain", afterAll, last), chain); - } - - @Test - public void class_name_is_always_provided() { - Filter a = new FilterA(); - Filter b = new FilterB(); - - Chain<Filter> chain = getChain(). - add(b, after(a.getClass().getName())).add(a).build(); - - assertEquals(new Chain<>("myChain", a, b), chain); - } - - @Test - public void provides_annotation_on_superclass_is_inherited_by_subclasses() { - Filter extendsA = new ExtendsProvidesA(); - Filter first = new FilterA(); - Filter last = new FilterB(); - - Chain<Filter> chain = getChain(). - add(last, after("A")).add(first, before("A")).add(extendsA).build(); - - assertEquals(new Chain<>("myChain", first, extendsA, last), chain); - } - - @Test - public void provides_annotation_on_superclass_is_inherited_by_a_subclass_that_has_its_own_provides_annotation() { - Filter extendsA = new ProvidesA_and_ProvidesExtendsA(); - Filter first = new FilterA(); - Filter last = new FilterB(); - - Chain<Filter> chain = getChain(). - add(last, after("A")).add(first, before("ExtendsA")).add(extendsA).build(); - - assertEquals(new Chain<>("myChain", first, extendsA, last), chain); - } - - @Test - public void before_annotation_on_superclass_is_inherited_by_a_subclass_that_has_its_own_before_annotation() { - Filter beforeA_and_beforeB = new BeforeA_and_BeforeB(); - Filter A = new ProvidesA(); - Filter B = new ProvidesB(); - - Chain<Filter> chain = getChain(). - add(A, before("*")).add(beforeA_and_beforeB).add(B).build(); - assertEquals(new Chain<>("myChain", beforeA_and_beforeB, A, B), chain); - } - - @Test - public void add_accepts_multiple_dependencies() { - Filter a = new Filter(); - Filter b = new Filter(); - Filter c = new Filter(); - - Chain<Filter> chain = getChain(). - add(a).add(c).add(b, after(a), before(c)).build(); - - assertEquals(new Chain<>("myChain", a, b, c), chain); - } - - private ChainBuilder<Filter> getChain() { - return new ChainBuilder<>("myChain"); - } -} diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/ChainTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/ChainTest.java deleted file mode 100644 index 84038170f7c..00000000000 --- a/yolean/src/test/java/com/yahoo/yolean/chain/ChainTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -/** - * @author Tony Vaagenes - */ -public class ChainTest { - - public static class Filter { - - } - - public static class OtherFilter extends Filter { - - } - - @Test - public void empty_chain_toString() { - Chain<Filter> c = new Chain<>("myChain"); - assertEquals("chain 'myChain'{}", c.toString()); - } - - @Test - public void singleton_chain_toString() { - Chain<Filter> c = new Chain<>("myChain", new Filter()); - assertEquals("chain 'myChain'{ Filter }", c.toString()); - } - - @Test - public void chain_toString() { - Chain<Filter> c = new Chain<>("myChain", new Filter(), new Filter(), new OtherFilter()); - assertEquals("chain 'myChain'{ Filter -> Filter -> OtherFilter }", c.toString()); - } - - @Test - public void non_equal_due_to_different_components() { - assertNotEquals(new Chain<>("a", new Filter()), new Chain<>("a", new Filter())); - } - - @Test - public void non_equal_due_to_different_size_comopnents() { - assertNotEquals(new Chain<>("a", new Filter()), new Chain<Filter>("a")); - } - - @Test - public void hashCode_equals() { - assertEquals(new Chain<>("a").hashCode(), new Chain<Filter>("a").hashCode()); - } -} diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/DirectedGraphTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/DirectedGraphTest.java deleted file mode 100644 index 6b9b29729c7..00000000000 --- a/yolean/src/test/java/com/yahoo/yolean/chain/DirectedGraphTest.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.assertTrue; - -public class DirectedGraphTest { - - private DirectedGraph graph; - private final Vertex[] v = new Vertex[10]; - - @Before - public void setup() { - for (int i = 0; i < v.length; i++) { - v[i] = new TestVertex(i); - } - - graph = new DirectedGraph(); - } - - @Test - public void before_all_are_prioritized_first() { - graph.addVertex(v[0]); - graph.addBeginningVertex(v[1]); - - assertTrue(graph.topologicalSort().containsAll(Arrays.asList(v[1], v[0]))); - } - - @Test - public void vertex_can_be_placed_before_before_all_vertices() { - graph.addVertex(v[0]); - graph.addBeginningVertex(v[1]); - graph.addEdge(v[0], v[1]); - - assertTrue(graph.topologicalSort().containsAll(Arrays.asList(v[0], v[1]))); - } - - static class TestVertex implements Vertex { - - private final int id; - - TestVertex(int id) { - this.id = id; - } - - @Override - public String toString() { - return "Vertex{" + id + '}'; - } - } -} diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/EnumeratedIdentitySetTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/EnumeratedIdentitySetTest.java deleted file mode 100644 index 66b2fc84f39..00000000000 --- a/yolean/src/test/java/com/yahoo/yolean/chain/EnumeratedIdentitySetTest.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import static java.util.Collections.singleton; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * Tests for EnumeratedIdentitySet. - */ -public class EnumeratedIdentitySetTest { - - private final List<Element> elements; - - public EnumeratedIdentitySetTest() { - elements = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - elements.add(new Element()); - } - } - - @Test - public void size() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - assertEquals(elements.size(), set.size()); - - set.add(elements.get(0)); - assertEquals(elements.size(), set.size()); - - set.remove(elements.get(0)); - assertEquals(elements.size() - 1, set.size()); - } - - @Test - public void isEmpty() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(); - assertTrue(set.isEmpty()); - - set.add(elements.get(0)); - assertFalse(set.isEmpty()); - } - - @Test - public void contains2() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - assertTrue(set.contains(elements.get(0))); - assertFalse(set.contains(new Element())); - } - - @Test - public void iterator() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - - IdentityHashMap<Element, Void> collectedElements = new IdentityHashMap<>(); - int count = 0; - for (Element element : set) { - collectedElements.put(element, null); - count++; - } - - assertEquals(count, collectedElements.size()); - assertEquals(elements.size(), collectedElements.size()); - - for (Element element : elements) { - assertTrue(collectedElements.containsKey(element)); - } - } - private static boolean containsSame(Object a, Collection<?> coll) { - for (Object b : coll) { - if (a == b) return true; - } - return false; - } - private static boolean containsSubsetSame(Collection<?> subSet, Collection<?> superSet) { - for (Object a : subSet) { - if ( ! containsSame(a, superSet)) return false; - } - return true; - } - - private static boolean containsAllSame(Collection<?> a, Collection<?> b) { - return containsSubsetSame(a, b) && containsSubsetSame(b, a); - } - - private static <T> Set<T> toIdentitySet(Collection<? extends T> collection) { - Set<T> identitySet = Collections.newSetFromMap(new IdentityHashMap<>()); - identitySet.addAll(collection); - return identitySet; - } - - @Test - public void toArray() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - - Object[] array = set.toArray(); - Element[] array2 = set.toArray(new Element[0]); - - assertTrue(set.containsAll(Arrays.asList(array))); - assertTrue(containsAllSame(Arrays.asList(array), set)); - assertTrue(containsAllSame(Arrays.asList(array2), set)); - } - - @Test - public void add() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(); - assertTrue(set.add(elements.get(0))); - assertFalse(set.add(elements.get(0))); - } - - @Test - public void remove() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - assertTrue(set.remove(elements.get(0))); - assertFalse(set.remove(elements.get(0))); - } - - @Test - public void containsAll() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - assertTrue(set.containsAll(elements.subList(0, 7))); - assertTrue(set.containsAll(elements)); - - List<Element> extendedElements = new ArrayList<>(elements); - extendedElements.add(new Element()); - assertFalse(set.containsAll(extendedElements)); - } - - @Test - public void addAll() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(); - set.addAll(elements); - - assertTrue(containsAllSame(set, elements)); - } - - @Test - public void retainAll() { - Set<Element> set = new EnumeratedIdentitySet<>(); - - set.addAll(elements.subList(0, 5)); - boolean changed = set.retainAll(toIdentitySet(elements.subList(3, 10))); - - assertTrue(changed); - assertTrue(containsAllSame(set, elements.subList(3, 5))); - - changed = set.retainAll(toIdentitySet(elements)); - assertFalse(changed); - } - - @Test - public void removeAll() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - set.removeAll(elements.subList(0, 5)); - assertTrue(containsAllSame(set, elements.subList(5, 10))); - } - - @Test - public void clear() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - set.clear(); - assertTrue(set.isEmpty()); - } - - @Test - public void removeNulls() { - Element[] singletonArray = { null, elements.get(0), null }; - assertTrue(containsAllSame(EnumeratedIdentitySet.removeNulls(singletonArray), - Arrays.asList(elements.get(0)))); - - Element[] elementsWithNull = new Element[20]; - - Iterator<Element> iterator = elements.iterator(); - - copyElementsTo(iterator, elementsWithNull, 2, 1); - copyElementsTo(iterator, elementsWithNull, 4, 8); - copyElementsTo(iterator, elementsWithNull, 19, 1); - - assertTrue(containsAllSame(EnumeratedIdentitySet.removeNulls(elementsWithNull), elements)); - } - - private void copyElementsTo(Iterator<Element> iterator, Element[] array, int startIndex, int numItems) { - for (int i = 0; i < numItems; i++) { - array[i + startIndex] = iterator.next(); - } - } - - @Test - public void renumber_preserves_ordering() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(); - - for (int i = 0; i < 200; i++) { - set.add(new Element()); - } - - set.addAll(elements); - - EnumeratedIdentitySet<Element> elementsToPreserve = new EnumeratedIdentitySet<>(elements); - - for (Iterator<Element> i = set.iterator(); i.hasNext(); ) { - if (!elementsToPreserve.contains(i.next())) { - i.remove(); - } - } - - List<Element> forceRenumber = set.insertionOrderedList(); - assertEquals(elements.size(), forceRenumber.size()); - - for (int i = 0; i < elements.size(); i++) { - assertSame(forceRenumber.get(i), elements.get(i)); - } - - set.add(new Element()); - assertTrue(containsAllSame(set.numbers(), range(0, 10))); - } - - @Test - public void renumber_when_empty() { - EnumeratedIdentitySet<Element> set = new EnumeratedIdentitySet<>(elements); - for (Iterator<Element> i = set.iterator(); i.hasNext(); ) { - i.next(); - i.remove(); - } - - set.insertionOrderedList(); - assertTrue(set.numbers().isEmpty()); - - set.add(new Element()); - assertTrue(containsAllSame(set.numbers(), singleton(0))); - } - - private List<Integer> range(int start, int endInclusive) { - List<Integer> result = new ArrayList<>(); - for (int i = start; i <= endInclusive; i++) { - result.add(i); - } - - return result; - } - - static class Element { - - @Override - public int hashCode() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean equals(Object obj) { - throw new UnsupportedOperationException(); - } - - @Override - public String toString() { - return "Element@" + System.identityHashCode(this); - } - } -} diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index e0de2622149..d227f8490dc 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -1,119 +1,4 @@ { - "com.yahoo.vespa.curator.CompletionTimeoutException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.Curator$CompletionWaiter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void awaitCompletion(java.time.Duration)", - "public abstract void notifyCompletion()" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.Curator$DirectoryCache": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void start()", - "public abstract void addListener(org.apache.curator.framework.recipes.cache.PathChildrenCacheListener)", - "public abstract java.util.List getCurrentData()", - "public abstract org.apache.curator.framework.recipes.cache.ChildData getCurrentData(com.yahoo.path.Path)", - "public abstract void close()" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.Curator$FileCache": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void start()", - "public abstract void addListener(org.apache.curator.framework.recipes.cache.NodeCacheListener)", - "public abstract org.apache.curator.framework.recipes.cache.ChildData getCurrentData()", - "public abstract void close()" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.Curator": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.vespa.curator.api.VespaCurator", - "java.lang.AutoCloseable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.vespa.curator.Curator create(java.lang.String)", - "public static com.yahoo.vespa.curator.Curator create(java.lang.String, java.util.Optional)", - "public void <init>(com.yahoo.cloud.config.CuratorConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", - "public void <init>(com.yahoo.cloud.config.ConfigserverConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", - "protected void <init>(java.lang.String, java.lang.String, java.util.function.Function)", - "public java.lang.String connectionSpec()", - "public org.apache.curator.framework.recipes.atomic.DistributedAtomicLong createAtomicCounter(java.lang.String)", - "public org.apache.curator.framework.recipes.locks.InterProcessLock createMutex(java.lang.String)", - "public com.yahoo.vespa.curator.Curator$CompletionWaiter getCompletionWaiter(com.yahoo.path.Path, int, java.lang.String)", - "public com.yahoo.vespa.curator.Curator$CompletionWaiter createCompletionWaiter(com.yahoo.path.Path, java.lang.String, int, java.lang.String)", - "public com.yahoo.vespa.curator.Curator$DirectoryCache createDirectoryCache(java.lang.String, boolean, boolean, java.util.concurrent.ExecutorService)", - "public com.yahoo.vespa.curator.Curator$FileCache createFileCache(java.lang.String, boolean)", - "public boolean exists(com.yahoo.path.Path)", - "public void set(com.yahoo.path.Path, byte[])", - "public boolean create(com.yahoo.path.Path)", - "public varargs void createAtomically(com.yahoo.path.Path[])", - "public void delete(com.yahoo.path.Path)", - "public java.util.List getChildren(com.yahoo.path.Path)", - "public java.util.Optional getData(com.yahoo.path.Path)", - "public java.util.Optional getStat(com.yahoo.path.Path)", - "public com.yahoo.vespa.curator.Lock lock(com.yahoo.path.Path, java.time.Duration)", - "public org.apache.curator.framework.CuratorFramework framework()", - "public void close()", - "public java.lang.String zooKeeperEnsembleConnectionSpec()", - "public int zooKeeperEnsembleCount()", - "public bridge synthetic java.lang.AutoCloseable lock(com.yahoo.path.Path, java.time.Duration)" - ], - "fields": [ - "protected final org.apache.curator.RetryPolicy retryPolicy" - ] - }, - "com.yahoo.vespa.curator.Lock": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.transaction.Mutex" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.vespa.curator.Curator)", - "public void <init>(java.lang.String, org.apache.curator.framework.recipes.locks.InterProcessLock)", - "public void acquire(java.time.Duration)", - "public void close()" - ], - "fields": [] - }, "com.yahoo.vespa.curator.api.VespaCurator": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index b82978836d5..f96e46d90d8 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.path.Path; import com.yahoo.vespa.curator.api.VespaCurator; @@ -62,8 +61,7 @@ public class Curator implements VespaCurator, AutoCloseable { private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1); private static final int MAX_RETRIES = 10; private static final RetryPolicy DEFAULT_RETRY_POLICY = new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES); - - protected final RetryPolicy retryPolicy = DEFAULT_RETRY_POLICY; + private static final long juteMaxBuffer = 52428800; // Should correspond with value in ZookeeperServerConfig private final CuratorFramework curatorFramework; private final ConnectionSpec connectionSpec; @@ -92,15 +90,6 @@ public class Curator implements VespaCurator, AutoCloseable { Optional.of(ZK_CLIENT_CONFIG_FILE)); } - // TODO: This can be removed when this package is no longer public API. - public Curator(ConfigserverConfig configserverConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { - this(ConnectionSpec.create(configserverConfig.zookeeperserver(), - ConfigserverConfig.Zookeeperserver::hostname, - ConfigserverConfig.Zookeeperserver::port, - configserverConfig.zookeeperLocalhostAffinity()), - Optional.of(ZK_CLIENT_CONFIG_FILE)); - } - protected Curator(String connectionSpec, String zooKeeperEnsembleConnectionSpec, Function<RetryPolicy, CuratorFramework> curatorFactory) { this(ConnectionSpec.create(connectionSpec, zooKeeperEnsembleConnectionSpec), curatorFactory.apply(DEFAULT_RETRY_POLICY)); } @@ -139,16 +128,6 @@ public class Curator implements VespaCurator, AutoCloseable { } } - /** - * Returns the ZooKeeper "connect string" used by curator: a comma-separated list of - * host:port of ZooKeeper endpoints to connect to. This may be a subset of - * zooKeeperEnsembleConnectionSpec() if there's some affinity, e.g. for - * performance reasons. - * - * This may be empty but never null - */ - public String connectionSpec() { return connectionSpec.local(); } - /** For internal use; prefer creating a {@link CuratorCounter} */ public DistributedAtomicLong createAtomicCounter(String path) { return new DistributedAtomicLong(curatorFramework, path, new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES)); @@ -203,6 +182,10 @@ public class Curator implements VespaCurator, AutoCloseable { */ // TODO: Use create().orSetData() in Curator 4 and later public void set(Path path, byte[] data) { + if (data.length > juteMaxBuffer) + throw new IllegalArgumentException("Cannot not set data at " + path.getAbsolute() + ", " + + data.length + " bytes, max bytes is " + juteMaxBuffer); + if ( ! exists(path)) create(path); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 4b0ea433cdd..56ae65bb317 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.stats.LockStats; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index 40bfb70109d..4cb510e7904 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.mock; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.collections.Pair; import com.yahoo.concurrent.Lock; import com.yahoo.concurrent.Locks; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java index ede5ef402a5..281e8b369c6 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi // TODO: Revoke this on Vespa 8. package com.yahoo.vespa.curator; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 607ab4004a7..e5a1ea7c683 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.stats; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.curator.Lock; import org.apache.curator.framework.recipes.locks.InterProcessLock; import org.junit.Before; diff --git a/zookeeper-command-line-client/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli index 75de99f3366..bab29218207 100755 --- a/zookeeper-command-line-client/src/main/sh/vespa-zkcli +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli @@ -74,21 +74,6 @@ findhost # END environment bootstrap section -usage() { - echo "Run Zookeeper command-line client" - echo "The following options are recognized:" - echo "" - - echo "-h|-help) print this help text" -} - -while [ $# -gt 0 ]; do - case $1 in - -h|-help) usage; exit 0;; - *) echo "Unrecognized option '$1'" >&2; exit 1;; - esac -done - java -cp $VESPA_HOME/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar \ -Dlog4j.configuration="log4j-vespa.properties" -Xms32m -Xmx512m \ com.yahoo.vespa.zookeeper.cli.Main "$@" diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 6582b4602af..a4e4ddaeb2d 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.6.3) add_subdirectory(zookeeper-server-3.7.0) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index e482282388f..3efdde2cd81 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,6 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.6.3</module> <module>zookeeper-server-3.7.0</module> </modules> <dependencies> diff --git a/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt deleted file mode 100644 index b7871cfbde1..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.6.3) -# Needs to be included when this is the wanted default version (and symlinks for other versions need to be removed) -#install_symlink(lib/jars/zookeeper-server-3.6.3-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml deleted file mode 100644 index f7e6f512f7c..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml +++ /dev/null @@ -1,120 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.6.3</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-client-common</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <!-- Don't use ZK version from zookeeper-client-common --> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.6.3</version> - <exclusions> - <!-- - Container provides wiring for all common log libraries - Duplicate embedding results in various warnings being printed to stderr - --> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- snappy-java and metrics-core are included here - to be able to work with ZooKeeper 3.6.3 due to - class loading issues --> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - <scope>compile</scope> - <version>3.2.5</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - <scope>compile</scope> - <version>1.1.7</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java deleted file mode 100644 index c002ffa72ce..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Starts or reconfigures zookeeper cluster. - * The QuorumPeer conditionally created here is owned by the Reconfigurer; - * when it already has a peer, that peer is used here in case start or shutdown is required. - * - * @author hmusum - */ -public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private final AtomicReference<QuorumPeer> peer = new AtomicReference<>(); - - @Inject - public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - reconfigurer.startOrReconfigure(zookeeperServerConfig, this, VespaQuorumPeer::new, peer::set); - } - - @Override - public void shutdown() { - peer.get().shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.get().start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return true; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java deleted file mode 100644 index 66742b0e05b..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.auth.AuthenticationProvider; -import org.apache.zookeeper.server.auth.X509AuthenticationProvider; - -import java.security.cert.X509Certificate; -import java.util.logging.Logger; - -/** - * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS - * - * @author bjorncs - */ -public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { - - private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); - - public VespaMtlsAuthenticationProvider() throws X509Exception { super(null, null);} - - @Override - public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { - // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. - // The client is authorized once the SSL handshake has completed. - X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); - if (certificateChain == null || certificateChain.length == 0) { - log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); - return KeeperException.Code.AUTHFAILED; - } - X509Certificate certificate = certificateChain[0]; - cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); - return KeeperException.Code.OK; - } - - @Override public String getScheme() { return "x509"; } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java deleted file mode 100644 index 47ec03367c1..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.protect.Process; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps - * exceptions so it can be used by code that does not depend on ZooKeeper. - * - * @author hmusum - */ -class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); - - @Override - public void start(Path path) { - initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); - } - - @Override - public void shutdown(Duration timeout) { - if (quorumPeer != null) { - log.log(Level.FINE, "Shutting down ZooKeeper server"); - try { - quorumPeer.shutdown(); - quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete - if (quorumPeer.isAlive()) - throw new IllegalStateException("Peer still alive after " + timeout); - } catch (RuntimeException | InterruptedException e) { - // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. - // - // When a VespaZooKeeperServer component receives a new config, the container will try to start a new - // server with the new config, this will fail until the old server is deconstructed. If the old server - // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is - // the better option. - Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); - } - } - } - - @Override - protected void initializeAndRun(String[] args) { - try { - super.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { - throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java deleted file mode 100644 index 27aa18c64c7..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.admin.ZooKeeperAdmin; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -@SuppressWarnings("unused") // Created by injection -public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); - - @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { - ZooKeeperAdmin zooKeeperAdmin = null; - try { - zooKeeperAdmin = createAdmin(connectionSpec); - long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null); - log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } catch (KeeperException e) { - if (retryOn(e)) - throw new ReconfigException(e); - else - throw new RuntimeException(e); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } finally { - if (zooKeeperAdmin != null) { - try { - zooKeeperAdmin.close(); - } catch (InterruptedException e) { /* ignore */} - } - } - } - - private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { - return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); - } - - private static boolean retryOn(KeeperException e) { - return e instanceof KeeperException.ReconfigInProgress || - e instanceof KeeperException.ConnectionLossException || - e instanceof KeeperException.NewConfigNoQuorum; - } - -} - diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 8f3a5a91a43..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; - -/** - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - private final ZooKeeperRunner runner; - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this.peer = new VespaQuorumPeer(); - this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); - } - - @Override - public void deconstruct() { - runner.shutdown(); - super.deconstruct(); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return false; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java deleted file mode 100644 index 33ec9b1303a..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.common; - -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; - -/** - * This class contains common utilities for netstuff. Like printing IPv6 literals correctly - */ -public class NetUtils { - - // Note: Changed from original to use hostname from InetSocketAddress if there exists one - public static String formatInetAddr(InetSocketAddress addr) { - String hostName = addr.getHostName(); - if (hostName != null) { - return String.format("%s:%s", hostName, addr.getPort()); - } - - InetAddress ia = addr.getAddress(); - - if (ia == null) { - return String.format("%s:%s", addr.getHostString(), addr.getPort()); - } - if (ia instanceof Inet6Address) { - return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); - } else { - return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); - } - } - - /** - * Separates host and port from given host port string if host port string is enclosed - * within square bracket. - * - * @param hostPort host port string - * @return String[]{host, port} if host port string is host:port - * or String[] {host, port:port} if host port string is host:port:port - * or String[] {host} if host port string is host - * or String[]{} if not a ipv6 host port string. - */ - public static String[] getIPV6HostAndPort(String hostPort) { - if (hostPort.startsWith("[")) { - int i = hostPort.lastIndexOf(']'); - if (i < 0) { - throw new IllegalArgumentException( - hostPort + " starts with '[' but has no matching ']'"); - } - String host = hostPort.substring(1, i); - if (host.isEmpty()) { - throw new IllegalArgumentException(host + " is empty."); - } - if (hostPort.length() > i + 1) { - return getHostPort(hostPort, i, host); - } - return new String[] { host }; - } else { - //Not an IPV6 host port string - return new String[] {}; - } - } - - private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { - // [127::1]:2181 , check separator : exits - if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { - throw new IllegalArgumentException(hostPort + " does not have : after ]"); - } - // [127::1]: scenario - if (indexOfClosingBracket + 2 == hostPort.length()) { - throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); - } - //do not include - String port = hostPort.substring(indexOfClosingBracket + 2); - return new String[] { host, port }; - } -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java deleted file mode 100644 index fdfe0fe8467..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.apache.zookeeper.server; - -import com.yahoo.vespa.zookeeper.Configurator; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.logging.Logger; - -/** - * Overrides secure setting with value from {@link Configurator}. - * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 - * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. - * - * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. - * - * @author bjorncs - */ -public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { - - private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); - - private final boolean isSecure; - - public VespaNettyServerCnxnFactory() { - super(); - this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; - boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); - log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); - } - - @Override - public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { - log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); - super.configure(addr, maxClientCnxns, backlog, isSecure); - } -} diff --git a/zookeeper-server/zookeeper-server-3.7.0/pom.xml b/zookeeper-server/zookeeper-server-3.7.0/pom.xml index ac7db35e6af..01fd83a496b 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/pom.xml +++ b/zookeeper-server/zookeeper-server-3.7.0/pom.xml @@ -11,6 +11,9 @@ <artifactId>zookeeper-server-3.7.0</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> + <properties> + <zookeeper.version>3.7.0</zookeeper.version> + </properties> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -32,7 +35,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.7.0</version> + <version>${zookeeper.version}</version> <exclusions> <!-- Container provides wiring for all common log libraries @@ -87,7 +90,6 @@ <configuration> <compilerArgs> <arg>-Xlint:all</arg> - <arg>-Werror</arg> </compilerArgs> </configuration> </plugin> @@ -97,6 +99,9 @@ <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> <forkMode>once</forkMode> + <systemPropertyVariables> + <zk-version>${zookeeper.version}</zk-version> + </systemPropertyVariables> </configuration> </plugin> <plugin> diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index c002ffa72ce..246911fdfc7 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -18,21 +18,21 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - private final AtomicReference<QuorumPeer> peer = new AtomicReference<>(); + private QuorumPeer peer; @Inject public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - reconfigurer.startOrReconfigure(zookeeperServerConfig, this, VespaQuorumPeer::new, peer::set); + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); } @Override public void shutdown() { - peer.get().shutdown(Duration.ofMinutes(1)); + peer.shutdown(Duration.ofMinutes(1)); } @Override public void start(Path configFilePath) { - peer.get().start(configFilePath); + peer.start(configFilePath); } @Override diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index 27aa18c64c7..ae7bf8d84f5 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -19,27 +23,28 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { - ZooKeeperAdmin zooKeeperAdmin = null; - try { - zooKeeperAdmin = createAdmin(connectionSpec); + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null); + // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } catch (KeeperException e) { - if (retryOn(e)) - throw new ReconfigException(e); - else - throw new RuntimeException(e); - } catch (IOException | InterruptedException e) { + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | IOException | InterruptedException e) { throw new RuntimeException(e); - } finally { - if (zooKeeperAdmin != null) { - try { - zooKeeperAdmin.close(); - } catch (InterruptedException e) { /* ignore */} - } } } @@ -48,11 +53,5 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); } - private static boolean retryOn(KeeperException e) { - return e instanceof KeeperException.ReconfigInProgress || - e instanceof KeeperException.ConnectionLossException || - e instanceof KeeperException.NewConfigNoQuorum; - } - } diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java new file mode 100644 index 00000000000..db643d76e0d --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java @@ -0,0 +1,259 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.Reconfigurer; +import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +public class VespaZooKeeperTest { + + static final Path tempDirRoot = getTmpDir(); + static final List<Integer> ports = new ArrayList<>(); + + /** + * Performs dynamic reconfiguration of ZooKeeper servers. + * + * First, a cluster of 3 servers is set up, and some data is written to it. + * Then, 3 new servers are added, and the first 3 marked for retirement; + * this should force the quorum to move the 3 new servers, but not disconnect the old ones. + * Next, the old servers are removed. + * Then, the cluster is reduced to size 1. + * Finally, the cluster grows to size 3 again. + * + * Throughout all of this, quorum should remain, and the data should remain the same. + */ + @Test(timeout = 120_000) + @Ignore // Unstable, some ZK server keeps resetting connections sometimes. + public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { + List<ZooKeeper> keepers = new ArrayList<>(); + for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); + for (int i = 0; i < 8; i++) keepers.get(i).run(); + + // Start the first three servers. + List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); + for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for all servers to be up and running. + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Write data to verify later. + String path = writeData(configs.get(0)); + + // Let three new servers join, causing the three older ones to retire and leave the ensemble. + configs = getConfigs(0, 3, 3, 3); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // The existing servers can't reconfigure and leave before the joiners are up. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for new quorum to be established. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + // Old servers are removed. + configs = getConfigs(3, 0, 3, 0); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // Old servers shut down, while the newer servers remain. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Ensure old servers shut down properly. + for (int i = 0; i < 3; i++) keepers.get(i).await(); + // Ensure new servers have reconfigured. + for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + + // Cluster shrinks to a single server. + configs = getConfigs(5, 0, 1, 0); + for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // We let the remaining server reconfigure the others out before they die. + for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 3; i < 5; i++) keepers.get(i).await(); + verifyData(path, configs.get(5)); + + // Cluster grows to 3 servers again. + configs = getConfigs(5, 0, 3, 2); + for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Wait for the joiners. + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + verifyData(path, configs.get(7)); + + // Let the remaining servers terminate. + for (int i = 5; i < 8; i++) keepers.get(i).config = null; + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 8; i++) keepers.get(i).await(); + } + + static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + try (ZooKeeperAdmin admin = createAdmin(config)) { + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); + String read = new String(admin.getData(node, false, new Stat()), UTF_8); + assertEquals("hi", read); + return node; + } + } + + static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + for (int i = 0; i < 10; i++) { + try (ZooKeeperAdmin admin = createAdmin(config)) { + assertEquals("hi", new String(admin.getData(path, false, new Stat()), UTF_8)); + return; + } + catch (KeeperException.ConnectionLossException e) { + e.printStackTrace(); + Thread.sleep(10 << i); + } + } + } + + static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { + return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), + 10_000, + System.err::println, + new ZkClientConfigBuilder().toConfig()); + } + + static class ZooKeeper { + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Phaser phaser = new Phaser(2); + final AtomicReference<Future<?>> future = new AtomicReference<>(); + ZookeeperServerConfig config; + + void run() { + future.set(executor.submit(() -> { + Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); + phaser.arriveAndAwaitAdvance(); + while (config != null) { + new ReconfigurableVespaZooKeeperServer(reconfigurer, config); + phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here + phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff + } + reconfigurer.deconstruct(); + })); + } + + void await() throws ExecutionException, InterruptedException, TimeoutException { + future.get().get(30, SECONDS); + } + } + + static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { + return IntStream.rangeClosed(1, removed + retired + active) + .mapToObj(id -> getConfig(removed, retired, active, joining, id)) + .collect(toList()); + } + + // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. + static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { + if (id <= removed) + return null; + + Path tempDir = tempDirRoot.resolve("zookeeper-" + id); + return new ZookeeperServerConfig.Builder() + .clientPort(getPorts(id).get(0)) + .dataDir(tempDir.toString()) + .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) + .myid(id) + .myidFile(tempDir.resolve("myid").toString()) + .dynamicReconfiguration(true) + .server(IntStream.rangeClosed(removed + 1, removed + retired + active) + .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() + .id(i) + .clientPort(getPorts(i).get(0)) + .electionPort(getPorts(i).get(1)) + .quorumPort(getPorts(i).get(2)) + .hostname("localhost") + .joining(i - removed > retired + active - joining) + .retired(i - removed <= retired)) + .collect(toList())) + .build(); + } + + static List<Integer> getPorts(int id) { + if (ports.size() < id * 3) { + int previousPort; + if (ports.isEmpty()) { + String[] version = System.getProperty("zk-version").split("\\."); + int versionPortOffset = 0; + for (String part : version) + versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); + previousPort = 20000 + versionPortOffset % 30000; + } + else + previousPort = ports.get(ports.size() - 1); + + for (int i = 0; i < 3; i++) + ports.add(previousPort = nextPort(previousPort)); + } + return ports.subList(id * 3 - 3, id * 3); + } + + static int nextPort(int previousPort) { + for (int j = 1; j <= 30000; j++) { + int port = (previousPort + j); + while (port > 50000) + port -= 30000; + + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } + catch (IOException e) { + System.err.println("Could not bind port " + port + ": " + e); + } + } + throw new RuntimeException("No free ports"); + } + + static Path getTmpDir() { + try { + Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); + tempDir.toFile().deleteOnExit(); + return tempDir.toAbsolutePath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 39d0312915f..8b22f658a94 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -86,7 +86,7 @@ public class Configurator { sb.append("reconfigEnabled=true").append("\n"); sb.append("skipACL=yes").append("\n"); ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> addServerToCfg(sb, server, config.clientPort())); + config.server().forEach(server -> sb.append(serverSpec(server, config.clientPort(), server.joining())).append("\n")); sb.append(new TlsQuorumConfig().createConfig(vespaTlsConfig)); sb.append(new TlsClientServerConfig().createConfig(vespaTlsConfig)); return sb.toString(); @@ -111,7 +111,8 @@ public class Configurator { } } - private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server, int clientPort) { + static String serverSpec(ZookeeperServerConfig.Server server, int clientPort, boolean joining) { + StringBuilder sb = new StringBuilder(); sb.append("server.") .append(server.id()) .append("=") @@ -120,7 +121,7 @@ public class Configurator { .append(server.quorumPort()) .append(":") .append(server.electionPort()); - if (server.joining()) { + if (joining) { // Servers that are joining an existing cluster must be marked as observers. Note that this will NOT // actually make the server an observer, but prevent it from forming an ensemble independently of the // existing cluster. @@ -130,8 +131,8 @@ public class Configurator { .append("observer"); } sb.append(";") - .append(clientPort) - .append("\n"); + .append(server.clientPort()); + return sb.toString(); } static List<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index d4223e4d815..604419c063d 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -10,14 +10,14 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; + +import static com.yahoo.vespa.zookeeper.Configurator.serverSpec; +import static java.util.stream.Collectors.toList; /** * Starts zookeeper server and supports reconfiguring zookeeper cluster. Keep this as a component @@ -50,17 +50,22 @@ public class Reconfigurer extends AbstractComponent { this.sleeper = Objects.requireNonNull(sleeper); } - void startOrReconfigure(ZookeeperServerConfig newConfig, VespaZooKeeperServer server, - Supplier<QuorumPeer> quorumPeerGetter, Consumer<QuorumPeer> quorumPeerSetter) { + @Override + public void deconstruct() { + shutdown(); + } + + QuorumPeer startOrReconfigure(ZookeeperServerConfig newConfig, VespaZooKeeperServer server, + Supplier<QuorumPeer> quorumPeerCreator) { if (zooKeeperRunner == null) { - peer = quorumPeerGetter.get(); // Obtain the peer from the server. This will be shared with later servers. + peer = quorumPeerCreator.get(); // Obtain the peer from the server. This will be shared with later servers. zooKeeperRunner = startServer(newConfig, server); } - quorumPeerSetter.accept(peer); - if (shouldReconfigure(newConfig)) { + if (newConfig.dynamicReconfiguration()) { reconfigure(newConfig); } + return peer; } ZookeeperServerConfig activeConfig() { @@ -73,42 +78,30 @@ public class Reconfigurer extends AbstractComponent { } } - private boolean shouldReconfigure(ZookeeperServerConfig newConfig) { - if (!newConfig.dynamicReconfiguration()) return false; - if (activeConfig == null) return false; - return !newConfig.equals(activeConfig()); - } - private ZooKeeperRunner startServer(ZookeeperServerConfig zookeeperServerConfig, VespaZooKeeperServer server) { ZooKeeperRunner runner = new ZooKeeperRunner(zookeeperServerConfig, server); activeConfig = zookeeperServerConfig; return runner; } + // TODO jonmv: read dynamic file, discard if old quorum impossible (config file + .dynamic.<id>) + // TODO jonmv: if dynamic file, all unlisted servers are observers; otherwise joiners are observers + // TODO jonmv: wrap Curator in Provider, for Curator shutdown private void reconfigure(ZookeeperServerConfig newConfig) { Instant reconfigTriggered = Instant.now(); - // No point in trying to reconfigure if there is only one server in the new ensemble, - // the others will be shutdown or are about to be shutdown - if (newConfig.server().size() == 1) shutdownAndDie(Duration.ZERO); - - List<String> newServers = difference(servers(newConfig), servers(activeConfig)); - String leavingServerIds = String.join(",", serverIdsDifference(activeConfig, newConfig)); - String joiningServersSpec = String.join(",", newServers); - leavingServerIds = leavingServerIds.isEmpty() ? null : leavingServerIds; - joiningServersSpec = joiningServersSpec.isEmpty() ? null : joiningServersSpec; - log.log(Level.INFO, "Will reconfigure ZooKeeper cluster. \nJoining servers: " + joiningServersSpec + - "\nleaving servers: " + leavingServerIds + + String newServers = String.join(",", servers(newConfig)); + log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." + "\nServers in active config:" + servers(activeConfig) + "\nServers in new config:" + servers(newConfig)); String connectionSpec = localConnectionSpec(activeConfig); Instant now = Instant.now(); - Duration reconfigTimeout = reconfigTimeout(newServers.size()); + Duration reconfigTimeout = reconfigTimeout(newConfig.server().size()); Instant end = now.plus(reconfigTimeout); // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed for (int attempt = 1; now.isBefore(end); attempt++) { try { Instant reconfigStarted = Instant.now(); - vespaZooKeeperAdmin.reconfigure(connectionSpec, joiningServersSpec, leavingServerIds); + vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers); Instant reconfigEnded = Instant.now(); log.log(Level.INFO, "Reconfiguration completed in " + Duration.between(reconfigTriggered, reconfigEnded) + @@ -147,24 +140,11 @@ public class Reconfigurer extends AbstractComponent { return HostName.getLocalhost() + ":" + config.clientPort(); } - private static List<String> serverIdsDifference(ZookeeperServerConfig oldConfig, ZookeeperServerConfig newConfig) { - return difference(servers(oldConfig), servers(newConfig)).stream() - .map(server -> server.substring(0, server.indexOf('='))) - .collect(Collectors.toList()); - } - private static List<String> servers(ZookeeperServerConfig config) { - // See https://zookeeper.apache.org/doc/r3.6.3/zookeeperReconfig.html#sc_reconfig_clientport for format return config.server().stream() - .map(server -> server.id() + "=" + server.hostname() + ":" + server.quorumPort() + ":" + - server.electionPort() + ";" + config.clientPort()) - .collect(Collectors.toList()); - } - - private static <T> List<T> difference(List<T> list1, List<T> list2) { - List<T> copy = new ArrayList<>(list1); - copy.removeAll(list2); - return copy; + .filter(server -> ! server.retired()) + .map(server -> serverSpec(server, config.clientPort(), false)) + .collect(toList()); } } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java index 8809dca0def..59c9628bcab 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java @@ -10,7 +10,7 @@ import java.time.Duration; */ public interface VespaZooKeeperAdmin { - void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException; + void reconfigure(String connectionSpec, String servers) throws ReconfigException; /* Timeout for connecting to ZooKeeper */ default Duration sessionTimeout() { return Duration.ofSeconds(30); } diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java index 1211624e3d6..760c326cf5d 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java @@ -17,7 +17,6 @@ import java.util.Arrays; import java.util.concurrent.Phaser; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; /** @@ -51,31 +50,26 @@ public class ReconfigurerTest { ZookeeperServerConfig nextConfig = createConfig(5, true); reconfigurer.startOrReconfigure(nextConfig); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers()); - assertNull("No servers are leaving", reconfigurer.leavingServers()); - assertEquals(1, reconfigurer.reconfigurations()); - assertSame(nextConfig, reconfigurer.activeConfig()); - - // No reconfiguration happens with same config - reconfigurer.startOrReconfigure(nextConfig); - assertEquals(1, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181,server.3=node3:2182:2183;2181,server.4=node4:2182:2183;2181", + reconfigurer.servers()); + assertEquals(2, reconfigurer.reconfigurations()); assertSame(nextConfig, reconfigurer.activeConfig()); // Cluster shrinks nextConfig = createConfig(3, true); reconfigurer.startOrReconfigure(nextConfig); - assertEquals(2, reconfigurer.reconfigurations()); + assertEquals(3, reconfigurer.reconfigurations()); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertNull("No servers are joining", reconfigurer.joiningServers()); - assertEquals("3,4", reconfigurer.leavingServers()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181", + reconfigurer.servers()); assertSame(nextConfig, reconfigurer.activeConfig()); // Cluster loses node1, but node3 joins. Indices are shuffled. nextConfig = createConfig(3, true, 1); reconfigurer.startOrReconfigure(nextConfig); - assertEquals(3, reconfigurer.reconfigurations()); - assertEquals("1=node2:2182:2183;2181,2=node3:2182:2183;2181", reconfigurer.joiningServers()); - assertEquals("1,2", reconfigurer.leavingServers()); + assertEquals(4, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node2:2182:2183;2181,server.2=node3:2182:2183;2181", + reconfigurer.servers()); assertSame(nextConfig, reconfigurer.activeConfig()); } @@ -89,9 +83,9 @@ public class ReconfigurerTest { ZookeeperServerConfig nextConfig = createConfig(5, true); reconfigurer.startOrReconfigure(nextConfig); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers()); - assertNull("No servers are leaving", reconfigurer.leavingServers()); - assertEquals(1, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181,server.3=node3:2182:2183;2181,server.4=node4:2182:2183;2181", + reconfigurer.servers()); + assertEquals(2, reconfigurer.reconfigurations()); assertSame(nextConfig, reconfigurer.activeConfig()); } @@ -112,24 +106,27 @@ public class ReconfigurerTest { reconfigurer.shutdown(); } - private ZookeeperServerConfig createConfig(int numberOfServers, boolean dynamicReconfiguration, int... skipIndices) { - Arrays.sort(skipIndices); + private ZookeeperServerConfig createConfig(int numberOfServers, boolean dynamicReconfiguration, int... retiredIndices) { + Arrays.sort(retiredIndices); ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); builder.myidFile(idFile.getAbsolutePath()); for (int i = 0, index = 0; i < numberOfServers; i++, index++) { - while (Arrays.binarySearch(skipIndices, index) >= 0) index++; - builder.server(newServer(i, "node" + index)); + boolean retired = Arrays.binarySearch(retiredIndices, index) >= 0; + if (retired) i--; + builder.server(newServer(i, "node" + index, retired)); } + builder.myid(0); builder.dynamicReconfiguration(dynamicReconfiguration); return builder.build(); } - private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName) { + private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, boolean retired) { ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); builder.id(id); builder.hostname(hostName); + builder.retired(retired); return builder; } @@ -142,6 +139,7 @@ public class ReconfigurerTest { private static class TestableReconfigurer extends Reconfigurer implements VespaZooKeeperServer { private final TestableVespaZooKeeperAdmin zooKeeperAdmin; + private final Phaser phaser = new Phaser(2); private QuorumPeer serverPeer; TestableReconfigurer(TestableVespaZooKeeperAdmin zooKeeperAdmin) { @@ -156,19 +154,16 @@ public class ReconfigurerTest { } void startOrReconfigure(ZookeeperServerConfig newConfig) { - startOrReconfigure(newConfig, this, MockQuorumPeer::new, peer -> serverPeer = peer); + serverPeer = startOrReconfigure(newConfig, this, MockQuorumPeer::new); + phaser.arriveAndDeregister(); } String connectionSpec() { return zooKeeperAdmin.connectionSpec; } - String joiningServers() { - return zooKeeperAdmin.joiningServers; - } - - String leavingServers() { - return zooKeeperAdmin.leavingServers; + String servers() { + return zooKeeperAdmin.servers; } int reconfigurations() { @@ -177,10 +172,14 @@ public class ReconfigurerTest { @Override public void shutdown() { + phaser.arriveAndAwaitAdvance(); serverPeer.shutdown(Duration.ofSeconds(1)); } @Override - public void start(Path configFilePath) { serverPeer.start(configFilePath); } + public void start(Path configFilePath) { + phaser.arriveAndAwaitAdvance(); + serverPeer.start(configFilePath); + } @Override public boolean reconfigurable() { @@ -192,8 +191,7 @@ public class ReconfigurerTest { private static class TestableVespaZooKeeperAdmin implements VespaZooKeeperAdmin { String connectionSpec; - String joiningServers; - String leavingServers; + String servers; int reconfigurations = 0; private int failures = 0; @@ -205,12 +203,11 @@ public class ReconfigurerTest { } @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { if (++attempts < failures) throw new ReconfigException("Reconfig failed"); this.connectionSpec = connectionSpec; - this.joiningServers = joiningServers; - this.leavingServers = leavingServers; + this.servers = servers; this.reconfigurations++; } |