diff options
152 files changed, 989 insertions, 365 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ddc316e5b9c..487d7fb8522 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,60 +1,42 @@ # Contributing to Vespa -We appreciate contributions to Vespa! -Below is a quick how-to. - - -## Reporting issues -Reporting a problem is a valuable contribution. -Use [GitHub issues](https://github.com/vespa-engine/vespa/issues) to report bugs. -Issues are evaluated daily. -If you read this, you are probably a developer who knows how to write good bug reports - -make it easy to for others to reproduce the problem (include a test case!), -include the Vespa version, -and make it easy for others to understand the importance of the problem. - - -## Check the ToDo list -Future features are kept on the [ToDo list](TODO.md) - -minor fixes better reported and tracked in [issues](https://github.com/vespa-engine/vespa/issues). - +Contributions to [Vespa](http://github.com/vespa-engine/vespa) +and the [Vespa documentation](http://github.com/vespa-engine/documentation) +are welcome. +This documents tells you what you need to know to contribute. + +##Â Open development +All work on Vespa happens directly on Github, +using the [Github flow model](https://guides.github.com/introduction/flow/). +We release the master branch a few times a week and you should expect it to almost always work. +In addition to the [public Travis build](https://travis-ci.org/vespa-engine/vespa) +we have a large acceptance and performance test suite which +is also run continuously. We plan to add this to the open source code base later. + +All pull requests are reviewed by a member of the +[Vespa committers](https://github.com/orgs/vespa-engine/teams/vespa/members) team, regardless +of who made it. If you want to become a committer, making some quality contributions is the way to start. ## Versioning -Vespa uses semantic versioning, -read [this guide](http://docs.vespa.ai/documentation/vespa-versions.html) to understand -deprecations and changes to APIs and stored data across releases. -Vespa releases more often than weekly, and the team does not write release notes or a changelog - -instead, track issues labeled _Feature_. +Vespa uses semantic versioning - see +[vespa versions](http://docs.vespa.ai/documentation/vespa-versions.html). +Notice in particular that any Java API in a package having a @PublicAPI +annotation in the package-info file cannot be changed in an incompatible way +between major versions: Existing types and method signatures must be preserved +(but can be marked deprecated). +## Issues +We track issues in [GitHub issues](https://github.com/vespa-engine/vespa/issues). +It is fine to submit issues also for feature requests and ideas, whether or not you intend to work on them. -## Build custom plugins -Vespa has great support for custom plugins - -you will often find that the best way to implement your application is by writing a plugin - -refer to the [APIs](http://docs.vespa.ai/documentation/api.html). - - -## Where to start contributing -Most features plug into the [Vespa Container](docs.vespa.ai/documentation/jdisc/index.html) - -this is the most likely place to write enhancements. -Discuss with the community if others have similar feature requests - make the feature generic. +There is also a [ToDo list](TODO.md) for larger things which nobody are working on yet. ### Getting started See [README](README.md) for how to build and test Vespa. -<!-- Do we have a link to code conventions - or just use below? --> -Java coding guidelines: -* 4 spaces indent <!-- Line width? --> -* Use Java coding standards -* No wildcard imports - -### Pull requests -The Vespa Team evaluates pull requests as fast as we can. -File an issue that you can refer to in the pull request - -The issue can be valid even though a pull request will not be merged. -Also add `Closes #XXX` or `Fixes #XXX` in commits - this will auto-close the issue. -The Vespa Team work on the master branch, and does not have branches for other major versions - -the current major version is the only active. -Submit unit tests with the changes and [update documentation](https://github.com/vespa-engine/documentation). -<!-- Do we need a Signed-off-by: Joe Smith <joe.smith@email.com> --> - - -## Community -List here - Slack channel? + +Vespa is large and getting an overview of the code can be a challenge. +It may help to read the READMEs of each module. + +## License and copyright +If you add new files you are welcome to use your own copyright. +In any case the code (or documentation) you submit will be licensed +under the Apache 2.0 license.
\ No newline at end of file diff --git a/application-deploy-plugin/src/main/java/com/yahoo/cloud/config/deploy/plugin/mojo/ApplicationDeployMojo.java b/application-deploy-plugin/src/main/java/com/yahoo/cloud/config/deploy/plugin/mojo/ApplicationDeployMojo.java index c001a935f5e..dc72f99eafd 100644 --- a/application-deploy-plugin/src/main/java/com/yahoo/cloud/config/deploy/plugin/mojo/ApplicationDeployMojo.java +++ b/application-deploy-plugin/src/main/java/com/yahoo/cloud/config/deploy/plugin/mojo/ApplicationDeployMojo.java @@ -17,7 +17,6 @@ import org.json.*; /** * @author hmusum - * @since 5.1.22 */ @org.apache.maven.plugins.annotations.Mojo(name = "deploy", defaultPhase = LifecyclePhase.GENERATE_SOURCES) public class ApplicationDeployMojo extends AbstractMojo { diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java index aa8f551d5b8..d2796f600c6 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java @@ -10,6 +10,7 @@ import java.util.Set; * @author bjorncs */ public class ApplicationInstance<STATUS> { + private final TenantId tenantId; private final ApplicationInstanceId applicationInstanceId; private final Set<ServiceCluster<STATUS>> serviceClusters; @@ -63,4 +64,5 @@ public class ApplicationInstance<STATUS> { ", serviceClusters=" + serviceClusters + '}'; } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java index b468b3dbd8c..df19f6cf275 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java @@ -10,6 +10,7 @@ import java.util.Objects; */ // TODO: Remove this and use ApplicationName/InstanceName instead (if you need it for the JSON stuff move it to that layer and don't let it leak) public class ApplicationInstanceId { + private final String id; public ApplicationInstanceId(String id) { @@ -41,4 +42,5 @@ public class ApplicationInstanceId { public int hashCode() { return Objects.hash(id); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java index d86bcb66014..e761e14caa4 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java @@ -11,6 +11,7 @@ import java.util.Objects; */ // TODO: Remove this and use ApplicationId instead (if you need it for the JSON stuff move it to that layer and don't let it leak) public class ApplicationInstanceReference { + private final TenantId tenantId; private final ApplicationInstanceId applicationInstanceId; @@ -54,4 +55,5 @@ public class ApplicationInstanceReference { public int hashCode() { return Objects.hash(tenantId, applicationInstanceId); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java index 91b326b1787..da5ea7f975d 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java @@ -41,4 +41,5 @@ public class ClusterId { public int hashCode() { return Objects.hash(id); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ConfigId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ConfigId.java index c4eb531de73..3a9db7b84c0 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ConfigId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ConfigId.java @@ -40,4 +40,5 @@ public class ConfigId { public int hashCode() { return Objects.hash(id); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/HostName.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/HostName.java index b5cb1518f3a..66e79917ccf 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/HostName.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/HostName.java @@ -46,4 +46,5 @@ public class HostName implements Comparable<HostName> { public int compareTo(HostName o) { return id.compareTo(o.id); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java index ee9ed11f9a4..84d1b44bbf2 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java @@ -12,6 +12,7 @@ import java.util.Set; * @author bjorncs */ public class ServiceCluster<STATUS> { + private final ClusterId clusterId; private final ServiceType serviceType; private final Set<ServiceInstance<STATUS>> serviceInstances; @@ -60,4 +61,5 @@ public class ServiceCluster<STATUS> { public int hashCode() { return Objects.hash(clusterId, serviceType, serviceInstances); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceClusterKey.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceClusterKey.java index 09208fab82c..843cbdd775b 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceClusterKey.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceClusterKey.java @@ -50,4 +50,5 @@ public class ServiceClusterKey { public int hashCode() { return Objects.hash(clusterId, serviceType); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java index 73e09476abc..cb3e6f5e077 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java @@ -9,6 +9,7 @@ import java.util.Objects; * @author bjorncs */ public class ServiceInstance<STATUS> { + private final ConfigId configId; private final HostName hostName; private final STATUS serviceStatus; @@ -57,4 +58,5 @@ public class ServiceInstance<STATUS> { public int hashCode() { return Objects.hash(configId, hostName, serviceStatus); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java index cf867a893e6..784ff9d1e38 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java @@ -41,4 +41,5 @@ public class ServiceType { public int hashCode() { return Objects.hash(id); } + } diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java index 31a34dc9d1b..ba9d6251569 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java @@ -10,6 +10,7 @@ import java.util.Objects; */ // TODO: Remove this and use TenantName instead (if you need it for the JSON stuff move it to that layer and don't let it leak) public class TenantId { + private final String id; public TenantId(String id) { @@ -41,4 +42,5 @@ public class TenantId { public int hashCode() { return Objects.hash(id); } + } diff --git a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java index 5689b6ab1bc..e879acef3bb 100644 --- a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java +++ b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java @@ -18,10 +18,10 @@ import java.util.Optional; /** * Main entry for preprocessing an application package. * - * @author lulf - * @since 5.25 + * @author Ulf Lilleengen */ public class ApplicationPreprocessor { + private final File applicationDir; private final Optional<File> outputDir; private final Optional<Environment> environment; @@ -65,4 +65,5 @@ public class ApplicationPreprocessor { System.exit(1); } } + } diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index bb29ff05fb8..88140873b7b 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -35,10 +35,10 @@ import java.util.*; /** * Contains one or more containers built from services.xml. * Other services present in the services.xml file might be mocked in future versions. - * <p> + * * Currently, only a single top level JDisc Container is allowed. Other clusters are ignored. * - * @author tonytv + * @author Tony Vaagenes */ @Beta public final class Application implements AutoCloseable { @@ -666,4 +666,5 @@ public final class Application implements AutoCloseable { } } } + } diff --git a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java index 1d288759d23..9d7cf2c8673 100644 --- a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java +++ b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java @@ -15,10 +15,11 @@ import static java.nio.file.Files.createTempDirectory; /** * Builds an application package on disk and returns a path to the result. * - * @author tonytv + * @author Tony Vaagenes */ @Beta public class ApplicationBuilder { + private Path applicationDir = createTempDirectory("application"); private Networking networking = Networking.disable; @@ -93,4 +94,5 @@ public class ApplicationBuilder { Path getPath() { return applicationDir; } + } diff --git a/application/src/main/java/com/yahoo/application/Networking.java b/application/src/main/java/com/yahoo/application/Networking.java index b23d2e58456..8dd77c6b0f6 100644 --- a/application/src/main/java/com/yahoo/application/Networking.java +++ b/application/src/main/java/com/yahoo/application/Networking.java @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application; -import com.google.common.annotations.Beta; - /** - * @author tonytv + * @author Tony Vaagenes */ public enum Networking { + enable, disable + } diff --git a/application/src/main/java/com/yahoo/application/container/ApplicationException.java b/application/src/main/java/com/yahoo/application/container/ApplicationException.java index d3e4100cfef..424c66313a7 100644 --- a/application/src/main/java/com/yahoo/application/container/ApplicationException.java +++ b/application/src/main/java/com/yahoo/application/container/ApplicationException.java @@ -5,10 +5,11 @@ package com.yahoo.application.container; * Wraps an Exception in a RuntimeException, for user convenience. * * @author gjoranv - * @since 5.1.15 */ class ApplicationException extends RuntimeException { + ApplicationException(Exception e) { super(e); } + } diff --git a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java index 2cf46c43d06..8e9180f00c7 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java @@ -20,10 +20,11 @@ import java.util.Map; /** * For doing document processing with {@link JDisc}. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ @Beta public final class DocumentProcessing { + private final DocumentProcessingHandler handler; private final Map<String, DocumentType> documentTypes; diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index 6cf98072232..ed0c29a3917 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -28,10 +28,9 @@ import java.nio.file.Path; /** * A JDisc Container configured from XML. * - * @author tonytv - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Tony Vaagenes + * @author Einar M R Rosenvinge * @author gjoranv - * @since 5.1.15 */ @Beta public final class JDisc implements AutoCloseable { diff --git a/application/src/main/java/com/yahoo/application/container/Processing.java b/application/src/main/java/com/yahoo/application/container/Processing.java index 5fc9abce8fe..f0ee3cdd1b9 100644 --- a/application/src/main/java/com/yahoo/application/container/Processing.java +++ b/application/src/main/java/com/yahoo/application/container/Processing.java @@ -17,7 +17,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @author gjoranv */ @Beta diff --git a/application/src/main/java/com/yahoo/application/container/ProcessingBase.java b/application/src/main/java/com/yahoo/application/container/ProcessingBase.java index 14ba6eb394e..f1505e4429a 100644 --- a/application/src/main/java/com/yahoo/application/container/ProcessingBase.java +++ b/application/src/main/java/com/yahoo/application/container/ProcessingBase.java @@ -19,7 +19,6 @@ import java.util.concurrent.Executors; /** * @author gjoranv - * @since 5.1.15 */ @Beta public abstract class ProcessingBase< @@ -27,6 +26,7 @@ public abstract class ProcessingBase< RESPONSE extends Response, PROCESSOR extends Processor> { + /** * Returns a registry of configured chains. * diff --git a/application/src/main/java/com/yahoo/application/container/Search.java b/application/src/main/java/com/yahoo/application/container/Search.java index ba241c70b5b..9503ce4fb50 100644 --- a/application/src/main/java/com/yahoo/application/container/Search.java +++ b/application/src/main/java/com/yahoo/application/container/Search.java @@ -18,7 +18,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @author gjoranv */ @Beta diff --git a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java index 1d64d2df2a9..c5ca67d4428 100644 --- a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java +++ b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ @ThreadSafe @Beta @@ -156,6 +156,7 @@ final class SynchronousRequestResponseHandler { } private static class BlockingCompletionHandler implements CompletionHandler { + private volatile Throwable throwable; private CountDownLatch doneLatch = new CountDownLatch(1); @@ -184,5 +185,7 @@ final class SynchronousRequestResponseHandler { } } } + } + } diff --git a/application/src/main/java/com/yahoo/application/container/handler/Headers.java b/application/src/main/java/com/yahoo/application/container/handler/Headers.java index 8c49d36c078..cb4ebc3d8b9 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Headers.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Headers.java @@ -15,8 +15,8 @@ import java.util.Set; * * @see Request * @see Response - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Einar M R Rosenvinge + * @author Simon Thoresen */ @NotThreadSafe @Beta @@ -99,10 +99,6 @@ public class Headers implements Map<String, List<String>> { } - /* - CONVENIENCE METHODS: - */ - /** * <p>Convenience method for checking whether or not a named header contains a specific value. If the named header * is not set, or if the given value is not contained within that header's value list, this method returns @@ -223,4 +219,5 @@ public class Headers implements Map<String, List<String>> { public List<Entry<String, String>> entries() { return h.entries(); } + } diff --git a/application/src/main/java/com/yahoo/application/container/handler/Response.java b/application/src/main/java/com/yahoo/application/container/handler/Response.java index 6502cfe6da2..35d42e3e147 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Response.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Response.java @@ -17,13 +17,13 @@ import java.util.regex.Pattern; /** * A response for use with {@link com.yahoo.application.container.JDisc#handleRequest(Request)}. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.15 + * @author Einar M R Rosenvinge * @see Request */ @Immutable @Beta public class Response { + private final static Pattern charsetPattern = Pattern.compile("charset=([^\\s\\;]+)", Pattern.CASE_INSENSITIVE); private final int status; private final Headers headers = new Headers(); @@ -122,4 +122,5 @@ public class Response { } return Utf8.getCharset(); } + } diff --git a/application/src/main/java/com/yahoo/application/content/ContentCluster.java b/application/src/main/java/com/yahoo/application/content/ContentCluster.java index f68e329c12f..93c83f3b323 100644 --- a/application/src/main/java/com/yahoo/application/content/ContentCluster.java +++ b/application/src/main/java/com/yahoo/application/content/ContentCluster.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application.content; +import com.google.common.annotations.Beta; + import java.nio.file.Path; import java.util.Collections; import java.util.List; @@ -8,6 +10,7 @@ import java.util.List; /** * @author bratseth */ +@Beta public class ContentCluster { /** diff --git a/application/src/test/java/com/yahoo/application/ApplicationFacade.java b/application/src/test/java/com/yahoo/application/ApplicationFacade.java index 65f36b0804f..744ebb138c9 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationFacade.java +++ b/application/src/test/java/com/yahoo/application/ApplicationFacade.java @@ -158,4 +158,5 @@ public class ApplicationFacade implements AutoCloseable { public void close() { application.close(); } + } diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java index 2ce11335894..2a363916fa3 100644 --- a/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java @@ -150,4 +150,5 @@ public class JDiscContainerDocprocTest { } } + } diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java index 134d7d64f20..4c8c7f1ce06 100644 --- a/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java @@ -17,7 +17,7 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThat; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class JDiscContainerProcessingTest { diff --git a/application/src/test/java/com/yahoo/application/container/MockClient.java b/application/src/test/java/com/yahoo/application/container/MockClient.java index 5479f4db58e..6909b1059ff 100644 --- a/application/src/test/java/com/yahoo/application/container/MockClient.java +++ b/application/src/test/java/com/yahoo/application/container/MockClient.java @@ -11,10 +11,10 @@ import com.yahoo.jdisc.service.AbstractClientProvider; import java.util.concurrent.atomic.AtomicInteger; /** - * * @author Christian Andersen */ public class MockClient extends AbstractClientProvider { + private final AtomicInteger counter = new AtomicInteger(); @Override @@ -40,4 +40,5 @@ public class MockClient extends AbstractClientProvider { // Ignored } }; + } diff --git a/application/src/test/java/com/yahoo/application/container/MockServer.java b/application/src/test/java/com/yahoo/application/container/MockServer.java index d8d7d927f7d..cdb320fd69e 100644 --- a/application/src/test/java/com/yahoo/application/container/MockServer.java +++ b/application/src/test/java/com/yahoo/application/container/MockServer.java @@ -5,10 +5,10 @@ import com.yahoo.jdisc.service.AbstractServerProvider; import com.yahoo.jdisc.service.CurrentContainer; /** - * * @author Christian Andersen */ public class MockServer extends AbstractServerProvider { + private boolean started = false; public MockServer(CurrentContainer container) { @@ -28,4 +28,5 @@ public class MockServer extends AbstractServerProvider { public boolean isStarted() { return started; } + } diff --git a/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java b/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java index 57e69e4d1b4..ca221fa5199 100644 --- a/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java +++ b/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java @@ -11,7 +11,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import java.util.concurrent.atomic.AtomicInteger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class Rot13DocumentProcessor extends DocumentProcessor { private static final String FIELD_NAME = "title"; diff --git a/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java index 2638bcfdfef..d99fc0c2b9a 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java @@ -13,8 +13,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** -* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> -*/ + * @author Einar M R Rosenvinge + */ public class DelayedThrowingInWriteRequestHandler extends AbstractRequestHandler { private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); diff --git a/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java b/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java index 01cebc3a3e9..054a63fef66 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java @@ -2,7 +2,7 @@ package com.yahoo.application.container.handlers; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class DelayedWriteException extends RuntimeException { } diff --git a/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java index 7749902f25a..c47d0afdd02 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java @@ -12,8 +12,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** -* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> -*/ + * @author Einar M R Rosenvinge + */ public class HeaderEchoRequestHandler extends AbstractRequestHandler { private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); diff --git a/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java index 3e2d5eab559..643205e9c57 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java @@ -11,10 +11,10 @@ import java.io.PrintStream; import java.util.concurrent.Executor; /** - * * @author Christian Andersen */ public class MockHttpHandler extends ThreadedHttpRequestHandler { + public MockHttpHandler(Executor executor) { super(executor); } @@ -30,4 +30,5 @@ public class MockHttpHandler extends ThreadedHttpRequestHandler { } }; } + } diff --git a/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java index 535dcca16dc..b1f0480adc7 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java @@ -11,9 +11,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** -* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +* @author Einar M R Rosenvinge */ public class ThrowingInWriteRequestHandler extends AbstractRequestHandler { + private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); @Override @@ -50,4 +51,5 @@ public class ThrowingInWriteRequestHandler extends AbstractRequestHandler { handler.completed(); } } + } diff --git a/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java b/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java index 3ec0eb63a48..22ce6652c87 100644 --- a/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java +++ b/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java @@ -2,7 +2,7 @@ package com.yahoo.application.container.handlers; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class WriteException extends RuntimeException { } diff --git a/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java b/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java index 8191fe4a2ee..99bb6c7b067 100644 --- a/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java +++ b/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java @@ -8,10 +8,10 @@ import java.io.IOException; import java.io.Writer; /** - * * @author Christian Andersen */ public class MockRenderer extends Renderer { + public MockRenderer() { } @@ -29,4 +29,5 @@ public class MockRenderer extends Renderer { protected void render(Writer writer, Result result) throws IOException { writer.write("<mock hits=\"" + result.hits().size() + "\" />"); } + } diff --git a/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java index 867f3e52015..34913a1beb5 100644 --- a/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java +++ b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java @@ -9,7 +9,6 @@ import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; /** - * * @author Christian Andersen */ public class MockSearcher extends Searcher { diff --git a/chain/src/main/java/com/yahoo/component/chain/Chain.java b/chain/src/main/java/com/yahoo/component/chain/Chain.java index 48ed39ba5da..1c628f3dfa4 100644 --- a/chain/src/main/java/com/yahoo/component/chain/Chain.java +++ b/chain/src/main/java/com/yahoo/component/chain/Chain.java @@ -13,7 +13,7 @@ import java.util.List; /** * An immutable ordered list of components * - * @author tonytv + * @author Tony Vaagenes */ public class Chain<COMPONENT extends ChainedComponent> { diff --git a/chain/src/main/java/com/yahoo/component/chain/ChainedComponent.java b/chain/src/main/java/com/yahoo/component/chain/ChainedComponent.java index c381cc6d591..7e0eb8b72ec 100644 --- a/chain/src/main/java/com/yahoo/component/chain/ChainedComponent.java +++ b/chain/src/main/java/com/yahoo/component/chain/ChainedComponent.java @@ -19,7 +19,7 @@ import java.util.List; /** * Component with dependencies. * - * @author tonytv + * @author Tony Vaagenes */ public abstract class ChainedComponent extends AbstractComponent { diff --git a/chain/src/main/java/com/yahoo/component/chain/ChainsConfigurer.java b/chain/src/main/java/com/yahoo/component/chain/ChainsConfigurer.java index c73dc7c16f2..969c1c7d66a 100644 --- a/chain/src/main/java/com/yahoo/component/chain/ChainsConfigurer.java +++ b/chain/src/main/java/com/yahoo/component/chain/ChainsConfigurer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.component.chain; -import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.chain.model.ChainedComponentModel; diff --git a/chain/src/main/java/com/yahoo/component/chain/Phase.java b/chain/src/main/java/com/yahoo/component/chain/Phase.java index a291f471737..4148bdc6258 100644 --- a/chain/src/main/java/com/yahoo/component/chain/Phase.java +++ b/chain/src/main/java/com/yahoo/component/chain/Phase.java @@ -10,10 +10,11 @@ import java.util.TreeSet; /** * Used for many to many constraints on searcher ordering. * - * @author tonytv + * @author Tony Vaagenes */ @Immutable public class Phase { + public final Dependencies dependencies; public Phase(String name, Set<String> before, Set<String> after) { @@ -49,4 +50,5 @@ public class Phase { Dependencies union = dependencies.union(phase.dependencies); return new Phase(getName(), union.before(), union.after()); } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/After.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/After.java index f510e8f4c8b..be10c82d2e1 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/After.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/After.java @@ -10,11 +10,13 @@ import java.lang.annotation.*; * See {@link com.yahoo.component.chain.dependencies.ordering.ChainBuilder} * for dependency handling information. * - * @author tonytv + * @author Tony Vaagenes */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) @Inherited public @interface After { - public abstract String[] value() default {}; + + String[] value() default {}; + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/Before.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/Before.java index 01ad8f6e563..4542140c574 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/Before.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/Before.java @@ -10,11 +10,13 @@ import java.lang.annotation.*; * See {@link com.yahoo.component.chain.dependencies.ordering.ChainBuilder} * for dependency handling information. * - * @author tonytv + * @author Tony Vaagenes */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) @Inherited public @interface Before { - public abstract String[] value() default {}; + + String[] value() default {}; + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/Dependencies.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/Dependencies.java index a817bbb5116..46ea3518656 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/Dependencies.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/Dependencies.java @@ -9,7 +9,7 @@ import net.jcip.annotations.Immutable; /** * Constraints for ordering ChainedComponents in chains. * - * @author tonytv + * @author Tony Vaagenes */ @Immutable public class Dependencies { diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/Provides.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/Provides.java index 23becf6b4a7..b014681c469 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/Provides.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/Provides.java @@ -8,11 +8,13 @@ import java.lang.annotation.*; * Other components can then mark themselves as "before" and "after" the string provided here, * to impose constraints on ordering. * - * @author tonytv + * @author Tony Vaagenes */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) @Inherited public @interface Provides { - public abstract String[] value() default {}; + + String[] value() default {}; + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java index 637cacf2bb6..beafe7d2b98 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java @@ -31,9 +31,10 @@ import com.yahoo.component.chain.Phase; * A warning will be logged if multiple components of different types provides the * same name. A component can not provide the same name as a phase. * - * @author tonytv + * @author Tony Vaagenes */ public class ChainBuilder<T extends ChainedComponent> { + private final ComponentId id; private int numComponents = 0; private int priority = 1; @@ -166,4 +167,5 @@ public class ChainBuilder<T extends ChainedComponent> { } return readyNodes; } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java index 4c1eeff3c61..b9e8f56d15c 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java @@ -11,7 +11,7 @@ import com.yahoo.component.chain.ChainedComponent; /** * A set of components providing a given name. * - * @author tonytv + * @author Tony Vaagenes */ class ComponentNameProvider extends NameProvider { diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNode.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNode.java index e6a8b982ad8..2bc81542d52 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNode.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNode.java @@ -7,9 +7,10 @@ import com.yahoo.component.chain.ChainedComponent; * A node representing a given component. * * @see Node - * @author tonytv + * @author Tony Vaagenes */ class ComponentNode<T extends ChainedComponent> extends Node { + private T component; public ComponentNode(T component, int priority) { @@ -31,5 +32,6 @@ class ComponentNode<T extends ChainedComponent> extends Node { int classPriority() { return 2; } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ConflictingNodeTypeException.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ConflictingNodeTypeException.java index a233f0cd79f..edf4a119e5c 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ConflictingNodeTypeException.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ConflictingNodeTypeException.java @@ -4,7 +4,7 @@ package com.yahoo.component.chain.dependencies.ordering; /** * Thrown if a searcher provides the same name as a phase. * - * @author tonytv + * @author Tony Vaagenes */ @SuppressWarnings("serial") public class ConflictingNodeTypeException extends RuntimeException { diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/CycleDependenciesException.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/CycleDependenciesException.java index fd05ab20b02..40a78030c41 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/CycleDependenciesException.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/CycleDependenciesException.java @@ -10,10 +10,11 @@ import java.util.Set; * representation of the cycle is available to help solve the problem (<a * href="http://graphviz.org/">GraphViz</a>). * - * @author tonytv + * @author Tony Vaagenes */ @SuppressWarnings("serial") public class CycleDependenciesException extends RuntimeException { + public Map<String, NameProvider> cycleNodes; CycleDependenciesException(Map<String, NameProvider> cycleNodes) { @@ -41,5 +42,4 @@ public class CycleDependenciesException extends RuntimeException { return createDotString(cycleNodes); } - } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/NameProvider.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/NameProvider.java index 526e65146d9..773b6d07035 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/NameProvider.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/NameProvider.java @@ -4,9 +4,10 @@ package com.yahoo.component.chain.dependencies.ordering; /** * A node containing nodes providing a given name. * - * @author tonytv + * @author Tony Vaagenes */ abstract class NameProvider extends Node { + final String name; public NameProvider(String name, int priority) { @@ -24,6 +25,7 @@ abstract class NameProvider extends Node { protected String dotName() { return name; } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/Node.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/Node.java index cb5b6a1cbff..da652fde614 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/Node.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/Node.java @@ -17,7 +17,7 @@ import java.util.Set; * * Where name designates a NameProvider( either a phase or a set of searchers). * - * @author tonytv + * @author Tony Vaagenes */ abstract class Node { //How this node should be prioritized if its compared with a node of the same class, see class priority. diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodes.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodes.java index a9650fe751b..7af863a37c3 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodes.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodes.java @@ -8,9 +8,10 @@ import java.util.PriorityQueue; /** * Ensures that Searchers are ordered deterministically. * - * @author tonytv + * @author Tony Vaagenes */ class OrderedReadyNodes { + private class PriorityComparator implements Comparator<Node> { @Override public int compare(Node lhs, Node rhs) { @@ -22,8 +23,7 @@ class OrderedReadyNodes { } } - final private PriorityQueue<Node> nodes = - new PriorityQueue<>(10, new PriorityComparator()); + final private PriorityQueue<Node> nodes = new PriorityQueue<>(10, new PriorityComparator()); public void add(Node node) { nodes.add(node); @@ -36,4 +36,5 @@ class OrderedReadyNodes { public boolean isEmpty() { return nodes.isEmpty(); } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/PhaseNameProvider.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/PhaseNameProvider.java index a44cbad4b2f..f99a9191150 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/PhaseNameProvider.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/PhaseNameProvider.java @@ -4,9 +4,10 @@ package com.yahoo.component.chain.dependencies.ordering; /** * A phase providing a given name. * - * @author tonytv + * @author Tony Vaagenes */ class PhaseNameProvider extends NameProvider { + public PhaseNameProvider(String name, int priority) { super(name,priority); } @@ -25,4 +26,5 @@ class PhaseNameProvider extends NameProvider { int classPriority() { return 0; } + } diff --git a/chain/src/main/java/com/yahoo/component/chain/model/ChainSpecification.java b/chain/src/main/java/com/yahoo/component/chain/model/ChainSpecification.java index 2f6a03cbb87..eac76afa275 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/ChainSpecification.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/ChainSpecification.java @@ -12,10 +12,11 @@ import java.util.*; /** * Specifies how the components should be selected to create a chain. * - * @author tonytv + * @author Tony Vaagenes */ @Immutable public class ChainSpecification { + public static class Inheritance { public final Set<ComponentSpecification> chainSpecifications; public final Set<ComponentSpecification> excludedComponents; diff --git a/chain/src/main/java/com/yahoo/component/chain/model/ChainedComponentModel.java b/chain/src/main/java/com/yahoo/component/chain/model/ChainedComponentModel.java index bc728f0fdd1..cce51824942 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/ChainedComponentModel.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/ChainedComponentModel.java @@ -9,8 +9,8 @@ import net.jcip.annotations.Immutable; /** * Describes how a chained component should be created. * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author tonytv + * @author Arne Bergene Fossaa + * @author Tony Vaagenes */ @Immutable public class ChainedComponentModel extends ComponentModel { diff --git a/chain/src/main/java/com/yahoo/component/chain/model/ChainsModel.java b/chain/src/main/java/com/yahoo/component/chain/model/ChainsModel.java index be0b124c383..08cb6b7ccfd 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/ChainsModel.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/ChainsModel.java @@ -12,7 +12,7 @@ import com.yahoo.component.provider.ComponentRegistry; /** * A model of how the chains and components should be created. * - * @author tonytv + * @author Tony Vaagenes */ public class ChainsModel { diff --git a/chain/src/main/java/com/yahoo/component/chain/model/ChainsModelBuilder.java b/chain/src/main/java/com/yahoo/component/chain/model/ChainsModelBuilder.java index f7c771c1e60..5f9f2daa177 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/ChainsModelBuilder.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/ChainsModelBuilder.java @@ -14,7 +14,7 @@ import com.yahoo.container.core.ChainsConfig; /** * Builds a chains model from config. * - * @author tonytv + * @author Tony Vaagenes */ public class ChainsModelBuilder { diff --git a/chain/src/main/java/com/yahoo/component/chain/model/ComponentAdaptor.java b/chain/src/main/java/com/yahoo/component/chain/model/ComponentAdaptor.java index 142af91f0fd..2961768a7de 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/ComponentAdaptor.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/ComponentAdaptor.java @@ -7,7 +7,7 @@ import com.yahoo.component.ComponentId; /** * For using non-component model classes with ComponentRegistry. * - * @author tonytv + * @author Tony Vaagenes */ public final class ComponentAdaptor<T> extends AbstractComponent { diff --git a/chain/src/main/java/com/yahoo/component/chain/model/Resolver.java b/chain/src/main/java/com/yahoo/component/chain/model/Resolver.java index 5f6b3ce7905..15acff0db0b 100644 --- a/chain/src/main/java/com/yahoo/component/chain/model/Resolver.java +++ b/chain/src/main/java/com/yahoo/component/chain/model/Resolver.java @@ -6,8 +6,10 @@ import com.yahoo.component.ComponentSpecification; /** * Maps component specifications to matching instances. * - * @author tonytv + * @author Tony Vaagenes */ public interface Resolver<T> { + T resolve(ComponentSpecification componentSpecification); + } diff --git a/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java b/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java index 9d302174a47..7e74a732893 100644 --- a/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java +++ b/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java @@ -22,8 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** - * @author tonytv - * @since 5.1.10 + * @author Tony Vaagenes */ @SuppressWarnings({"rawtypes", "unchecked"}) public class ChainBuilderTest { diff --git a/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java b/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java index 8dde4ac3ea5..77729a99012 100644 --- a/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java +++ b/chain/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java @@ -16,10 +16,12 @@ import com.yahoo.component.ComponentId; /** * Test for OrderedReadyNodes. - * @author tonytv + * + * @author Tony Vaagenes */ @SuppressWarnings("rawtypes") public class OrderedReadyNodesTest { + class ComponentA extends ChainedComponent { public ComponentA(ComponentId id) { super(id); @@ -101,4 +103,5 @@ public class OrderedReadyNodesTest { private Node pop() { return readyNodes.pop(); } + } diff --git a/chain/src/test/java/com/yahoo/component/chain/model/ChainsModelBuilderTest.java b/chain/src/test/java/com/yahoo/component/chain/model/ChainsModelBuilderTest.java index a946d568704..2a22b5afbf7 100644 --- a/chain/src/test/java/com/yahoo/component/chain/model/ChainsModelBuilderTest.java +++ b/chain/src/test/java/com/yahoo/component/chain/model/ChainsModelBuilderTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertTrue; /** * @author gjoranv - * @since 5.1.10 */ public class ChainsModelBuilderTest { @@ -69,4 +68,5 @@ public class ChainsModelBuilderTest { getComponentsByName(Set<ComponentSpecification> componentSpecifications) { return ChainSpecification.componentsByName(componentSpecifications); } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 06f3ecbf5e3..4bc09898361 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -104,4 +104,5 @@ public class ClusterController extends AbstractComponent void shutdownController(FleetController controller) throws Exception { controller.shutdown(); } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index 98c09bdd61a..0936f9d781a 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -98,4 +98,5 @@ public class ClusterControllerClusterConfigurer { } } } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java index 320020ef6d4..f961297643e 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; /** * A dummy zookeeper provider when we do not run our own zookeeper instance. * - * @author lulf - * @since 5.26 + * @author Ulf Lilleengen */ public class DummyZooKeeperProvider implements ZooKeeperProvider { } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java index 1cc8155657f..cbb1eb41eac 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java @@ -6,10 +6,11 @@ import com.yahoo.vespa.zookeeper.ZooKeeperServer; /** * ZooKeeper provider that ensures we are running our own instance of zookeeper. * - * @author lulf - * @since 5.26 + * @author Ulf Lilleengen */ public class StandaloneZooKeeperProvider implements ZooKeeperProvider { + public StandaloneZooKeeperProvider(ZooKeeperServer server) { } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index cb0132c18d6..431fc797df6 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -62,4 +62,5 @@ public class StateRestApiV2Handler extends JDiscHttpRequestHandler { } return set; } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java index 16ac6bfcbe7..ae7c32e0f95 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpReq import java.util.concurrent.Executor; public class StatusHandler extends JDiscHttpRequestHandler { + private final com.yahoo.vespa.clustercontroller.core.status.StatusHandler statusHandler; @Inject @@ -19,4 +20,5 @@ public class StatusHandler extends JDiscHttpRequestHandler { super(handler, executor, accessLog); this.statusHandler = handler; } + } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java index 68baa5f0480..bb18bcc65d6 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; /** * Abstraction we can depend on providing us with a zookeeper server being up. * - * @author lulf - * @since 5.25 + * @author Ulf Lilleengen */ public interface ZooKeeperProvider { } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java index 3316f05ce60..f437e6aa67d 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java @@ -79,4 +79,5 @@ public class ClusterControllerClusterConfigurerTest extends TestCase { assertEquals("Must set zookeeper server with multiple fleetcontrollers", e.getMessage()); } } + } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java index f026aa3f0ef..fcef483e7d7 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java @@ -13,6 +13,7 @@ import junit.framework.TestCase; import java.util.Map; public class ClusterControllerTest extends TestCase { + private FleetControllerOptions options = new FleetControllerOptions("storage"); private Metric metric = new Metric() { @Override diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java index 25d6e1846da..524aba1398b 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java @@ -48,4 +48,5 @@ public class StateRestApiV2HandlerTest extends TestCase { assertEquals(expected, mapping); } + } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java index a904f46c2ae..66cb477e793 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java @@ -16,4 +16,5 @@ public class StatusHandlerTest extends TestCase { StatusHandler handler = new StatusHandler(controller, executor, AccessLog.voidAccessLog()); executor.shutdown(); } + } diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheAsyncHttpClient.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheAsyncHttpClient.java index 89127b99d7c..11d746ef3ce 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheAsyncHttpClient.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheAsyncHttpClient.java @@ -21,10 +21,13 @@ import java.util.logging.Logger; * This class gets around these issues by creating one instance per unique setting, and ensuring only one request use a given instance at a time. */ public class ApacheAsyncHttpClient implements AsyncHttpClient<HttpResult> { + private static final Logger log = Logger.getLogger(ApacheAsyncHttpClient.class.getName()); + public interface SyncHttpClientFactory { SyncHttpClient createInstance(String proxyHost, int proxyPort, long timeoutMs); } + public static class Settings { String proxyHost; int proxyPort; @@ -176,4 +179,5 @@ public class ApacheAsyncHttpClient implements AsyncHttpClient<HttpResult> { apacheInstances.clear(); } } + } diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheHttpInstance.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheHttpInstance.java index 02195701243..3eafd2ae1d5 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheHttpInstance.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/ApacheHttpInstance.java @@ -30,6 +30,7 @@ import org.apache.http.params.HttpParams; * Synchronous http client using Apache commons. */ public class ApacheHttpInstance implements SyncHttpClient { + private static final Logger log = Logger.getLogger(ApacheHttpInstance.class.getName()); DefaultHttpClient client; @@ -128,4 +129,5 @@ public class ApacheHttpInstance implements SyncHttpClient { public void close() { client.getConnectionManager().shutdown(); } + } diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java index aa44845561e..1069bd79b4f 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java @@ -19,7 +19,7 @@ import java.util.logging.Logger; /** * Note. This class is tested through apache http instance test, using this as other endpoint. - * @author humbe + * @author Haakon Humberset * @author Harald Musum * @author Vegard Sjonfjell */ @@ -124,4 +124,5 @@ public class JDiscHttpRequestHandler extends LoggingRequestHandler { } return headers; } + } diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapper.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapper.java index 0f203dac3bf..559eaee4821 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapper.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapper.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import java.util.logging.Logger; public class JDiscMetricWrapper implements MetricReporter { + private final Object lock = new Object(); private Metric m; @@ -47,4 +48,5 @@ public class JDiscMetricWrapper implements MetricReporter { return new ContextWrapper(m.createContext(stringMap)); } } + } diff --git a/clustercontroller-apputil/src/test/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapperTest.java b/clustercontroller-apputil/src/test/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapperTest.java index 278f6a9e1fa..13abfa0ecd5 100644 --- a/clustercontroller-apputil/src/test/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapperTest.java +++ b/clustercontroller-apputil/src/test/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscMetricWrapperTest.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class JDiscMetricWrapperTest { + class MetricImpl implements Metric { int calls = 0; @Override @@ -41,4 +42,5 @@ public class JDiscMetricWrapperTest { assertEquals(5, impl2.calls); } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 68c1a897dc3..72f19ed64b5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -15,10 +15,12 @@ import java.io.Reader; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -57,6 +59,7 @@ public class DeploymentSpec { this.upgradePolicy = upgradePolicy; this.steps = ImmutableList.copyOf(completeSteps(new ArrayList<>(steps))); this.xmlForm = xmlForm; + validateZones(this.steps); } /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ @@ -68,7 +71,25 @@ public class DeploymentSpec { throw new IllegalArgumentException("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) + " but max 24 hours is allowed"); } + + /** Throw an IllegalArgumentException if any production zone is declared multiple times */ + private void validateZones(List<Step> steps) { + Set<DeclaredZone> zones = new HashSet<>(); + + steps.stream().filter(step -> step instanceof DeclaredZone) + .map(DeclaredZone.class::cast) + .forEach(zone -> ensureUnique(zone, zones)); + steps.stream().filter(step -> step instanceof ParallelZones) + .map(ParallelZones.class::cast) + .flatMap(parallelZones -> parallelZones.zones().stream()) + .forEach(zone -> ensureUnique(zone, zones)); + } + private void ensureUnique(DeclaredZone zone, Set<DeclaredZone> zones) { + if ( ! zones.add(zone)) + throw new IllegalArgumentException(zone + " is listed twice in deployment.xml"); + } + /** Adds missing required steps and reorders steps to a permissible order */ private static List<Step> completeSteps(List<Step> steps) { // Ensure no duplicate deployments to the same zone @@ -123,7 +144,7 @@ public class DeploymentSpec { public List<Step> steps() { return steps; } /** Returns only the DeclaredZone deployment steps of this in the order they will be performed */ - public List<DeclaredZone> zones() { + public List<DeclaredZone> zones() { return steps.stream().filter(step -> step instanceof DeclaredZone).map(DeclaredZone.class::cast) .collect(Collectors.toList()); } @@ -168,17 +189,21 @@ public class DeploymentSpec { if (environment == Environment.prod) { for (Element stepTag : XML.getChildren(environmentTag)) { - if (stepTag.getTagName().equals("delay")) - steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + - longAttribute("minutes", stepTag) * 60 + - longAttribute("seconds", stepTag)))); - else // a region: deploy step - steps.add(new DeclaredZone(environment, - Optional.of(RegionName.from(XML.getValue(stepTag).trim())), - readActive(stepTag))); + if (stepTag.getTagName().equals("delay")) { + steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + + longAttribute("minutes", stepTag) * 60 + + longAttribute("seconds", stepTag)))); + } else if (stepTag.getTagName().equals("parallel")) { + List<DeclaredZone> zones = new ArrayList<>(); + for (Element regionTag : XML.getChildren(stepTag)) { + zones.add(readDeclaredZone(environment, regionTag)); + } + steps.add(new ParallelZones(zones)); + } else { // a region: deploy step + steps.add(readDeclaredZone(environment, stepTag)); + } } - } - else { + } else { steps.add(new DeclaredZone(environment)); } @@ -207,6 +232,11 @@ public class DeploymentSpec { return tagName.equals("test") || tagName.equals("staging") || tagName.equals("prod"); } + private static DeclaredZone readDeclaredZone(Environment environment, Element regionTag) { + return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), + readActive(regionTag)); + } + private static Optional<String> readGlobalServiceId(Element environmentTag) { String globalServiceId = environmentTag.getAttribute("global-service-id"); if (globalServiceId == null || globalServiceId.isEmpty()) { @@ -360,7 +390,43 @@ public class DeploymentSpec { if ( ! this.region.equals(other.region())) return false; return true; } + + @Override + public String toString() { + return environment + ( region.isPresent() ? "." + region.get() : ""); + } + + } + + /** A deployment step which is to run deployment to multiple zones in parallel */ + public static class ParallelZones extends Step { + + private final List<DeclaredZone> zones; + + public ParallelZones(List<DeclaredZone> zones) { + this.zones = ImmutableList.copyOf(zones); + } + + /** The list of zones to deploy in */ + public List<DeclaredZone> zones() { return this.zones; } + + @Override + public boolean deploysTo(Environment environment, Optional<RegionName> region) { + return zones.stream().anyMatch(zone -> zone.deploysTo(environment, region)); + } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ParallelZones)) return false; + ParallelZones that = (ParallelZones) o; + return Objects.equals(zones, that.zones); + } + + @Override + public int hashCode() { + return Objects.hash(zones); + } } /** Controls when this application will be upgraded to new Vespa versions */ diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 0458e9d4c15..95f9963d6f4 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -4,14 +4,15 @@ package com.yahoo.config.application.api; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; import java.io.StringReader; import java.util.Optional; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * @author bratseth */ @@ -233,4 +234,46 @@ public class DeploymentSpecTest { assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); } + @Test + public void productionSpecWithParallelDeployments() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod> \n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.ParallelZones parallelZones = ((DeploymentSpec.ParallelZones) spec.steps().get(3)); + assertEquals(2, parallelZones.zones().size()); + assertEquals(RegionName.from("us-central-1"), parallelZones.zones().get(0).region().get()); + assertEquals(RegionName.from("us-east-3"), parallelZones.zones().get(1).region().get()); + } + + @Test + public void productionSpecWithDuplicateRegions() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-west-1</region>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + try { + DeploymentSpec.fromXml(r); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("prod.us-west-1 is listed twice in deployment.xml", e.getMessage()); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 6d4945f23ad..38e417aca9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -45,7 +45,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** The optional PRELOAD libraries for this Service. */ // Please keep non-null, as passed to command line in service startup - private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); + private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"); // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. private long mmapNoCoreLimit = -1l; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 414cc56b1e3..c9cc51af867 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -19,8 +19,7 @@ import java.util.List; /** * Builds the config model for the standalone config server. * - * @author lulf - * @since 5.16 + * @author Ulf Lilleengen */ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 3ce5e002e53..36897643964 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -25,7 +25,8 @@ Staging = element staging { Prod = element prod { attribute global-service-id { text }? & Region* & - Delay* + Delay* & + Parallel* } Region = element region { @@ -38,3 +39,7 @@ Delay = element delay { attribute minutes { xsd:long }? & attribute seconds { xsd:long }? } + +Parallel = element parallel { + Region* +} diff --git a/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/deployment.xml b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/deployment.xml new file mode 100644 index 00000000000..0d3b74b8119 --- /dev/null +++ b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/deployment.xml @@ -0,0 +1,11 @@ +<deployment version="1.0"> + <test/> + <staging/> + <prod global-service-id="query"> + <parallel> + <region active="true">us-east-3</region> + <delay hours="1"/> + <region active="true">us-west-1</region> + </parallel> + </prod> +</deployment> diff --git a/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/hosts.xml b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/hosts.xml new file mode 100644 index 00000000000..115efd488d0 --- /dev/null +++ b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/hosts.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> + <host name="schmocalhost"> + <alias>node2</alias> + </host> +</hosts> diff --git a/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/services.xml b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/services.xml new file mode 100644 index 00000000000..03d8fc012ac --- /dev/null +++ b/config-model/src/test/cfg/application/invalid_parallel_deployment_xml/services.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + + <container version="1.0"> + <nodes> + <node hostalias="node1" /> + </nodes> + <search/> + </container> + +</services> diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 1e1b8cd2ac8..3d5c9b1c187 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -196,7 +196,7 @@ public class ApplicationDeployTest { @Test public void testThatModelIsRebuiltWhenSearchDefinitionIsAdded() throws IOException { - File tmpDir = Files.createTempDir(); + File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir); FilesApplicationPackage app = createAppPkg(tmpDir.getAbsolutePath()); assertThat(getSearchDefinitions(app).size(), is(5)); @@ -208,25 +208,37 @@ public class ApplicationDeployTest { @Test public void testThatAppWithDeploymentXmlIsValid() throws IOException { - File tmpDir = Files.createTempDir(); + File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir); createAppPkg(tmpDir.getAbsolutePath()); } @Test(expected = IllegalArgumentException.class) public void testThatAppWithIllegalDeploymentXmlIsNotValid() throws IOException { - File tmpDir = Files.createTempDir(); + File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "app_invalid_deployment_xml"), tmpDir); createAppPkg(tmpDir.getAbsolutePath()); } @Test public void testThatAppWithIllegalEmptyProdRegion() throws IOException { - File tmpDir = Files.createTempDir(); + File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "empty_prod_region_in_deployment_xml"), tmpDir); createAppPkg(tmpDir.getAbsolutePath()); } + @Test + public void testThatAppWithInvalidParallelDeploymentFails() throws IOException { + File tmpDir = tmpFolder.getRoot(); + IOUtils.copyDirectory(new File(TESTDIR, "invalid_parallel_deployment_xml"), tmpDir); + try { + createAppPkg(tmpDir.getAbsolutePath()); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("element \"delay\" not allowed here")); + } + } + private List<SearchDefinition> getSearchDefinitions(FilesApplicationPackage app) { return new DeployState.Builder().applicationPackage(app).build().getSearchDefinitions(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 63d5d37598b..98f599769c0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -110,9 +110,9 @@ public class ModelProvisioningTest { assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); assertThat(model.getContainerClusters().get("mydisc").getMemoryPercentage(), is(Optional.empty())); assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc")); diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index 89b52bc44ca..99b1dc1be69 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -9,5 +9,13 @@ <region active='true'>us-central-1</region> <delay hours='3' minutes='7' seconds='13'/> <region active='true'>us-east-3</region> + <parallel> + <region active='true'>us-north-1</region> + <region active='true'>us-south-1</region> + </parallel> + <parallel> + <region active='true'>us-north-2</region> + <region active='true'>us-south-2</region> + </parallel> </prod> </deployment> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 377b3997e43..70be29fa80d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -51,14 +51,12 @@ import java.util.logging.Logger; * @author Vegard Havdal * @author Ulf Lilleengen */ -//TODO Rename to TenantRepository +// TODO: Rename to TenantRepository public class Tenants implements ConnectionStateListener, PathChildrenCacheListener { public static final TenantName HOSTED_VESPA_TENANT = TenantName.from("hosted-vespa"); private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); - private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList( - DEFAULT_TENANT, - HOSTED_VESPA_TENANT); + private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList(DEFAULT_TENANT, HOSTED_VESPA_TENANT); private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); private static final Path vespaPath = Path.fromString("/vespa"); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 06b50145fc1..2eec7109722 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -15,7 +15,7 @@ import com.yahoo.data.access.simple.Value.StringValue; * A regular hit from a Vespa backend * * @author bratseth - * @author steinar + * @author Steinar Knutsen */ public class FastHit extends Hit { @@ -271,12 +271,14 @@ public class FastHit extends Hit { public void addSummary(Docsum docsum) { LazyDocsumValue lazyDocsumValue = new LazyDocsumValue(docsum); + reserve(docsum.getDefinition().getFieldCount()); for (DocsumField field : docsum.getDefinition().getFields()) { setDocsumFieldIfNotPresent(field.getName(), lazyDocsumValue); } } void addSummary(DocsumDefinition docsumDef, Inspector value) { + reserve(docsumDef.getFieldCount()); for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); if (value.type() == Type.STRING && diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index b03c0cb752f..6b4d3594087 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -227,6 +227,7 @@ public class Dispatcher extends AbstractComponent { } private void fill(FastHit hit, Inspector summary) { + hit.reserve(summary.fieldCount()); summary.traverse((String name, Inspector value) -> { hit.setField(name, nativeTypeOf(value)); }); diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 815006edbf5..0bfbecfd9ab 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -101,8 +101,12 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi public static final String SDDOCNAME_FIELD = "sddocname"; private Map<String,Object> getFieldMap() { + return getFieldMap(16); + } + private Map<String,Object> getFieldMap(int minSize) { if (fields == null) { - fields = new LinkedHashMap<>(16); + // Compensate for loadfactor and then some, rounded up.... + fields = new LinkedHashMap<>(2*minSize); } return fields; } @@ -448,6 +452,14 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi public final Map<String,Object> fields() { return getUnmodifiableFieldMap(); } /** + * Will preallocate in order to avoid resizing. + * @param minSize + */ + public void reserve(int minSize) { + getFieldMap(minSize); + } + + /** * Fields * @return An iterator for traversing the fields * @since 5.1.3 diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 27992220f1c..043a4fd4f3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -238,24 +238,25 @@ public class ApplicationController { // Ensure that the deploying change is tested // FIXME: For now only for non-self-triggering applications - VESPA-8418 - if ( ! application.deploymentJobs().isSelfTriggering() && ! zone.environment().isManuallyDeployed() && ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) + if ( ! application.deploymentJobs().isSelfTriggering() && + ! zone.environment().isManuallyDeployed() && + ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as pending " + application.deploying().get() + - " is untested"); + " as pending " + application.deploying().get() + + " is untested"); DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); - // Don't update/store applicationpackage information when deploying previous application package (initial staging step) - if(! options.deployCurrentVersion) { - // Add missing information to application + if( ! options.deployCurrentVersion) { + // Add missing information to application (unless we're deploying the previous version (initial staging step) application = application.with(applicationPackage.deploymentSpec()); application = application.with(applicationPackage.validationOverrides()); if (options.screwdriverBuildJob.isPresent() && options.screwdriverBuildJob.get().screwdriverId != null) application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if (!triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { + if ( ! triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { // Triggering information is used to store which changes were made or attempted // - For self-triggered applications we don't have any trigger information, so we add it here. // - For all applications, we don't have complete control over which revision is actually built, @@ -489,8 +490,7 @@ public class ApplicationController { public Application deactivate(Application application, Deployment deployment, boolean requireThatDeploymentHasExpired) { try (Lock lock = lock(application.id())) { - // TODO: ignore no application errors for config server client, - // only return such errors from sherpa client. + // TODO: ignore no application errors for config server client, only return such errors from sherpa client. if (requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment, clock.instant())) return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index b3d75106d2f..22d0a56c367 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -47,9 +47,11 @@ public class Upgrader extends Maintainer { switch (target.confidence()) { case broken: - log.info(String.format("Version %s is broken, cancelling all upgrades", target.versionNumber())); - cancelUpgradesOf(applications().upgradingTo(target.versionNumber()) - .without(UpgradePolicy.canary)); // keep trying canaries + ApplicationList toCancel = applications().upgradingTo(target.versionNumber()) + .without(UpgradePolicy.canary); + if (toCancel.isEmpty()) break; + log.info("Version " + target.versionNumber() + " is broken, cancelling all upgrades"); + cancelUpgradesOf(toCancel); break; case low: upgrade(applications().with(UpgradePolicy.canary), target.versionNumber()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 4650e9bf317..bfabf0a4e4e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -149,6 +149,7 @@ public class NodeRepositoryImplTest { NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); waitForJdiscContainerToServe(); + nodeRepositoryApi.markAsDirty("host5.yahoo.com"); nodeRepositoryApi.markNodeAvailableForNewAllocation("host5.yahoo.com"); try { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 68bdffd75f9..7f595b4a541 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -467,29 +467,88 @@ public class NodeRepository extends AbstractComponent { } } + /* + * This method is used to enable a smooth rollout of dynamic docker flavor allocations. Once we have switch + * everything this can be simplified to only deleting the node. + * + * Should only be called by node-admin for docker containers + */ + public List<Node> markNodeAvailableForNewAllocation(String hostname) { + Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); + if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) { + throw new IllegalArgumentException( + "Cannot make " + hostname + " available for new allocation, must be a docker container node"); + } else if (node.state() != Node.State.dirty) { + throw new IllegalArgumentException( + "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state()); + } + + if (dynamicAllocationEnabled()) { + return removeRecursively(node, true); + } else { + return setReady(Collections.singletonList(node)); + } + } + /** - * Removes a node. A node must be in a legal state before it can be removed. + * Removes all the nodes that are children of hostname before finally removing the hostname itself. + * + * @return List of all the nodes that have been removed */ - public void remove(String hostname) { - Node nodeToRemove = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); - List<Node.State> legalStates = dynamicAllocationEnabled() ? - Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.dirty) : - Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked); - - if (! legalStates.contains(nodeToRemove.state())) { - throw new IllegalArgumentException("Can only remove node from following states: " + - legalStates.stream().map(Node.State::name).collect(Collectors.joining(", "))); + public List<Node> removeRecursively(String hostname) { + Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); + return removeRecursively(node, false); + } + + private List<Node> removeRecursively(Node node, boolean force) { + try (Mutex lock = lockUnallocated()) { + List<Node> removed = node.type() != NodeType.host ? + new ArrayList<>() : + getChildNodes(node.hostname()).stream() + .filter(child -> force || verifyRemovalIsAllowed(child, true)) + .collect(Collectors.toList()); + + if (force || verifyRemovalIsAllowed(node, false)) removed.add(node); + db.removeNodes(removed); + + return removed; + } catch (RuntimeException e) { + throw new IllegalArgumentException("Failed to delete " + node.hostname(), e); } + } - if (nodeToRemove.state().equals(Node.State.dirty)) { - if (!(nodeToRemove.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))) { - throw new IllegalArgumentException("Only docker nodes can be deleted from state dirty"); + /** + * Allowed to a node delete if: + * Non-docker-container node: iff in state provisioned|failed|parked + * Docker-container-node: + * If only removing the container node: node in state ready + * If also removing the parent node: child is in state provisioned|failed|parked|ready + */ + private boolean verifyRemovalIsAllowed(Node nodeToRemove, boolean deletingAsChild) { + // TODO: Enable once controller no longer deletes child nodes manually + /*if (nodeToRemove.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !deletingAsChild) { + if (nodeToRemove.state() != Node.State.ready) { + throw new IllegalArgumentException( + String.format("Docker container node %s can only be removed when in state ready", nodeToRemove.hostname())); } - } - try (Mutex lock = lock(nodeToRemove)) { - db.removeNode(nodeToRemove.state(), hostname); + } else */ if (nodeToRemove.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { + List<Node.State> legalStates = Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.ready); + + if (! legalStates.contains(nodeToRemove.state())) { + throw new IllegalArgumentException(String.format("Child node %s can only be removed from following states: %s", + nodeToRemove.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", ")))); + } + } else { + List<Node.State> legalStates = Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked); + + if (! legalStates.contains(nodeToRemove.state())) { + throw new IllegalArgumentException(String.format("Node %s can only be removed from following states: %s", + nodeToRemove.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", ")))); + } } + + return true; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index d057cf492ba..520ceaf323b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer { @Override protected void expire(List<Node> expired) { - for (Node expiredNode : expired.stream().collect(Collectors.toList())) + for (Node expiredNode : expired) nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index c7ef7f35ce0..2057fd7e36f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -44,6 +44,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final RetiredEarlyExpirer retiredEarlyExpirer; private final FailedExpirer failedExpirer; private final DirtyExpirer dirtyExpirer; + private final ProvisionedExpirer provisionedExpirer; private final NodeRebooter nodeRebooter; private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; @@ -73,6 +74,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { inactiveExpirer = new InactiveExpirer(nodeRepository, clock, durationFromEnv("inactive_expiry").orElse(defaults.inactiveExpiry), jobControl); failedExpirer = new FailedExpirer(nodeRepository, zone, clock, durationFromEnv("failed_expiry").orElse(defaults.failedExpiry), jobControl); dirtyExpirer = new DirtyExpirer(nodeRepository, clock, durationFromEnv("dirty_expiry").orElse(defaults.dirtyExpiry), jobControl); + provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl); nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); @@ -96,6 +98,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dirtyExpirer.deconstruct(); nodeRebooter.deconstruct(); nodeRetirer.deconstruct(); + provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); } @@ -133,6 +136,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration retiredExpiry; private final Duration failedExpiry; private final Duration dirtyExpiry; + private final Duration provisionedExpiry; private final Duration rebootInterval; private final Duration nodeRetirerInterval; private final Duration metricsInterval; @@ -154,6 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { retiredEarlyInterval = Duration.ofMinutes(29); failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night dirtyExpiry = Duration.ofHours(2); // enough time to clean the node + provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); @@ -171,6 +176,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { retiredEarlyInterval = Duration.ofMinutes(5); failedExpiry = Duration.ofMinutes(10); dirtyExpiry = Duration.ofMinutes(30); + provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java new file mode 100644 index 00000000000..b41eedd4694 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; + +/** + * This moves nodes from provisioned to parked if they have been in provisioned too long. + * + * @author freva + */ +public class ProvisionedExpirer extends Expirer { + + private final NodeRepository nodeRepository; + + public ProvisionedExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, JobControl jobControl) { + super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, clock, dirtyTimeout, jobControl); + this.nodeRepository = nodeRepository; + } + + @Override + protected void expire(List<Node> expired) { + for (Node expiredNode : expired) + nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned"); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 8f218d7e6fc..d38c9179986 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -106,18 +106,21 @@ public class CuratorDatabaseClient { } /** - * Removes a node. + * Removes multiple nodes in a single transaction. * - * @param state the current state of the node - * @param hostName the host name of the node to remove + * @param nodes list of the nodes to remove */ - public void removeNode(Node.State state, String hostName) { - Path path = toPath(state, hostName); + public void removeNodes(List<Node> nodes) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); + + for (Node node : nodes) { + Path path = toPath(node.state(), node.hostname()); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); + } + transaction.commit(); - log.log(LogLevel.INFO, "Removed: " + state + " node " + hostName); + nodes.forEach(node -> log.log(LogLevel.INFO, "Removed node " + node.hostname() + " in state " + node.state())); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 1ac86ad9f4b..3df79174b5c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -329,7 +329,7 @@ class NodePrioritizer { if (a.allocation().isPresent() && !b.allocation().isPresent()) return 1; // Choose container over content nodes - if (a.allocation().isPresent() && a.allocation().isPresent()) { + if (a.allocation().isPresent() && b.allocation().isPresent()) { if (a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) && !b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container)) return -1; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 4e7bb1f7d16..b16ce5f818e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -125,22 +125,10 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Moved " + lastElement(path) + " to active"); } else if (path.startsWith("/nodes/v2/state/availablefornewallocations/")) { - /** - * This is a temporary "state" or rest call that we use to enable a smooth rollout of - * dynamic docker flavor allocations. Once we have switch everything we remove this - * and change the code in the nodeadmin to delete directly (remember to allow deletion of dirty nodes then). - * - * Should only be called by node-admin for docker containers (the docker constraint is - * enforced in the remove method) - */ String hostname = lastElement(path); - if (nodeRepository.dynamicAllocationEnabled()) { - nodeRepository.remove(hostname); - return new MessageResponse("Removed " + hostname); - } else { - nodeRepository.setReady(hostname); - return new MessageResponse("Moved " + hostname + " to ready"); - } + List<Node> available = nodeRepository.markNodeAvailableForNewAllocation(hostname); + return new MessageResponse("Marked following nodes as available for new allocation: " + + available.stream().map(Node::hostname).collect(Collectors.joining(", "))); } throw new NotFoundException("Cannot put to path '" + path + "'"); @@ -180,8 +168,8 @@ public class NodesApiHandler extends LoggingRequestHandler { String path = request.getUri().getPath(); if (path.startsWith("/nodes/v2/node/")) { String hostname = lastElement(path); - nodeRepository.remove(hostname); - return new MessageResponse("Removed " + hostname); + List<Node> removedNodes = nodeRepository.removeRecursively(hostname); + return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); } else if (path.startsWith("/nodes/v2/maintenance/inactive/")) { return setActive(lastElement(path), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index bcae9d293f6..8eec56a1c00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.vespa.hosted.provision.node.Agent; +import org.junit.Ignore; import org.junit.Test; import java.nio.charset.StandardCharsets; @@ -28,18 +29,18 @@ public class NodeRepositoryTest { @Test public void nodeRepositoryTest() { NodeRepositoryTester tester = new NodeRepositoryTester(); - assertEquals(0, tester.getNodes(NodeType.tenant).size()); + assertEquals(0, tester.nodeRepository().getNodes().size()); tester.addNode("id1", "host1", "default", NodeType.tenant); tester.addNode("id2", "host2", "default", NodeType.tenant); tester.addNode("id3", "host3", "default", NodeType.tenant); - assertEquals(3, tester.getNodes(NodeType.tenant).size()); + assertEquals(3, tester.nodeRepository().getNodes().size()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - tester.nodeRepository().remove("host2"); + tester.nodeRepository().removeRecursively("host2"); - assertEquals(2, tester.getNodes(NodeType.tenant).size()); + assertEquals(2, tester.nodeRepository().getNodes().size()); } @Test @@ -69,21 +70,55 @@ public class NodeRepositoryTest { assertTrue(tester.nodeRepository().dynamicAllocationEnabled()); } + @Test @Ignore // TODO: Enable once controller no longer deletes child nodes manually + public void only_allow_docker_containers_remove_in_ready() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + tester.addNode("id1", "host1", "docker", NodeType.tenant); + + try { + tester.nodeRepository().removeRecursively("host1"); // host1 is in state provisioned + fail("Should not be able to delete docker container node by itself in state provisioned"); + } catch (IllegalArgumentException ignored) { + // Expected + } + + tester.nodeRepository().setDirty("host1"); + tester.nodeRepository().setReady("host1"); + tester.nodeRepository().removeRecursively("host1"); + } + @Test - public void only_allow_to_delete_dirty_nodes_when_dynamic_allocation_feature_enabled() { + public void delete_host_only_after_all_the_children_have_been_deleted() { NodeRepositoryTester tester = new NodeRepositoryTester(); + tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "docker", NodeType.tenant); - tester.nodeRepository().setDirty("host2"); + tester.addNode("id2", "host2", "default", NodeType.host); + tester.addNode("node10", "node10", "host1", "docker", NodeType.tenant); + tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); + tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); + tester.addNode("node20", "node20", "host2", "docker", NodeType.tenant); + assertEquals(6, tester.nodeRepository().getNodes().size()); + + tester.nodeRepository().setDirty("node11"); try { - tester.nodeRepository().remove("host2"); - fail("Should not be able to delete tenant node in state dirty"); + tester.nodeRepository().removeRecursively("host1"); + fail("Should not be able to delete host node, one of the children is in state dirty"); } catch (IllegalArgumentException ignored) { // Expected } + assertEquals(6, tester.nodeRepository().getNodes().size()); - tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); - tester.nodeRepository().remove("host2"); + // Should be OK to delete host2 as both host2 and its only child, node20, are in state provisioned + tester.nodeRepository().removeRecursively("host2"); + assertEquals(4, tester.nodeRepository().getNodes().size()); + + // Now node10 and node12 are in provisioned, set node11 to ready, and it should be OK to delete host1 + tester.nodeRepository().setReady("node11"); + tester.nodeRepository().removeRecursively("node11"); // Remove one of the children first instead + assertEquals(3, tester.nodeRepository().getNodes().size()); + + tester.nodeRepository().removeRecursively("host1"); + assertEquals(0, tester.nodeRepository().getNodes().size()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 784fc1a274a..3d01bde4291 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -51,6 +51,12 @@ public class NodeRepositoryTester { return nodeRepository.addNodes(Collections.singletonList(node)).get(0); } + public Node addNode(String id, String hostname, String parentHostname, String flavor, NodeType type) { + Node node = nodeRepository.createNode(id, hostname, Optional.of(parentHostname), + nodeFlavors.getFlavorOrThrow(flavor), type); + return nodeRepository.addNodes(Collections.singletonList(node)).get(0); + } + private FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index e304aac5463..8fd67f949d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -342,7 +342,7 @@ public class NodeFailerTest { assertEquals(15, tester.nodeRepository.getNodes(NodeType.proxy, Node.State.active).size()); // The first down host is removed, which causes the second one to be moved to failed - tester.nodeRepository.remove(failedHost1); + tester.nodeRepository.removeRecursively(failedHost1); tester.failer.run(); assertEquals( 2, tester.deployer.redeployments); assertEquals(14, tester.nodeRepository.getNodes(NodeType.proxy, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java index cb46d0a4624..bba5aa2db8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.zookeeper.ZooKeeperServer; @@ -49,7 +48,7 @@ public class ZooKeeperAccessMaintainerTest { assertEquals(asSet("host1,host2,host3,host4,host5,server1,server2"), ZooKeeperServer.getAllowedClientHostnames()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - tester.nodeRepository().remove("host2"); + tester.nodeRepository().removeRecursively("host2"); maintainer.maintain(); assertEquals(2, tester.getNodes(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java index 407101c7b8b..04c0af5d98a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java @@ -48,7 +48,7 @@ public class NodePrioritizerTest { Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); // Unallocated over allocated - ClusterSpec spec = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); + ClusterSpec spec = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); c = c.allocate(ApplicationId.defaultId(), ClusterMembership.from(spec, 0), Instant.now()); nodes.remove(c); nodes.add(c); @@ -58,12 +58,12 @@ public class NodePrioritizerTest { Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); // Container over content - ClusterSpec spec2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); - d = d.allocate(ApplicationId.defaultId(), ClusterMembership.from(spec, 0), Instant.now()); + ClusterSpec spec2 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); + d = d.allocate(ApplicationId.defaultId(), ClusterMembership.from(spec2, 0), Instant.now()); nodes.remove(d); nodes.add(d); - Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); - Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, d, parent)); + Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, d, parent)); } private static Node createNode(Node parent, String hostname, String flavor) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 565dfe0457e..38d3bf46028 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -401,7 +401,7 @@ public class RestApiTest { // Attempt to DELETE a node which is not put in a deletable state first assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", new byte[0], Request.Method.DELETE), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only remove node from following states: provisioned, failed, parked, dirty\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Failed to delete host2.yahoo.com: Node host2.yahoo.com can only be removed from following states: provisioned, failed, parked\"}"); // PUT current restart generation with string instead of long assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index deadafcad8f..679108ba872 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -494,11 +494,11 @@ TEST("require that sortspec can be used (multi-threaded)") { SearchReply::UP reply = world.performSearch(request, threads); ASSERT_EQUAL(9u, reply->hits.size()); EXPECT_EQUAL(document::DocumentId("doc::100").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(0.0, reply->hits[0].metric); + EXPECT_EQUAL(default_rank_value, reply->hits[0].metric); EXPECT_EQUAL(document::DocumentId("doc::200").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(0.0, reply->hits[1].metric); + EXPECT_EQUAL(default_rank_value, reply->hits[1].metric); EXPECT_EQUAL(document::DocumentId("doc::300").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(0.0, reply->hits[2].metric); + EXPECT_EQUAL(default_rank_value, reply->hits[2].metric); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index cd01c330931..2d056d0e761 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -53,7 +53,7 @@ private: Context(double rankDropLimit, MatchTools &matchTools, RankProgram & ranking, HitCollector & hits, uint32_t num_threads) __attribute__((noinline)); void rankHit(uint32_t docId); - void addHit(uint32_t docId) { _hits.addHit(docId, 0.0); } + void addHit(uint32_t docId) { _hits.addHit(docId, search::default_rank_value); } bool isBelowLimit() const { return matches < _matches_limit; } bool isAtLimit() const { return matches == _matches_limit; } bool atSoftDoom() const { return _softDoom.doom(); } diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index b38f5cdb168..3f49c6969a0 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -30,7 +30,7 @@ struct PredefinedScorer : public HitCollector::DocumentScorer ScoreMap _scores; PredefinedScorer(const ScoreMap &scores) : _scores(scores) {} virtual feature_t score(uint32_t docId) override { - feature_t retval = 0.0; + feature_t retval = default_rank_value; auto itr = _scores.find(docId); if (itr != _scores.end()) { retval = itr->second; @@ -436,7 +436,7 @@ TEST_F("require that result set is merged correctly with first phase ranking", expRh.push_back(RankedHit()); expRh.back()._docId = i; // only the maxHitsSize best hits gets a score - expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? 0 : i + 1000; + expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? default_rank_value : i + 1000; } std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); TEST_DO(checkResult(*rs.get(), expRh)); @@ -448,7 +448,7 @@ addExpectedHitForMergeTest(const MergeResultSetFixture &f, std::vector<RankedHit expRh.push_back(RankedHit()); expRh.back()._docId = docId; if (docId < f.numDocs - f.maxHitsSize) { // only the maxHitsSize best hits gets a score - expRh.back()._rankValue = 0; + expRh.back()._rankValue = default_rank_value; } else if (docId < f.numDocs - f.maxHeapSize) { // only first phase ranking expRh.back()._rankValue = docId + 500; // adjusted with - 500 } else { // second phase ranking on the maxHeapSize best hits @@ -497,7 +497,7 @@ TEST("require that hits can be added out of order when passing array limit") { for (uint32_t i = 0; i < numHits; ++i) { expRh.push_back(RankedHit()); expRh.back()._docId = i; - expRh.back()._rankValue = (i < 50) ? 0 : (i + 100); + expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } // add results in reverse order for (uint32_t i = numHits; i-- > 0; ) { @@ -516,7 +516,7 @@ TEST("require that hits can be added out of order only after passing array limit for (uint32_t i = 0; i < numHits; ++i) { expRh.push_back(RankedHit()); expRh.back()._docId = i; - expRh.back()._rankValue = (i < 50) ? 0 : (i + 100); + expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } // add results in reverse order const uint32_t numInOrder = numHits - 30; diff --git a/searchlib/src/vespa/searchlib/common/hitrank.h b/searchlib/src/vespa/searchlib/common/hitrank.h index abc714660cf..9f40cf9277c 100644 --- a/searchlib/src/vespa/searchlib/common/hitrank.h +++ b/searchlib/src/vespa/searchlib/common/hitrank.h @@ -2,10 +2,13 @@ #pragma once +#include <cmath> + namespace search { typedef double HitRank; typedef double SignedHitRank; +constexpr HitRank default_rank_value = -HUGE_VAL; } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/rankedhit.h b/searchlib/src/vespa/searchlib/common/rankedhit.h index bd66f884a08..d56fa92442f 100644 --- a/searchlib/src/vespa/searchlib/common/rankedhit.h +++ b/searchlib/src/vespa/searchlib/common/rankedhit.h @@ -11,8 +11,8 @@ namespace search { struct RankedHit { - RankedHit() : _docId(0), _rankValue(0) { } - RankedHit(unsigned int docId, HitRank rank=0.0) : _docId(docId), _rankValue(rank) { } + RankedHit() : _docId(0), _rankValue(default_rank_value) { } + RankedHit(unsigned int docId, HitRank rank = default_rank_value) : _docId(docId), _rankValue(rank) { } unsigned int getDocId() const { return _docId & 0x7fffffff; } bool hasMore() const { return _docId & 0x80000000; } HitRank getRank() const { return _rankValue; } diff --git a/searchlib/src/vespa/searchlib/common/resultset.cpp b/searchlib/src/vespa/searchlib/common/resultset.cpp index 69f3b13d709..5ad89bf49d7 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.cpp +++ b/searchlib/src/vespa/searchlib/common/resultset.cpp @@ -111,7 +111,7 @@ ResultSet::mergeWithBitOverflow() // bitvector hits before array hits while (bidx < firstArrayHit) { tgtA->_docId = bidx; - tgtA->_rankValue = 0; + tgtA->_rankValue = default_rank_value; tgtA++; bidx = bitVector->getNextTrueBit(bidx + 1); } @@ -123,7 +123,7 @@ ResultSet::mergeWithBitOverflow() tgtA->_rankValue = oldA->_rankValue; oldA++; } else { - tgtA->_rankValue = 0; + tgtA->_rankValue = default_rank_value; } tgtA++; bidx = bitVector->getNextTrueBit(bidx + 1); @@ -134,7 +134,7 @@ ResultSet::mergeWithBitOverflow() // bitvector hits after array hits while (tgtA < tgtAEnd) { tgtA->_docId = bidx; - tgtA->_rankValue = 0; + tgtA->_rankValue = default_rank_value; tgtA++; bidx = bitVector->getNextTrueBit(bidx + 1); } diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index 3601a93fdc2..7d79d0f659e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -306,7 +306,7 @@ HitCollector::getResultSet() rh[j]._rankValue = getReScore(_hits[i].second); ++i; } else { - rh[j]._rankValue = 0; + rh[j]._rankValue = default_rank_value; } } } else { @@ -317,7 +317,7 @@ HitCollector::getResultSet() rh[j]._rankValue = _hits[i].second; ++i; } else { - rh[j]._rankValue = 0; + rh[j]._rankValue = default_rank_value; } } } diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 7238e7de5b4..74b2f5bdbdb 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -35,7 +35,7 @@ Vagrant.configure("2") do |config| yum-builddep -y /vagrant/dist/vespa.spec echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf - wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.2.1.tar.gz | tar -C /opt -zx - ln -sf /opt/clion-2017.2.1/bin/clion.sh /usr/bin/clion + wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.2.2.tar.gz | tar -C /opt -zx + ln -sf /opt/clion-2017.2.2/bin/clion.sh /usr/bin/clion SHELL end diff --git a/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java index d6f73cea3e2..d8b964a2a3c 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java @@ -5,9 +5,10 @@ package com.yahoo.collections; * Utility class which is useful when implementing <code>Comparable</code> and one needs to * compare byte arrays as instance variables. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class ByteArrayComparator { + /** * Compare the arguments. Shorter arrays are always considered * smaller than longer arrays. For arrays of equal lengths, the elements @@ -29,17 +30,18 @@ public class ByteArrayComparator { return 1; } - //lengths are equal, compare contents + // lengths are equal, compare contents for (int i = 0; i < first.length; i++) { if (first[i] < second[i]) { return -1; } else if (first[i] > second[i]) { return 1; } - //values at index i are equal, continue... + // values at index i are equal, continue... } - //we haven't returned yet; contents must be equal: + // we haven't returned yet; contents must be equal: return 0; } + } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java index 5bf8016aad3..bb4487e68d6 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java @@ -12,7 +12,9 @@ import java.util.Map; * Instrospection methods are available, but you can also use accessors * with a default value if you expect a certain type and just want your * default value if some field doesn't exist or was of the wrong type. - **/ + * + * @author havardpe + */ public interface Inspector extends Inspectable { /** @@ -20,55 +22,55 @@ public interface Inspector extends Inspectable { * If you try to access a field or array entry that does not exist, * you will get an invalid Inspector returned. */ - public boolean valid(); + boolean valid(); /** Get the type of an inspector */ - public Type type(); + Type type(); /** Get the number of entries in an ARRAY (always returns 0 for non-arrays) */ - public int entryCount(); + int entryCount(); /** Get the number of fields in an OBJECT (always returns 0 for non-objects) */ - public int fieldCount(); + int fieldCount(); /** Access the inspector's value if it's a BOOLEAN; otherwise throws exception */ - public boolean asBool(); + boolean asBool(); /** Access the inspector's value if it's a LONG (or DOUBLE); otherwise throws exception */ - public long asLong(); + long asLong(); /** Access the inspector's value if it's a DOUBLE (or LONG); otherwise throws exception */ - public double asDouble(); + double asDouble(); /** Access the inspector's value if it's a STRING; otherwise throws exception */ - public String asString(); + String asString(); /** * Access the inspector's value (in utf-8 representation) if it's * a STRING; otherwise throws exception - **/ - public byte[] asUtf8(); + */ + byte[] asUtf8(); /** Access the inspector's value if it's DATA; otherwise throws exception */ - public byte[] asData(); + byte[] asData(); /** Get the inspector's value (or the supplied default), never throws */ - public boolean asBool(boolean defaultValue); + boolean asBool(boolean defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public long asLong(long defaultValue); + long asLong(long defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public double asDouble(double defaultValue); + double asDouble(double defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public String asString(String defaultValue); + String asString(String defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public byte[] asUtf8(byte[] defaultValue); + byte[] asUtf8(byte[] defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public byte[] asData(byte[] defaultValue); + byte[] asData(byte[] defaultValue); /** * Traverse an array value, performing callbacks for each entry. @@ -76,10 +78,11 @@ public interface Inspector extends Inspectable { * If the current Inspector is connected to an array value, * perform callbacks to the given traverser for each entry * contained in the array. Otherwise a no-op. - * @param at traverser callback object. - **/ + * + * @param at traverser callback object + */ @SuppressWarnings("overloads") - public void traverse(ArrayTraverser at); + void traverse(ArrayTraverser at); /** * Traverse an object value, performing callbacks for each field. @@ -87,10 +90,11 @@ public interface Inspector extends Inspectable { * If the current Inspector is connected to an object value, * perform callbacks to the given traverser for each field * contained in the object. Otherwise a no-op. - * @param ot traverser callback object. - **/ + * + * @param ot traverser callback object + */ @SuppressWarnings("overloads") - public void traverse(ObjectTraverser ot); + void traverse(ObjectTraverser ot); /** * Access an array entry. @@ -98,10 +102,11 @@ public interface Inspector extends Inspectable { * If the current Inspector doesn't connect to an array value, * or the given array index is out of bounds, the returned * Inspector will be invalid. - * @param idx array index. - * @return a new Inspector for the entry value. - **/ - public Inspector entry(int idx); + * + * @param idx array index + * @return a new Inspector for the entry value + */ + Inspector entry(int idx); /** * Access an field in an object. @@ -109,20 +114,22 @@ public interface Inspector extends Inspectable { * If the current Inspector doesn't connect to an object value, or * the object value does not contain a field with the given symbol * name, the returned Inspector will be invalid. - * @param name symbol name. - * @return a new Inspector for the field value. - **/ - public Inspector field(String name); + * + * @param name symbol name + * @return a new Inspector for the field value + */ + Inspector field(String name); /** * Convert an array to an iterable list. Other types will just * return an empty list. - **/ - public Iterable<Inspector> entries(); + */ + Iterable<Inspector> entries(); /** * Convert an object to an iterable list of (name, value) pairs. * Other types will just return an empty list. - **/ - public Iterable<Map.Entry<String,Inspector>> fields(); + */ + Iterable<Map.Entry<String,Inspector>> fields(); + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java index 89e7b3383fc..99489faca47 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java @@ -9,59 +9,61 @@ package com.yahoo.slime; * current Inspector is invalid or the type doesn't match your * accessor type. If you want to do something exceptional instead * when the types don't match, you must check using type() first. - **/ + * + * @author havardpe + */ public interface Inspector { /** check if this inspector is valid */ - public boolean valid(); + boolean valid(); /** return an enum describing value type */ - public Type type(); + Type type(); /** * Check how many entries or fields are contained in the current value. * Useful for arrays and objects; anything else always returns 0. * @return number of entries/fields contained. - **/ - public int children(); + */ + int children(); /** * Check how many entries are contained in the current value. * Useful for arrays; anything else always returns 0. * @return number of entries contained. - **/ - public int entries(); + */ + int entries(); /** * Check how many fields are contained in the current value. * Useful for objects; anything else always returns 0. * @return number of fields contained. - **/ - public int fields(); + */ + int fields(); /** the current value (for booleans); default: false */ - public boolean asBool(); + boolean asBool(); /** the current value (for integers); default: 0 */ - public long asLong(); + long asLong(); /** the current value (for floating-point values); default: 0.0 */ - public double asDouble(); + double asDouble(); /** the current value (for string values); default: empty string */ - public String asString(); + String asString(); /** the current value encoded into UTF-8 (for string values); default: empty array */ - public byte[] asUtf8(); + byte[] asUtf8(); /** the current value (for data values); default: empty array */ - public byte[] asData(); + byte[] asData(); /** * Use the visitor pattern to resolve the underlying type of this value. * @param v the visitor - **/ - public void accept(Visitor v); + */ + void accept(Visitor v); /** * Traverse an array value, performing callbacks for each entry. @@ -70,9 +72,9 @@ public interface Inspector { * perform callbacks to the given traverser for each entry * contained in the array. * @param at traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ArrayTraverser at); + void traverse(ArrayTraverser at); /** * Traverse an object value, performing callbacks for each field. @@ -81,9 +83,9 @@ public interface Inspector { * perform callbacks to the given traverser for each field * contained in the object. * @param ot traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ObjectSymbolTraverser ot); + void traverse(ObjectSymbolTraverser ot); /** * Traverse an object value, performing callbacks for each field. @@ -92,9 +94,9 @@ public interface Inspector { * perform callbacks to the given traverser for each field * contained in the object. * @param ot traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ObjectTraverser ot); + void traverse(ObjectTraverser ot); /** * Access an array entry. @@ -104,8 +106,8 @@ public interface Inspector { * Inspector will be invalid. * @param idx array index. * @return a new Inspector for the entry value. - **/ - public Inspector entry(int idx); + */ + Inspector entry(int idx); /** * Access an field in an object by symbol id. @@ -115,8 +117,8 @@ public interface Inspector { * id, the returned Inspector will be invalid. * @param sym symbol id. * @return a new Inspector for the field value. - **/ - public Inspector field(int sym); + */ + Inspector field(int sym); /** * Access an field in an object by symbol name. @@ -126,6 +128,7 @@ public interface Inspector { * name, the returned Inspector will be invalid. * @param name symbol name. * @return a new Inspector for the field value. - **/ - public Inspector field(String name); + */ + Inspector field(String name); + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java index b601d0b72ff..84f193caa4d 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -5,26 +5,29 @@ package com.yahoo.slime; * Top-level value class that contains one Value data object and a * symbol table (shared between all directly or indirectly contained * ObjectValue data objects). + * + * @author havardpe **/ -public final class Slime -{ +public final class Slime { + private final SymbolTable names = new SymbolTable(); private Value root = NixValue.instance(); /** * Construct an empty Slime with an empty top-level value. - **/ + */ public Slime() {} - /** return count of names in the symbol table. */ + /** Returns a count of names in the symbol table. */ public int symbols() { return names.symbols(); } /** * Return the symbol name associated with an id. + * * @param symbol the id, must be in range [0, symbols()-1] - **/ + */ public String inspect(int symbol) { return names.inspect(symbol); } @@ -32,9 +35,10 @@ public final class Slime /** * Add a name to the symbol table; if the name is already * in the symbol table just returns the id it already had. + * * @param name the name to insert * @return the id now associated with the name - **/ + */ public int insert(String name) { return names.insert(name); } @@ -43,7 +47,7 @@ public final class Slime * Find the id associated with a symbol name; if the * name was not in the symbol table returns the * constant Integer.MAX_VALUE instead. - **/ + */ public int lookup(String name) { return names.lookup(name); } @@ -53,7 +57,7 @@ public final class Slime /** * Create a new empty value and make it the new top-level data object. - **/ + */ public Cursor setNix() { root = NixValue.instance(); return root; @@ -61,8 +65,9 @@ public final class Slime /** * Create a new boolean value and make it the new top-level data object. + * * @param bit the actual boolean value for the new value - **/ + */ public Cursor setBool(boolean bit) { root = BoolValue.instance(bit); return root; @@ -70,8 +75,9 @@ public final class Slime /** * Create a new double value and make it the new top-level data object. + * * @param l the actual long value for the new value - **/ + */ public Cursor setLong(long l) { root = new LongValue(l); return root; @@ -79,8 +85,9 @@ public final class Slime /** * Create a new double value and make it the new top-level data object. + * * @param d the actual double value for the new value - **/ + */ public Cursor setDouble(double d) { root = new DoubleValue(d); return root; @@ -88,8 +95,9 @@ public final class Slime /** * Create a new string value and make it the new top-level data object. + * * @param str the actual string for the new value - **/ + */ public Cursor setString(String str) { root = StringValue.create(str); return root; @@ -97,8 +105,9 @@ public final class Slime /** * Create a new string value and make it the new top-level data object. + * * @param utf8 the actual string (encoded as UTF-8 data) for the new value - **/ + */ public Cursor setString(byte[] utf8) { root = Utf8Value.create(utf8); return root; @@ -106,8 +115,9 @@ public final class Slime /** * Create a new data value and make it the new top-level data object. + * * @param data the actual data to be put into the new value. - **/ + */ public Cursor setData(byte[] data) { root = DataValue.create(data); return root; @@ -115,7 +125,7 @@ public final class Slime /** * Create a new array value and make it the new top-level data object. - **/ + */ public Cursor setArray() { root = new ArrayValue(names); return root; @@ -123,7 +133,7 @@ public final class Slime /** * Create a new object value and make it the new top-level data object. - **/ + */ public Cursor setObject() { root = new ObjectValue(names); return root; @@ -133,7 +143,7 @@ public final class Slime * Take the current top-level data object and make it a field in a * new ObjectValue with the given symbol id as field id; the new * ObjectValue will also become the new top-level data object. - **/ + */ public Cursor wrap(int sym) { root = new ObjectValue(names, sym, root); return root; @@ -143,8 +153,9 @@ public final class Slime * Take the current top-level data object and make it a field in a * new ObjectValue with the given symbol name as field name; the new * ObjectValue will also become the new top-level data object. - **/ + */ public Cursor wrap(String name) { return wrap(names.insert(name)); } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 3ac7b388ec4..b6f8775b64b 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -68,6 +68,7 @@ vespa_define_module( src/tests/simple_thread_bundle src/tests/slaveproc src/tests/slime + src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark src/tests/stash src/tests/stllike diff --git a/vespalib/src/tests/slime/external_data_value/CMakeLists.txt b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt new file mode 100644 index 00000000000..df4cb8a23c3 --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_external_data_value_test_app TEST + SOURCES + external_data_value_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_external_data_value_test_app COMMAND vespalib_external_data_value_test_app) diff --git a/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp new file mode 100644 index 00000000000..acf937c840d --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> + +using namespace vespalib::slime::convenience; +using vespalib::slime::ExternalMemory; + +struct MyMem : ExternalMemory { + const std::vector<char> space; + MyMem(Memory memory) + : space(memory.data, memory.data + memory.size) {} + Memory get() const override { + return Memory(&space[0], space.size()); + } + static UP create(Memory memory) { + return std::make_unique<MyMem>(memory); + } +}; + +void verify_data(const Inspector &pos, Memory expect) { + EXPECT_TRUE(pos.valid()); + EXPECT_EQUAL(vespalib::slime::DATA::ID, pos.type().getId()); + EXPECT_EQUAL(pos.asString(), Memory()); + EXPECT_EQUAL(pos.asData(), expect); +} + +TEST("require that external memory can be used for data values") { + Slime slime; + TEST_DO(verify_data(slime.setData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that nullptr external memory gives empty data value") { + Slime slime; + TEST_DO(verify_data(slime.setData(ExternalMemory::UP(nullptr)), Memory(""))); + TEST_DO(verify_data(slime.get(), Memory(""))); +} + +TEST("require that external memory can be used with array data values") { + Slime slime; + TEST_DO(verify_data(slime.setArray().addData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (name)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData("field", MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (symbol)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData(Symbol(5), MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST("require that external memory can be used with slime inserter") { + Slime slime; + SlimeInserter inserter(slime); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that external memory can be used with array inserter") { + Slime slime; + ArrayInserter inserter(slime.setArray()); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object inserter") { + Slime slime; + ObjectInserter inserter(slime.setObject(), "field"); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object symbol inserter") { + Slime slime; + ObjectSymbolInserter inserter(slime.setObject(), Symbol(5)); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt index c5aa9b595d2..46a8d410878 100644 --- a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt @@ -9,6 +9,9 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT convenience.cpp cursor.cpp empty_value_factory.cpp + external_data_value.cpp + external_data_value_factory.cpp + external_memory.cpp inject.cpp inserter.cpp inspector.cpp diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h index 4242936a483..6815ad3ba83 100644 --- a/vespalib/src/vespa/vespalib/data/slime/cursor.h +++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h @@ -3,6 +3,7 @@ #pragma once #include "inspector.h" +#include "external_memory.h" namespace vespalib { namespace slime { @@ -18,6 +19,7 @@ struct Cursor : public Inspector { virtual Cursor &addDouble(double d) = 0; virtual Cursor &addString(Memory str) = 0; virtual Cursor &addData(Memory data) = 0; + virtual Cursor &addData(ExternalMemory::UP data) = 0; virtual Cursor &addArray() = 0; virtual Cursor &addObject() = 0; @@ -27,6 +29,7 @@ struct Cursor : public Inspector { virtual Cursor &setDouble(Symbol sym, double d) = 0; virtual Cursor &setString(Symbol sym, Memory str) = 0; virtual Cursor &setData(Symbol sym, Memory data) = 0; + virtual Cursor &setData(Symbol sym, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Symbol sym) = 0; virtual Cursor &setObject(Symbol sym) = 0; @@ -35,7 +38,8 @@ struct Cursor : public Inspector { virtual Cursor &setLong(Memory name, int64_t l) = 0; virtual Cursor &setDouble(Memory name, double d) = 0; virtual Cursor &setString(Memory name, Memory str) = 0; - virtual Cursor &setData(Memory name, Memory str) = 0; + virtual Cursor &setData(Memory name, Memory data) = 0; + virtual Cursor &setData(Memory name, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Memory name) = 0; virtual Cursor &setObject(Memory name) = 0; diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp new file mode 100644 index 00000000000..1004f8aab24 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp @@ -0,0 +1,7 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value.h" + +namespace vespalib::slime { + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h new file mode 100644 index 00000000000..3acfb1350bd --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "external_memory.h" + +namespace vespalib::slime { + +/** + * A data value backed by external memory. + **/ +class ExternalDataValue : public Value +{ +private: + ExternalMemory::UP _value; +public: + ExternalDataValue(ExternalMemory::UP data) : _value(std::move(data)) {} + Memory asData() const override { return _value->get(); } + Type type() const override { return DATA::instance; } +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp new file mode 100644 index 00000000000..592ddcdb519 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value_factory.h" +#include "external_data_value.h" +#include "basic_value.h" + +namespace vespalib::slime { + +Value * +ExternalDataValueFactory::create(Stash &stash) const +{ + if (!input) { + return &stash.create<BasicDataValue>(Memory(), stash); + } + return &stash.create<ExternalDataValue>(std::move(input)); +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h new file mode 100644 index 00000000000..be85f28aebe --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value_factory.h" +#include "external_memory.h" +#include <vespa/vespalib/util/stash.h> + +namespace vespalib::slime { + +/** + * Value factory for data values using external memory. + **/ +struct ExternalDataValueFactory : public ValueFactory { + mutable ExternalMemory::UP input; + ExternalDataValueFactory(ExternalMemory::UP in) : input(std::move(in)) {} + Value *create(Stash &stash) const override; +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp new file mode 100644 index 00000000000..ebbe833098a --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_memory.h" + +namespace vespalib::slime { + +ExternalMemory::~ExternalMemory() +{ +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.h b/vespalib/src/vespa/vespalib/data/slime/external_memory.h new file mode 100644 index 00000000000..a1d4510d0c8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.h @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/vespalib/data/memory.h> + +namespace vespalib::slime { + +/** + * Interface used to access external memory. External memory does not + * need to be copied when added to a Slime object. The Memory obtained + * by calling the get function must be valid until the object + * implementing this interface is destructed. + **/ +struct ExternalMemory { + using UP = std::unique_ptr<ExternalMemory>; + virtual Memory get() const = 0; + virtual ~ExternalMemory(); +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp index 81b8ffe334c..f9d80e74e6f 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp @@ -6,12 +6,15 @@ namespace vespalib { namespace slime { +using ExtMemUP = ExternalMemory::UP; + Cursor &SlimeInserter::insertNix() const { return slime.setNix(); } Cursor &SlimeInserter::insertBool(bool value) const { return slime.setBool(value); } Cursor &SlimeInserter::insertLong(int64_t value) const { return slime.setLong(value); } Cursor &SlimeInserter::insertDouble(double value) const { return slime.setDouble(value); } Cursor &SlimeInserter::insertString(Memory value) const { return slime.setString(value); } Cursor &SlimeInserter::insertData(Memory value) const { return slime.setData(value); } +Cursor &SlimeInserter::insertData(ExtMemUP value) const { return slime.setData(std::move(value)); } Cursor &SlimeInserter::insertArray() const { return slime.setArray(); } Cursor &SlimeInserter::insertObject() const { return slime.setObject(); } @@ -21,6 +24,7 @@ Cursor &ArrayInserter::insertLong(int64_t value) const { return cursor.addLong( Cursor &ArrayInserter::insertDouble(double value) const { return cursor.addDouble(value); } Cursor &ArrayInserter::insertString(Memory value) const { return cursor.addString(value); } Cursor &ArrayInserter::insertData(Memory value) const { return cursor.addData(value); } +Cursor &ArrayInserter::insertData(ExtMemUP value) const { return cursor.addData(std::move(value)); } Cursor &ArrayInserter::insertArray() const { return cursor.addArray(); } Cursor &ArrayInserter::insertObject() const { return cursor.addObject(); } @@ -30,6 +34,7 @@ Cursor &ObjectSymbolInserter::insertLong(int64_t value) const { return cursor.s Cursor &ObjectSymbolInserter::insertDouble(double value) const { return cursor.setDouble(symbol, value); } Cursor &ObjectSymbolInserter::insertString(Memory value) const { return cursor.setString(symbol, value); } Cursor &ObjectSymbolInserter::insertData(Memory value) const { return cursor.setData(symbol, value); } +Cursor &ObjectSymbolInserter::insertData(ExtMemUP value) const { return cursor.setData(symbol, std::move(value)); } Cursor &ObjectSymbolInserter::insertArray() const { return cursor.setArray(symbol); } Cursor &ObjectSymbolInserter::insertObject() const { return cursor.setObject(symbol); } @@ -39,6 +44,7 @@ Cursor &ObjectInserter::insertLong(int64_t value) const { return cursor.setLong Cursor &ObjectInserter::insertDouble(double value) const { return cursor.setDouble(name, value); } Cursor &ObjectInserter::insertString(Memory value) const { return cursor.setString(name, value); } Cursor &ObjectInserter::insertData(Memory value) const { return cursor.setData(name, value); } +Cursor &ObjectInserter::insertData(ExtMemUP value) const { return cursor.setData(name, std::move(value)); } Cursor &ObjectInserter::insertArray() const { return cursor.setArray(name); } Cursor &ObjectInserter::insertObject() const { return cursor.setObject(name); } diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index b8762ed3794..dff37183ac7 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -5,6 +5,7 @@ #include "type.h" #include <vespa/vespalib/data/memory.h> #include "symbol.h" +#include "external_memory.h" namespace vespalib { @@ -27,6 +28,7 @@ struct Inserter { virtual Cursor &insertDouble(double value) const = 0; virtual Cursor &insertString(Memory value) const = 0; virtual Cursor &insertData(Memory value) const = 0; + virtual Cursor &insertData(ExternalMemory::UP value) const = 0; virtual Cursor &insertArray() const = 0; virtual Cursor &insertObject() const = 0; virtual ~Inserter() {} @@ -44,6 +46,7 @@ struct SlimeInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -58,6 +61,7 @@ struct ArrayInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -73,6 +77,7 @@ struct ObjectSymbolInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -88,6 +93,7 @@ struct ObjectInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index e5c1d6db1a8..af081887c9a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -28,6 +28,7 @@ #include "type.h" #include "value.h" #include "value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/input_reader.h> #include <vespa/vespalib/data/output_writer.h> #include <vespa/vespalib/data/simple_buffer.h> @@ -147,6 +148,9 @@ public: Cursor &setData(const Memory& data) { return _root.set(slime::DataValueFactory(data)); } + Cursor &setData(slime::ExternalMemory::UP data) { + return _root.set(slime::ExternalDataValueFactory(std::move(data))); + } Cursor &setArray() { return _root.set(slime::ArrayValueFactory(*_names)); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp index 9b415080d7c..2eae660f431 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp @@ -5,6 +5,7 @@ #include "resolved_symbol.h" #include "empty_value_factory.h" #include "basic_value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/simple_buffer.h> #include "json_format.h" @@ -80,7 +81,7 @@ Value::toString() const return buf.get().make_string(); } -// 6 x add +// 7 x add Cursor & Value::addNix() { return addLeaf(NixValueFactory()); } Cursor & @@ -93,8 +94,10 @@ Cursor & Value::addString(Memory str) { return addLeaf(StringValueFactory(str)); } Cursor & Value::addData(Memory data) { return addLeaf(DataValueFactory(data)); } +Cursor & +Value::addData(ExternalMemory::UP data) { return addLeaf(ExternalDataValueFactory(std::move(data))); } -// 6 x set (with numeric symbol id) +// 7 x set (with numeric symbol id) Cursor & Value::setNix(Symbol sym) { return setLeaf(sym, NixValueFactory()); } Cursor & @@ -107,8 +110,10 @@ Cursor & Value::setString(Symbol sym, Memory str) { return setLeaf(sym, StringValueFactory(str)); } Cursor & Value::setData(Symbol sym, Memory data) { return setLeaf(sym, DataValueFactory(data)); } +Cursor & +Value::setData(Symbol sym, ExternalMemory::UP data) { return setLeaf(sym, ExternalDataValueFactory(std::move(data))); } -// 6 x set (with symbol name) +// 7 x set (with symbol name) Cursor & Value::setNix(Memory name) { return setLeaf(name, NixValueFactory()); } Cursor & @@ -121,6 +126,8 @@ Cursor & Value::setString(Memory name, Memory str) { return setLeaf(name, StringValueFactory(str)); } Cursor & Value::setData(Memory name, Memory data) { return setLeaf(name, DataValueFactory(data)); } +Cursor & +Value::setData(Memory name, ExternalMemory::UP data) { return setLeaf(name, ExternalDataValueFactory(std::move(data))); } // nop defaults for array/objects Cursor & diff --git a/vespalib/src/vespa/vespalib/data/slime/value.h b/vespalib/src/vespa/vespalib/data/slime/value.h index dacc9b1800e..eaf14829ed9 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.h +++ b/vespalib/src/vespa/vespalib/data/slime/value.h @@ -57,6 +57,7 @@ public: Cursor &addDouble(double d) override; Cursor &addString(Memory str) override; Cursor &addData(Memory data) override; + Cursor &addData(ExternalMemory::UP data) override; Cursor &addArray() override; Cursor &addObject() override; @@ -66,6 +67,7 @@ public: Cursor &setDouble(Symbol sym, double d) override; Cursor &setString(Symbol sym, Memory str) override; Cursor &setData(Symbol sym, Memory data) override; + Cursor &setData(Symbol sym, ExternalMemory::UP data) override; Cursor &setArray(Symbol sym) override; Cursor &setObject(Symbol sym) override; @@ -75,6 +77,7 @@ public: Cursor &setDouble(Memory name, double d) override; Cursor &setString(Memory name, Memory str) override; Cursor &setData(Memory name, Memory str) override; + Cursor &setData(Memory name, ExternalMemory::UP data) override; Cursor &setArray(Memory name) override; Cursor &setObject(Memory name) override; diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt index cf0aab705a3..52c0c03d079 100644 --- a/vespamalloc/src/tests/allocfree/CMakeLists.txt +++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt @@ -25,4 +25,4 @@ vespa_add_executable(vespamalloc_linklist_test_app ) vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app - vespamalloc vespamalloc_vespamallocd) + vespamalloc vespamallocd) diff --git a/vespamalloc/src/tests/allocfree/allocfree_test.sh b/vespamalloc/src/tests/allocfree/allocfree_test.sh index d52bec12bd9..810a66f632b 100755 --- a/vespamalloc/src/tests/allocfree/allocfree_test.sh +++ b/vespamalloc/src/tests/allocfree/allocfree_test.sh @@ -5,7 +5,7 @@ set -e TIME=/usr/bin/time VESPA_MALLOC_SO=../../../src/vespamalloc/libvespamalloc.so -VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamalloc_vespamallocd.so +VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamallocd.so LD_PRELOAD=$VESPA_MALLOC_SO ./vespamalloc_realloc_test_app LD_PRELOAD=$VESPA_MALLOC_SO_D ./vespamalloc_realloc_test_app diff --git a/vespamalloc/src/tests/doubledelete/CMakeLists.txt b/vespamalloc/src/tests/doubledelete/CMakeLists.txt index c6fe46be614..2017a242d1b 100644 --- a/vespamalloc/src/tests/doubledelete/CMakeLists.txt +++ b/vespamalloc/src/tests/doubledelete/CMakeLists.txt @@ -11,4 +11,4 @@ vespa_add_executable(vespamalloc_expectsignal_app ) vespa_add_test(NAME vespamalloc_doubledelete_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/doubledelete_test.sh DEPENDS vespamalloc_doubledelete_test_app vespamalloc_expectsignal_app - vespamalloc vespamalloc_vespamallocd) + vespamalloc vespamallocd) diff --git a/vespamalloc/src/tests/doubledelete/doubledelete_test.sh b/vespamalloc/src/tests/doubledelete/doubledelete_test.sh index 7bea46f6a57..ab0b4943b11 100755 --- a/vespamalloc/src/tests/doubledelete/doubledelete_test.sh +++ b/vespamalloc/src/tests/doubledelete/doubledelete_test.sh @@ -5,4 +5,4 @@ set -e LD_PRELOAD=../../../src/vespamalloc/libvespamalloc.so ./vespamalloc_doubledelete_test_app ulimit -c 0 -./vespamalloc_expectsignal_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamalloc_vespamallocd.so ./vespamalloc_doubledelete_test_app" +./vespamalloc_expectsignal_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamallocd.so ./vespamalloc_doubledelete_test_app" diff --git a/vespamalloc/src/tests/overwrite/CMakeLists.txt b/vespamalloc/src/tests/overwrite/CMakeLists.txt index 5855b35ada9..29b6ac46eb4 100644 --- a/vespamalloc/src/tests/overwrite/CMakeLists.txt +++ b/vespamalloc/src/tests/overwrite/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_executable(vespamalloc_expectsignal-overwrite_app ) vespa_add_test(NAME vespamalloc_overwrite_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/overwrite_test.sh DEPENDS vespamalloc_overwrite_test_app vespamalloc_expectsignal-overwrite_app - vespamalloc vespamalloc_vespamallocd) + vespamalloc vespamallocd) diff --git a/vespamalloc/src/tests/overwrite/overwrite_test.sh b/vespamalloc/src/tests/overwrite/overwrite_test.sh index 9d7de928a75..05b1836e94e 100755 --- a/vespamalloc/src/tests/overwrite/overwrite_test.sh +++ b/vespamalloc/src/tests/overwrite/overwrite_test.sh @@ -3,8 +3,8 @@ set -e LD_PRELOAD=../../../src/vespamalloc/libvespamalloc.so ./vespamalloc_overwrite_test_app -LD_PRELOAD=../../../src/vespamalloc/libvespamalloc_vespamallocd.so ./vespamalloc_overwrite_test_app testmemoryfill +LD_PRELOAD=../../../src/vespamalloc/libvespamallocd.so ./vespamalloc_overwrite_test_app testmemoryfill ulimit -c 0; -./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamalloc_vespamallocd.so ./vespamalloc_overwrite_test_app prewrite" -./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamalloc_vespamallocd.so ./vespamalloc_overwrite_test_app postwrite" -./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamalloc_vespamallocd.so ./vespamalloc_overwrite_test_app writeafterfree" +./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamallocd.so ./vespamalloc_overwrite_test_app prewrite" +./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamallocd.so ./vespamalloc_overwrite_test_app postwrite" +./vespamalloc_expectsignal-overwrite_app 6 "LD_PRELOAD=../../../src/vespamalloc/libvespamallocd.so ./vespamalloc_overwrite_test_app writeafterfree" diff --git a/vespamalloc/src/tests/stacktrace/CMakeLists.txt b/vespamalloc/src/tests/stacktrace/CMakeLists.txt index 1259f52c326..491b0d4089e 100644 --- a/vespamalloc/src/tests/stacktrace/CMakeLists.txt +++ b/vespamalloc/src/tests/stacktrace/CMakeLists.txt @@ -8,5 +8,5 @@ vespa_add_executable(vespamalloc_stacktrace_test_app TEST vespa_add_test( NAME vespamalloc_stacktrace_test_app NO_VALGRIND COMMAND vespamalloc_stacktrace_test_app - ENVIRONMENT "LD_PRELOAD=${CMAKE_CURRENT_BINARY_DIR}/../../vespamalloc/libvespamalloc_vespamallocdst16.so" + ENVIRONMENT "LD_PRELOAD=${CMAKE_CURRENT_BINARY_DIR}/../../vespamalloc/libvespamallocdst16.so" NO_VALGRIND) diff --git a/vespamalloc/src/tests/thread/CMakeLists.txt b/vespamalloc/src/tests/thread/CMakeLists.txt index 2be281a6bd0..53d61f4aa76 100644 --- a/vespamalloc/src/tests/thread/CMakeLists.txt +++ b/vespamalloc/src/tests/thread/CMakeLists.txt @@ -9,4 +9,4 @@ vespa_add_executable(vespamalloc_racemanythreads_test_app ) vespa_add_test(NAME vespamalloc_thread_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/thread_test.sh DEPENDS vespamalloc_thread_test_app vespamalloc_racemanythreads_test_app - vespamalloc vespamalloc_vespamallocd) + vespamalloc vespamallocd) diff --git a/vespamalloc/src/tests/thread/thread_test.sh b/vespamalloc/src/tests/thread/thread_test.sh index 8b3d94e922d..b06c27e4a17 100755 --- a/vespamalloc/src/tests/thread/thread_test.sh +++ b/vespamalloc/src/tests/thread/thread_test.sh @@ -15,7 +15,7 @@ else fi VESPA_MALLOC_SO=../../../src/vespamalloc/libvespamalloc.so -VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamalloc_vespamallocd.so +VESPA_MALLOC_SO_D=../../../src/vespamalloc/libvespamallocd.so LD_PRELOAD=$VESPA_MALLOC_SO ./vespamalloc_thread_test_app return 20 LD_PRELOAD=$VESPA_MALLOC_SO ./vespamalloc_thread_test_app exit 20 diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index b065df672dd..d43d779217d 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_library(vespamalloc DEPENDS dl ) -vespa_add_library(vespamalloc_vespamallocd +vespa_add_library(vespamallocd SOURCES $<TARGET_OBJECTS:vespamalloc_mallocd> $<TARGET_OBJECTS:vespamalloc_util> @@ -15,7 +15,7 @@ vespa_add_library(vespamalloc_vespamallocd DEPENDS dl ) -vespa_add_library(vespamalloc_vespamallocdst16 +vespa_add_library(vespamallocdst16 SOURCES $<TARGET_OBJECTS:vespamalloc_mallocdst16> $<TARGET_OBJECTS:vespamalloc_util> @@ -23,7 +23,7 @@ vespa_add_library(vespamalloc_vespamallocdst16 DEPENDS dl ) -vespa_add_library(vespamalloc_vespamallocdst16_nl +vespa_add_library(vespamallocdst16_nl SOURCES $<TARGET_OBJECTS:vespamalloc_mallocdst16_nl> $<TARGET_OBJECTS:vespamalloc_util> @@ -31,7 +31,7 @@ vespa_add_library(vespamalloc_vespamallocdst16_nl DEPENDS dl ) -vespa_add_library(vespamalloc_vespammap +vespa_add_library(vespammap SOURCES $<TARGET_OBJECTS:vespamalloc_mmap> INSTALL lib64/vespa/malloc |