diff options
518 files changed, 2173 insertions, 1264 deletions
diff --git a/Code-of-Conduct.md b/Code-of-Conduct.md index 7a594941b58..e1f94da1039 100644 --- a/Code-of-Conduct.md +++ b/Code-of-Conduct.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Oath Open Source Code of Conduct ## Summary diff --git a/abi-check-plugin/README.md b/abi-check-plugin/README.md index 23a68a94783..b468392a568 100644 --- a/abi-check-plugin/README.md +++ b/abi-check-plugin/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # abi-check-plugin Maven plugin for ensuring project ABI stability. diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java index c7863d85a3c..9d149e090e3 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.classtree; import java.io.IOException; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java index cc2b37f13c9..d6e61ab62aa 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.collector; import java.util.HashSet; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java index 8bd7f6f7927..27055724e11 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.collector; import com.yahoo.abicheck.signature.JavaClassSignature; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java index f308ad42db5..94bf3359021 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.collector; import java.util.ArrayList; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java index c5452ecdde3..d7bd74a2ed2 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.mojo; import com.google.gson.Gson; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/setmatcher/SetMatcher.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/setmatcher/SetMatcher.java index 56b08bef74f..8bd6167fe04 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/setmatcher/SetMatcher.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/setmatcher/SetMatcher.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.setmatcher; import com.google.common.collect.Sets; diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/signature/JavaClassSignature.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/signature/JavaClassSignature.java index 5e748d1b7e2..18ac5492d65 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/signature/JavaClassSignature.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/signature/JavaClassSignature.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.signature; import java.util.List; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java index eaa49789f2f..c8ab5d2b4db 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AnnotationCollectorTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AnnotationCollectorTest.java index 3104ce794cd..34054419f5f 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AnnotationCollectorTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AnnotationCollectorTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java index 5aa8ed2d770..18f0655735e 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/Public.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/Public.java index 8e7410e032d..1809774fef3 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/Public.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/Public.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import java.lang.annotation.ElementType; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/PublicSignatureCollectorTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/PublicSignatureCollectorTest.java index 784a81de50c..a9c80219c74 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/PublicSignatureCollectorTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/PublicSignatureCollectorTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/SetMatcherTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/SetMatcherTest.java index a818a4fb701..51ab26d75e6 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/SetMatcherTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/SetMatcherTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java index 4de6f186800..ea93bf288d3 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.mojo; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/abi-check-plugin/src/test/java/root/Root.java b/abi-check-plugin/src/test/java/root/Root.java index 58fd210159f..c9af7e34979 100644 --- a/abi-check-plugin/src/test/java/root/Root.java +++ b/abi-check-plugin/src/test/java/root/Root.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package root; public class Root { diff --git a/abi-check-plugin/src/test/java/root/package-info.java b/abi-check-plugin/src/test/java/root/package-info.java index 26f0f5c5a6f..919f49b53ba 100644 --- a/abi-check-plugin/src/test/java/root/package-info.java +++ b/abi-check-plugin/src/test/java/root/package-info.java @@ -1,4 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @Public package root; -import com.yahoo.abicheck.Public;
\ No newline at end of file +import com.yahoo.abicheck.Public; diff --git a/abi-check-plugin/src/test/java/root/sub/Sub.java b/abi-check-plugin/src/test/java/root/sub/Sub.java index c4f330644e9..e9334bc86f7 100644 --- a/abi-check-plugin/src/test/java/root/sub/Sub.java +++ b/abi-check-plugin/src/test/java/root/sub/Sub.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package root.sub; public class Sub { diff --git a/application/pom.xml b/application/pom.xml index 5a370e9ffe6..7bcda0ce386 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -97,10 +97,6 @@ </exclusions> </dependency> <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> - <dependency> <groupId>org.antlr</groupId> <artifactId>antlr-runtime</artifactId> </dependency> @@ -109,10 +105,6 @@ <artifactId>antlr4-runtime</artifactId> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </dependency> - <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> @@ -136,26 +128,6 @@ <scope>test</scope> </dependency> - <!-- These dependencies are necessary in test classpath when using jdisc_http_filters --> - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - <version>1.7.0</version> - </dependency> - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils-core</artifactId> - <version>1.8.0</version> - </dependency> - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - </dependency> - <dependency> - <groupId>commons-digester</groupId> - <artifactId>commons-digester</artifactId> - <version>1.8</version> - </dependency> </dependencies> <build> diff --git a/application/src/test/app-packages/athenz-in-deployment-xml/deployment.xml b/application/src/test/app-packages/athenz-in-deployment-xml/deployment.xml index 261ce9b89a8..1f23a74b840 100644 --- a/application/src/test/app-packages/athenz-in-deployment-xml/deployment.xml +++ b/application/src/test/app-packages/athenz-in-deployment-xml/deployment.xml @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <deployment version="1.0" athenz-domain="foo" athenz-service="bar"> <test /> <staging /> diff --git a/application/src/test/app-packages/model-evaluation/services.xml b/application/src/test/app-packages/model-evaluation/services.xml index 88f9ba14abe..81c42ce51fb 100644 --- a/application/src/test/app-packages/model-evaluation/services.xml +++ b/application/src/test/app-packages/model-evaluation/services.xml @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <container version="1.0"> <model-evaluation/> </container> diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index d4a9c75b713..f2af5537490 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -22,7 +22,6 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.handler.SearchHandler; -import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -34,6 +33,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.ConnectException; import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; import java.util.Map; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -223,7 +223,7 @@ public class ApplicationTest { public void document_type() throws Exception { try ( Application app = Application.fromBuilder(new Application.Builder() - .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .documentType("test", new String(this.getClass().getResourceAsStream("/test.sd").readAllBytes(), StandardCharsets.UTF_8)) .container("default", new Application.Builder.Container() .documentProcessor(MockDocproc.class) .config(new MockApplicationConfig(new MockApplicationConfig.Builder().mystruct(new MockApplicationConfig.Mystruct.Builder().id("foo").value("bar")))))) @@ -264,7 +264,7 @@ public class ApplicationTest { @Test public void client() throws Exception { try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder() - .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .documentType("test", new String(this.getClass().getResourceAsStream("/test.sd").readAllBytes(), StandardCharsets.UTF_8)) .container("default", new Application.Builder.Container() .client("mbus://*/*", MockClient.class) .documentProcessor(MockDispatchDocproc.class) @@ -320,7 +320,7 @@ public class ApplicationTest { "<field name=\"defaultage\" type=\"string\" />\n" + "</query-profile-type>") .rankExpression("re", "commonfirstphase(globalstaticrank)") - .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .documentType("test", new String(this.getClass().getResourceAsStream("/test.sd").readAllBytes(), StandardCharsets.UTF_8)) .container("default", new Application.Builder.Container() .search(true) ))) { diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java index b2d59dbe209..0eb308f124c 100644 --- a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java +++ b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java @@ -10,7 +10,6 @@ import com.yahoo.application.container.jersey.resources.nestedpackage2.NestedTes import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; import com.yahoo.osgi.maven.ProjectBundleClassPaths; import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; -import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; @@ -19,6 +18,7 @@ import org.apache.http.util.EntityUtils; import org.junit.Test; import javax.ws.rs.core.UriBuilder; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -158,7 +158,7 @@ public class JerseyTest { HttpResponse response = httpGetter.get(path(resource)); assertThat("Failed sending response to " + resource, response.getStatusLine().getStatusCode(), is(200)); - String content = IOUtils.toString(response.getEntity().getContent()); + String content = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); assertThat(content, is(TestResourceBase.content(resource))); } } diff --git a/athenz-identity-provider-service/README.md b/athenz-identity-provider-service/README.md index 389af0b8042..1d50340065a 100644 --- a/athenz-identity-provider-service/README.md +++ b/athenz-identity-provider-service/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Athenz Identity Provider Service An [Athenz Copper Argos](https://github.com/yahoo/athenz/blob/master/docs/copper_argos.md) provider implementation for configserver. diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index a117d283146..7dc3496adc9 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -45,6 +45,7 @@ import java.util.logging.Logger; * @author bjorncs */ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { + private static final String CERTIFICATE_ALIAS = "athenz"; private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 11a82ab7443..73fdcf1c471 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -12,7 +12,6 @@ import com.yahoo.container.plugin.osgi.ExportPackages; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ImportPackages.Import; import com.yahoo.container.plugin.util.Strings; -import org.apache.commons.lang3.tuple.Pair; import org.apache.maven.artifact.Artifact; import org.apache.maven.artifact.versioning.ArtifactVersion; import org.apache.maven.artifact.versioning.DefaultArtifactVersion; @@ -25,7 +24,6 @@ import org.apache.maven.project.MavenProject; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; @@ -231,29 +229,31 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { String exportPackage = osgiExportPackages(pluginPackageTally.exportedPackages()).stream().sorted() .collect(Collectors.joining(",")); - for (Pair<String, String> element : Arrays.asList(// - Pair.of("Created-By", "vespa container maven plugin"), // - Pair.of("Bundle-ManifestVersion", "2"), // - Pair.of("Bundle-Name", project.getName()), // - Pair.of("Bundle-SymbolicName", bundleSymbolicName), // - Pair.of("Bundle-Version", asBundleVersion(bundleVersion)), // - Pair.of("Bundle-Vendor", "Yahoo!"), // - Pair.of("Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)), // - Pair.of("Bundle-Activator", bundleActivator), // - Pair.of("X-JDisc-Privileged-Activator", jdiscPrivilegedActivator), // - Pair.of("Main-Class", mainClass), // - Pair.of("X-JDisc-Application", discApplicationClass), // - Pair.of("X-JDisc-Preinstall-Bundle", trimWhitespace(Optional.ofNullable(discPreInstallBundle))), // - Pair.of("WebInfUrl", webInfUrl), // - Pair.of("Import-Package", importPackage), // - Pair.of("Export-Package", exportPackage))) { - if (element.getValue() != null && ! element.getValue().isEmpty()) { - ret.put(element.getKey(), element.getValue()); - } - } + ret.put("Created-By", "vespa container maven plugin"); + ret.put("Bundle-ManifestVersion", "2"); + addIfNotEmpty(ret, "Bundle-Name", project.getName()); + addIfNotEmpty(ret, "Bundle-SymbolicName", bundleSymbolicName); + addIfNotEmpty(ret, "Bundle-Version", asBundleVersion(bundleVersion)); + ret.put("Bundle-Vendor", "Yahoo!"); + addIfNotEmpty(ret, "Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)); + addIfNotEmpty(ret, "Bundle-Activator", bundleActivator); + addIfNotEmpty(ret, "X-JDisc-Privileged-Activator", jdiscPrivilegedActivator); + addIfNotEmpty(ret, "Main-Class", mainClass); + addIfNotEmpty(ret, "X-JDisc-Application", discApplicationClass); + addIfNotEmpty(ret, "X-JDisc-Preinstall-Bundle", trimWhitespace(Optional.ofNullable(discPreInstallBundle))); + addIfNotEmpty(ret, "WebInfUrl", webInfUrl); + addIfNotEmpty(ret, "Import-Package", importPackage); + addIfNotEmpty(ret, "Export-Package", exportPackage); + return ret; } + private static void addIfNotEmpty(Map<String, String> map, String key, String value) { + if (value != null && ! value.isEmpty()) { + map.put(key, value); + } + } + private static String asOsgiImport(String packageName, Optional<String> version) { return version.map(s -> packageName + ";version=" + quote(s)).orElse(packageName); } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JdkPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JdkPackages.java index 0786272bc70..c6171e71319 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JdkPackages.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JdkPackages.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.util; import java.net.URL; diff --git a/clustercontroller-apputil/pom.xml b/clustercontroller-apputil/pom.xml index c8190c1a1ac..f0da97f5b94 100644 --- a/clustercontroller-apputil/pom.xml +++ b/clustercontroller-apputil/pom.xml @@ -35,11 +35,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>2.4</version> - </dependency> </dependencies> <build> <plugins> diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java index 1f3956ef875..f518fe23fe6 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java @@ -9,9 +9,9 @@ import com.yahoo.text.Utf8; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult; -import org.apache.commons.io.IOUtils; import java.io.*; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.logging.Logger; @@ -95,7 +95,7 @@ public class JDiscHttpRequestHandler extends LoggingRequestHandler { private void copyPostData(com.yahoo.container.jdisc.HttpRequest request, HttpRequest legacyRequest) { try { - legacyRequest.setPostContent(IOUtils.toString(request.getData(), "UTF-8")); + legacyRequest.setPostContent(new String(request.getData().readAllBytes(), StandardCharsets.UTF_8)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 9ac7c93134c..a952016e3bf 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -14,8 +14,9 @@ <name>${project.artifactId}</name> <dependencies> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>junit</groupId> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 4a619fa47f2..d943cf27f9c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.jrt.ListenFailedException; import com.yahoo.log.LogLevel; import com.yahoo.vdslib.distribution.ConfiguredNode; @@ -12,20 +13,24 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.*; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SystemStateListener; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; import com.yahoo.vespa.clustercontroller.core.rpc.SlobrokClient; -import com.yahoo.vespa.clustercontroller.core.status.*; +import com.yahoo.vespa.clustercontroller.core.status.ClusterStateRequestHandler; +import com.yahoo.vespa.clustercontroller.core.status.LegacyIndexPageRequestHandler; +import com.yahoo.vespa.clustercontroller.core.status.LegacyNodePageRequestHandler; +import com.yahoo.vespa.clustercontroller.core.status.NodeHealthRequestHandler; +import com.yahoo.vespa.clustercontroller.core.status.RunDataExtractor; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServerInterface; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; -import org.apache.commons.lang.exception.ExceptionUtils; import java.io.FileNotFoundException; - import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; @@ -542,7 +547,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } catch (Exception e) { responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; message = "Internal Server Error"; - hiddenMessage = ExceptionUtils.getStackTrace(e); + hiddenMessage = ExceptionUtils.getStackTraceAsString(e);; log.log(LogLevel.DEBUG, "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java index 84850eb344c..0b6ce9b645e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status.statuspage; -import org.apache.commons.lang.StringEscapeUtils; +import com.google.common.html.HtmlEscapers; -import java.io.*; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; public class StatusPageResponse { @@ -67,7 +70,7 @@ public class StatusPageResponse { } public void writeHtmlHeader(StringBuilder content, String title) { - String escaped_title = StringEscapeUtils.escapeHtml(title); + String escaped_title = HtmlEscapers.htmlEscaper().escape(title); content.append("<html>\n") .append("<head><title>").append(escaped_title).append("</title></head>") .append("<body>\n") @@ -76,7 +79,7 @@ public class StatusPageResponse { public void writeHtmlFooter(StringBuilder content, String hiddenMessage) { if (hiddenMessage != null && !hiddenMessage.isEmpty()) { - content.append("\n<!-- " + StringEscapeUtils.escapeHtml(hiddenMessage) + " -->\n"); + content.append("\n<!-- " + HtmlEscapers.htmlEscaper().escape(hiddenMessage) + " -->\n"); } content.append("</body>\n") .append("</html>\n"); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 52cd0a27933..6e4c0bc7eaa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -1,20 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status.statuspage; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.log.LogLevel; -import org.apache.commons.lang.exception.ExceptionUtils; -import java.io.*; -import java.net.ServerSocket; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; import java.net.InetSocketAddress; +import java.net.ServerSocket; import java.net.Socket; import java.net.SocketTimeoutException; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import java.util.logging.Logger; -import java.util.regex.Pattern; import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Shows status pages with debug information through a very simple HTTP interface. @@ -177,7 +185,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { response.setResponseCode(StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR); StringBuilder content = new StringBuilder(); response.writeHtmlHeader(content, "Internal Server Error"); - response.writeHtmlFooter(content, ExceptionUtils.getStackTrace(e)); + response.writeHtmlFooter(content, ExceptionUtils.getStackTraceAsString(e)); response.writeContent(content.toString()); } if (response == null) { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index e28c5eac0bb..1cdb54a743c 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -68,6 +68,7 @@ public class Xml { static DocumentBuilder getPreprocessDocumentBuilder() throws ParserConfigurationException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); // XXE prevention factory.setNamespaceAware(true); factory.setXIncludeAware(false); factory.setValidating(false); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java index d9253d6105b..872aa7fb56f 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.defaults.Defaults; -import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.IOException; @@ -54,7 +53,7 @@ public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { private void addConfigDefinition(ConfigDefinitionKey key, File defFile) throws IOException { String payload = IOUtils.readFile(defFile); - configDefinitions.put(key, new ConfigDefinition(key.getName(), StringUtils.split(payload, "\n"))); + configDefinitions.put(key, new ConfigDefinition(key.getName(), payload.split("[\\r\\n]+"))); } @Override diff --git a/config-class-plugin/pom.xml b/config-class-plugin/pom.xml index e0ea70bd4f0..4809db247d4 100644 --- a/config-class-plugin/pom.xml +++ b/config-class-plugin/pom.xml @@ -30,10 +30,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> <groupId>org.apache.maven.plugin-tools</groupId> <artifactId>maven-plugin-annotations</artifactId> <version>3.5</version> diff --git a/config-class-plugin/src/main/java/com/yahoo/vespa/ConfigGenMojo.java b/config-class-plugin/src/main/java/com/yahoo/vespa/ConfigGenMojo.java index 64941fdc1a2..9930d53ca6c 100644 --- a/config-class-plugin/src/main/java/com/yahoo/vespa/ConfigGenMojo.java +++ b/config-class-plugin/src/main/java/com/yahoo/vespa/ConfigGenMojo.java @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa; -import com.yahoo.config.codegen.DefParser; import com.yahoo.config.codegen.MakeConfig; import com.yahoo.config.codegen.MakeConfigProperties; import com.yahoo.config.codegen.PropertyException; -import org.apache.commons.io.FileUtils; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -16,9 +14,12 @@ import org.apache.maven.project.MavenProject; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX; @@ -160,8 +161,11 @@ public class ConfigGenMojo extends AbstractMojo { private boolean isSomeGeneratedFileStale(File outputDirectory, List<String> defFileNames) { long oldestGeneratedModifiedTime = Long.MAX_VALUE; - final Collection<File> files = FileUtils.listFiles(outputDirectory, null, true); - if (files != null) { + try { + List<File> files = Files.walk(outputDirectory.toPath()) + .filter(Files::isRegularFile) + .map(Path::toFile) + .collect(Collectors.toList()); for (File f : files) { getLog().debug("Checking generated file " + f); final long l = f.lastModified(); @@ -169,6 +173,8 @@ public class ConfigGenMojo extends AbstractMojo { oldestGeneratedModifiedTime = l; } } + } catch (IOException e) { + throw new UncheckedIOException(e); } long lastModifiedSource = 0; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index 99cb07f3104..9e96dcfbf64 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.yahoo.config.provision.RegionName; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java index 55269088196..d2c5bbca5b7 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; import com.google.common.collect.ImmutableList; diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java index 67524ed45ca..d13894d6ed7 100644 --- a/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api.container; import org.junit.Test; diff --git a/config-model/.gitignore b/config-model/.gitignore index 6adbe26187e..b0f358e8113 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -2,3 +2,5 @@ /tmp /temp /target +/src/test/integration/*/copy/ +/src/test/integration/*/models.generated/ diff --git a/config-model/pom.xml b/config-model/pom.xml index 180a2e40921..25b733985f5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -46,10 +46,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -297,14 +293,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_http_service</artifactId> <version>${project.version}</version> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java new file mode 100644 index 00000000000..d8f02b2a9c3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +/** + * @author bjorncs + */ +class Utils { + + private Utils() {} + + // Separate class since javacc does not accept Java code using lambdas + static int count(String str, char ch) { + return (int) str.chars().filter(c -> c == ch).count(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 0d99c698aca..8db4ec432c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -23,6 +23,7 @@ public class ImplicitSummaryFields extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { + if (docsum.getInherited() != null) continue; // Implicit fields are added to inheriting summaries through their parent addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java index 2fab941b10e..f629b046c7d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 8dc5356026b..3c6aa881af4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.Field; - import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -16,6 +14,7 @@ import java.util.List; public class DocumentSummary extends FieldView { private boolean fromDisk = false; + private DocumentSummary inherited; /** * Creates a DocumentSummary with the given name. @@ -44,13 +43,21 @@ public class DocumentSummary extends FieldView { } public SummaryField getSummaryField(String name) { + var parent = getInherited(); + if (parent != null) { + return parent.getSummaryField(name); + } return (SummaryField) get(name); } public Collection<SummaryField> getSummaryFields() { - ArrayList<SummaryField> fields = new ArrayList<>(getFields().size()); - for(Field f : getFields()) { - fields.add((SummaryField) f); + var fields = new ArrayList<SummaryField>(getFields().size()); + var parent = getInherited(); + if (parent != null) { + fields.addAll(parent.getSummaryFields()); + } + for (var field : getFields()) { + fields.add((SummaryField) field); } return fields; } @@ -80,8 +87,19 @@ public class DocumentSummary extends FieldView { } } + /** Sets the parent of this. Both summaries must be present in the same search definition */ + public void setInherited(DocumentSummary inherited) { + this.inherited = inherited; + } + + /** Returns the parent of this, or null if none is inherited */ + public DocumentSummary getInherited() { + return inherited; + } + public String toString() { - return "document summary '" + getName() + "'"; + return "document summary '" + getName() + "'" + + (inherited == null ? "" : " inheriting from '" + inherited.getName() + "'"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index bedf8057b41..08956b272f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -306,9 +306,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config * types in the model. * - * @param clazz The type of config - * @param configId The config id - * @return A config instance of the given type + * @param clazz the type of config + * @param configId the config id + * @return a config instance of the given type */ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) { try { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index 4ff577ab670..3f9edae10c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java index bc9cd62b444..0db48cb5e27 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java @@ -1,11 +1,10 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.http.MetricsHandler; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; +import com.yahoo.config.provision.ClusterMembership; import java.util.List; import java.util.stream.Collectors; @@ -22,11 +21,18 @@ public class MetricsNodesConfigGenerator { } private static MetricsNodesConfig.Node.Builder toNodeBuilder(MetricsProxyContainer container) { - return new MetricsNodesConfig.Node.Builder() - .configId(container.getConfigId()) + var builder = new MetricsNodesConfig.Node.Builder() + .nodeId(container.getHost().getConfigId()) .hostname(container.getHostName()) .metricsPort(MetricsProxyContainer.BASEPORT) .metricsPath(MetricsHandler.VALUES_PATH); + + if (container.isHostedVespa) + container.getHostResource().spec().membership() + .map(ClusterMembership::stringValue) + .ifPresent(builder::nodeId); + + return builder; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 719d3256889..08f782a0c54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; @@ -40,7 +38,7 @@ public class MetricsProxyContainer extends Container implements static final String CLUSTER_ID = "clusterid"; } - private final boolean isHostedVespa; + final boolean isHostedVespa; public MetricsProxyContainer(AbstractConfigProducer parent, String hostname, int index, boolean isHostedVespa) { super(parent, hostname, index); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 3216506bc24..b55d71bae25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java index c3652fe8ad1..990cc4f0edd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java index e8e6a987ebe..68b1fc3c983 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java index 434d6696261..19a62f47e39 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f82e9d7bec7..57d938f8a71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -245,6 +245,9 @@ public class VespaMetricSet { metrics.add(new Metric("hits_per_query.sum")); metrics.add(new Metric("hits_per_query.count")); metrics.add(new Metric("hits_per_query.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("query_hit_offset.max")); + metrics.add(new Metric("query_hit_offset.sum")); + metrics.add(new Metric("query_hit_offset.count")); metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); metrics.add(new Metric("dispatch_internal.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 30f08d13772..f9762ce58fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -10,7 +10,6 @@ import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; -import org.apache.commons.lang.StringUtils; import java.util.ArrayList; import java.util.Collection; @@ -67,7 +66,7 @@ public class ComplexAttributeFieldsValidator extends Validator { } private static String toString(ImmutableSDField field) { - return field.getName() + " (" + StringUtils.join(getStructFieldAttributes(field.getStructFields()), ", ") + ")"; + return field.getName() + " (" + String.join(", ", getStructFieldAttributes(field.getStructFields())) + ")"; } private static boolean hasStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java index 1018099cf05..2f972b8ecb3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.api.TlsSecrets; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index 3a97bf4b876..d94cd57357d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -4,14 +4,13 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.ChangesRequiringRestart; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.RestartConfigs; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.utils.internal.ReflectionUtil; -import org.apache.commons.lang3.ClassUtils; import java.time.Instant; import java.util.Arrays; @@ -95,7 +94,7 @@ public class ConfigValueChangeValidator implements ChangeValidator { * NOTE: Only the super classes that are subclass of Service are inspected. */ private static Stream<Class<? extends ConfigInstance>> getConfigInstancesFromServiceAnnotations(Class<? extends Service> serviceClass) { - List<Class<?>> classHierarchy = ClassUtils.getAllSuperclasses(serviceClass); + List<Class<?>> classHierarchy = ReflectionUtil.getAllSuperclasses(serviceClass); classHierarchy.add(serviceClass); return classHierarchy.stream() .filter(Service.class::isAssignableFrom) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 54850dedbba..f221b1974d7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -8,8 +8,11 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.text.XML; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; +import org.w3c.dom.Element; +import org.w3c.dom.Node; import java.util.Map; import java.util.Optional; @@ -41,6 +44,11 @@ public class NodesSpecification { private final boolean exclusive; + /** + * Whether this requires running container and content processes co-located on the same node. + */ + private final boolean combined; + /** The resources each node should have, or empty to use the default */ private final Optional<NodeResources> resources; @@ -48,7 +56,7 @@ public class NodesSpecification { private final Optional<String> dockerImage; private NodesSpecification(boolean dedicated, int count, int groups, Version version, - boolean required, boolean canFail, boolean exclusive, + boolean required, boolean canFail, boolean exclusive, boolean combined, Optional<NodeResources> resources, Optional<String> dockerImage) { this.dedicated = dedicated; this.count = count; @@ -59,9 +67,10 @@ public class NodesSpecification { this.exclusive = exclusive; this.resources = resources; this.dockerImage = dockerImage; + this.combined = combined; } - private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { + private NodesSpecification(boolean dedicated, boolean canFail, boolean combined, Version version, ModelElement nodesElement) { this(dedicated, nodesElement.integerAttribute("count", 1), nodesElement.integerAttribute("groups", 1), @@ -69,16 +78,23 @@ public class NodesSpecification { nodesElement.booleanAttribute("required", false), canFail, nodesElement.booleanAttribute("exclusive", false), + combined, getResources(nodesElement), Optional.ofNullable(nodesElement.stringAttribute("docker-image"))); } + private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { + var resolvedElement = resolveElement(nodesElement); + boolean combined = resolvedElement != nodesElement || isReferencedByOtherElement(nodesElement); + return new NodesSpecification(dedicated, canFail, combined, version, resolvedElement); + } + /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ public static NodesSpecification from(ModelElement nodesElement, ConfigModelContext context) { - return new NodesSpecification(true, - ! context.getDeployState().getProperties().isBootstrap(), - context.getDeployState().getWantedNodeVespaVersion(), - nodesElement); + return create(true, + ! context.getDeployState().getProperties().isBootstrap(), + context.getDeployState().getWantedNodeVespaVersion(), + nodesElement); } /** @@ -103,10 +119,9 @@ public class NodesSpecification { if (parentElement == null) return Optional.empty(); ModelElement nodesElement = parentElement.child("nodes"); if (nodesElement == null) return Optional.empty(); - return Optional.of(new NodesSpecification(nodesElement.booleanAttribute("dedicated", false), - ! context.getDeployState().getProperties().isBootstrap(), - context.getDeployState().getWantedNodeVespaVersion(), - nodesElement)); + return Optional.of(create(nodesElement.booleanAttribute("dedicated", false), + ! context.getDeployState().getProperties().isBootstrap(), + context.getDeployState().getWantedNodeVespaVersion(), nodesElement)); } /** Returns a requirement from <code>count</code> nondedicated nodes in one group */ @@ -118,6 +133,7 @@ public class NodesSpecification { false, ! context.getDeployState().getProperties().isBootstrap(), false, + false, Optional.empty(), Optional.empty()); } @@ -131,6 +147,7 @@ public class NodesSpecification { false, ! context.getDeployState().getProperties().isBootstrap(), false, + false, Optional.empty(), Optional.empty()); } @@ -158,6 +175,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger) { + if (combined) { + clusterType = ClusterSpec.Type.combined; + } ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive); return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); } @@ -239,6 +259,66 @@ public class NodesSpecification { } } + /** + * Resolve any reference in nodesElement and return the referred element. + * + * If nodesElement does not refer to a different element, this method behaves as the identity function. + */ + private static ModelElement resolveElement(ModelElement nodesElement) { + var element = nodesElement.getXml(); + var referenceId = element.getAttribute("of"); + if (referenceId.isEmpty()) return nodesElement; + + var services = findParentByTag("services", element).orElseThrow(() -> clusterReferenceNotFoundException(referenceId)); + var referencedService = findChildById(services, referenceId).orElseThrow(() -> clusterReferenceNotFoundException(referenceId)); + if ( ! referencedService.getTagName().equals("content")) + throw new IllegalArgumentException("service '" + referenceId + "' is not a content service"); + var referencedNodesElement = XML.getChild(referencedService, "nodes"); + if (referencedNodesElement == null) + throw new IllegalArgumentException("expected reference to service '" + referenceId + "' to supply nodes, " + + "but that service has no <nodes> element"); + + return new ModelElement(referencedNodesElement); + } + + /** Returns whether the given nodesElement is referenced by any other nodes element */ + private static boolean isReferencedByOtherElement(ModelElement nodesElement) { + var element = nodesElement.getXml(); + var services = findParentByTag("services", element); + if (services.isEmpty()) return false; + + var content = findParentByTag("content", element); + if (content.isEmpty()) return false; + var clusterId = content.get().getAttribute("id"); + for (var rootChild : XML.getChildren(services.get())) { + if (!"container".equals(rootChild.getTagName())) continue; // Only container can reference content + var nodes = XML.getChild(rootChild, "nodes"); + if (nodes == null) continue; + if (!clusterId.equals(nodes.getAttribute("of"))) continue; + return true; + } + return false; + } + + private static Optional<Element> findChildById(Element parent, String id) { + for (Element child : XML.getChildren(parent)) + if (id.equals(child.getAttribute("id"))) return Optional.of(child); + return Optional.empty(); + } + + private static Optional<Element> findParentByTag(String tag, Element element) { + Node parent = element.getParentNode(); + if (parent == null) return Optional.empty(); + if ( ! (parent instanceof Element)) return Optional.empty(); + Element parentElement = (Element) parent; + if (parentElement.getTagName().equals(tag)) return Optional.of(parentElement); + return findParentByTag(tag, parentElement); + } + + private static IllegalArgumentException clusterReferenceNotFoundException(String referenceId) { + return new IllegalArgumentException("referenced service '" + referenceId + "' is not defined"); + } + @Override public String toString() { return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 188a9b4765c..16ecf5f761c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -651,22 +651,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private List<ApplicationContainer> createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { - // Resolve references to content clusters at the XML level because content clusters must be built after container clusters + NodesSpecification nodeSpecification; + try { + nodeSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(cluster + " contains an invalid reference", e); + } String referenceId = nodesElement.getAttribute("of"); - Element services = servicesRootOf(nodesElement).orElseThrow(() -> clusterReferenceNotFoundException(cluster, referenceId)); - Element referencedService = findChildById(services, referenceId).orElseThrow(() -> clusterReferenceNotFoundException(cluster, referenceId)); - if ( ! referencedService.getTagName().equals("content")) - throw new IllegalArgumentException(cluster + " references service '" + referenceId + "', " + - "but that is not a content service"); - Element referencedNodesElement = XML.getChild(referencedService, "nodes"); - if (referencedNodesElement == null) - throw new IllegalArgumentException(cluster + " references service '" + referenceId + "' to supply nodes, " + - "but that service has no <nodes> element"); - cluster.setHostClusterId(referenceId); Map<HostResource, ClusterMembership> hosts = - StorageGroup.provisionHosts(NodesSpecification.from(new ModelElement(referencedNodesElement), context), + StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().getHostSystem(), context.getDeployLogger()); @@ -732,17 +727,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return nodes; } - private IllegalArgumentException clusterReferenceNotFoundException(ApplicationContainerCluster cluster, String referenceId) { - return new IllegalArgumentException(cluster + " references service '" + referenceId + - "' but this service is not defined"); - } - - private Optional<Element> findChildById(Element parent, String id) { - for (Element child : XML.getChildren(parent)) - if (id.equals(child.getAttribute("id"))) return Optional.of(child); - return Optional.empty(); - } - private static boolean useCpuSocketAffinity(Element nodesElement) { if (nodesElement.hasAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)) return Boolean.parseBoolean(nodesElement.getAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 79ea25f2fd6..cae7ede463d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -310,7 +310,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; - builder.feeding.concurrency(0.35); // As if specified 0.7 in services.xml + builder.feeding.concurrency(0.40); // As if specified 0.8 in services.xml boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java index ef021872efb..ab46bb287b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.model.ConfigProducer; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -80,6 +82,14 @@ public final class ReflectionUtil { } } + public static List<Class<?>> getAllSuperclasses(Class<?> cls) { + var result = new ArrayList<Class<?>>(); + for(Class<?> superClass = cls.getSuperclass(); superClass != null; superClass = superClass.getSuperclass()) { + result.add(superClass); + } + return result; + } + private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) { try { String defName = ConfigInstance.getDefName(configInstClass); diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index e560d78a116..52665ff56a9 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -63,7 +63,6 @@ import java.util.List; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.logging.Level; -import org.apache.commons.lang.StringUtils; /** * A search definition parser @@ -1706,6 +1705,7 @@ Object documentSummary(Search search) : { ( <DOCUMENTSUMMARY> name = identifierWithDash() { search.addSummary(summary = new DocumentSummary(name)); } + [inheritsDocumentSummary(summary, search)] lbrace() ( <FROMDISK> { summary.setFromDisk(true); } | @@ -1718,6 +1718,23 @@ Object documentSummary(Search search) : } /** + * This rule consumes an inherits statement of a document summary. + * + * @param documentSummary The document summary to modify. + * @param search The search object documentSummary is being added to. + */ +void inheritsDocumentSummary(DocumentSummary documentSummary, Search search) : +{ + String name; +} +{ + <INHERITS> name = identifierWithDash() + { + documentSummary.setInherited(search.getSummaries().get(name)); + } +} + +/** * Consumes a single document-summary item. * * @param summary The document summary to modify. @@ -2429,7 +2446,8 @@ void importField(Search search) : <IMPORT> <FIELD> fieldRefSpec = identifier() <AS> aliasFieldName = identifier() lbrace() <RBRACE> { - if (StringUtils.countMatches(fieldRefSpec, ".") != 1) { + long nDots = Utils.count(fieldRefSpec, '.'); + if (nDots != 1) { throw new IllegalArgumentException("Illegal field reference spec '" + fieldRefSpec + "': Does not include a single '.'"); } int indexOfDot = fieldRefSpec.indexOf('.'); diff --git a/config-model/src/test/cfg/application/app_complicated_deployment_spec/deployment.xml b/config-model/src/test/cfg/application/app_complicated_deployment_spec/deployment.xml index 36c1d449021..9416d6df88f 100644 --- a/config-model/src/test/cfg/application/app_complicated_deployment_spec/deployment.xml +++ b/config-model/src/test/cfg/application/app_complicated_deployment_spec/deployment.xml @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <deployment version='1.0' athenz-domain='domain'> <staging /> <parallel> diff --git a/config-model/src/test/derived/fieldset/test.sd b/config-model/src/test/derived/fieldset/test.sd index c5ed169cc94..336c370994e 100644 --- a/config-model/src/test/derived/fieldset/test.sd +++ b/config-model/src/test/derived/fieldset/test.sd @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search test { document test { @@ -22,4 +23,4 @@ search test { fields: word1,word2 } -}
\ No newline at end of file +} diff --git a/config-model/src/test/derived/neuralnet/neuralnet.sd b/config-model/src/test/derived/neuralnet/neuralnet.sd index f916b35cb75..5011ac5a75a 100644 --- a/config-model/src/test/derived/neuralnet/neuralnet.sd +++ b/config-model/src/test/derived/neuralnet/neuralnet.sd @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search neuralnet { document neuralnet { @@ -235,4 +236,4 @@ search neuralnet { type: tensor(out[1]) } -}
\ No newline at end of file +} diff --git a/config-model/src/test/derived/neuralnet/query-profiles/default.xml b/config-model/src/test/derived/neuralnet/query-profiles/default.xml index eef1aaa7f53..185dbba84db 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles/default.xml +++ b/config-model/src/test/derived/neuralnet/query-profiles/default.xml @@ -1,2 +1,3 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="default" type="DefaultQueryProfileType"> </query-profile> diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml index e1659479135..e74152638fb 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml +++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile-type id="DefaultQueryProfileType"> <field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" /> <field name="ranking.features.query(b_0)" type="tensor(hidden[9])" /> diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 2e0a207d249..4634e120a3a 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -59,7 +59,7 @@ attribute[].arity 8 attribute[].lowerbound -9223372036854775808 attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 -attribute[].tensortype "tensor(x[10],y[20])" +attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false attribute[].name "f5" attribute[].datatype TENSOR diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 72fae572b76..68bd394c9d6 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -35,7 +35,7 @@ documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x{})" documenttype[].datatype[].sstruct.field[].name "f4" documenttype[].datatype[].sstruct.field[].id 1224191509 documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[20])" +documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[10])" documenttype[].datatype[].sstruct.field[].name "f5" documenttype[].datatype[].sstruct.field[].id 329055840 documenttype[].datatype[].sstruct.field[].datatype 21 diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index 29dc39b01ce..7970e05b790 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -4,7 +4,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "unranked" @@ -21,7 +21,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile1" @@ -34,27 +34,27 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile2" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f2" rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile3" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -64,7 +64,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile4" @@ -77,7 +77,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile5" @@ -90,14 +90,14 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile6" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -107,7 +107,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile7" @@ -128,7 +128,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile8" @@ -143,7 +143,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile9" @@ -158,7 +158,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index aa33684a979..a7248fe3200 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -11,7 +11,7 @@ search tensor { field f3 type tensor<double>(x{}) { indexing: attribute | summary } - field f4 type tensor(x[10],y[20]) { + field f4 type tensor(x[10],y[10]) { indexing: attribute | summary } field f5 type tensor<float>(x[10]) { @@ -33,7 +33,7 @@ search tensor { rank-profile profile2 { first-phase { - expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x)) + expression: sum(matmul(attribute(f4), diag(x[10],y[10],z[3]), x)) } } diff --git a/config-model/src/test/examples/invalid-name.sd b/config-model/src/test/examples/invalid-name.sd index f26fcc723f4..b5f639312e3 100644 --- a/config-model/src/test/examples/invalid-name.sd +++ b/config-model/src/test/examples/invalid-name.sd @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Dashes in names are not allowed search invalid-name { @@ -9,4 +10,4 @@ search invalid-name { } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 93c3c9ea2ea..6011b138a61 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; @@ -31,6 +31,7 @@ import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; @@ -249,6 +250,10 @@ public class ModelProvisioningTest { assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals(2, model.getHostSystem().getHosts().stream() + .filter(hostResource -> hostResource.spec().membership().get().cluster().type() == ClusterSpec.Type.combined) + .count()); + } @Test @@ -330,7 +335,7 @@ public class ModelProvisioningTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("container cluster 'container1' references service 'container2' but this service is not defined", e.getMessage()); + assertEquals("container cluster 'container1' contains an invalid reference: referenced service 'container2' is not defined", Exceptions.toMessageString(e)); } } @@ -353,7 +358,7 @@ public class ModelProvisioningTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("container cluster 'container1' references service 'container2', but that is not a content service", e.getMessage()); + assertEquals("container cluster 'container1' contains an invalid reference: service 'container2' is not a content service", Exceptions.toMessageString(e)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 6fbd80adaa9..91599e6f607 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -1,13 +1,20 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import com.yahoo.vespa.objects.FieldBase; import org.junit.Test; +import java.util.Collection; +import java.util.List; import java.util.logging.Level; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests summary validation @@ -129,4 +136,119 @@ public class SummaryTestCase { assertTrue(logger.entries.isEmpty()); } + @Test + public void testInheritance() throws Exception { + String sd = + "search music {\n" + + "\n" + + " document music {\n" + + " field title type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " \n" + + " field artist type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " \n" + + " field album type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " }\n" + + " \n" + + " document-summary title {\n" + + " summary title type string {\n" + + " source: title\n" + + " }\n" + + " }\n" + + " document-summary title_artist inherits title {\n" + + " summary artist type string {\n" + + " source: artist\n" + + " }\n" + + " }\n" + + " document-summary everything inherits title_artist {\n" + + " summary album type string {\n" + + " source: album\n" + + " }\n" + + " }\n" + + "\n" + + "}"; + var logger = new DeployLoggerStub(); + var search = SearchBuilder.createFromString(sd, logger).getSearch(); + assertEquals(List.of(), logger.entries); + + var titleField = "title"; + var artistField = "artist"; + var albumField = "album"; + var titleSummary = search.getSummary(titleField); + var titleArtistSummary = search.getSummary(titleField + "_" + artistField); + var everythingSummary = search.getSummary("everything"); + + var implicitFields = List.of("rankfeatures", "summaryfeatures"); + var tests = List.of( + new TestValue(titleSummary, null, List.of(List.of(titleField), implicitFields)), + new TestValue(titleArtistSummary, titleSummary, List.of(List.of(titleField), implicitFields, List.of(artistField))), + new TestValue(everythingSummary, titleArtistSummary, List.of(List.of(titleField), implicitFields, List.of(artistField, albumField))) + ); + tests.forEach(testValue -> { + var actualFields = testValue.summary.getSummaryFields().stream() + .map(FieldBase::getName) + .collect(Collectors.toList()); + assertEquals(testValue.summary.getName() + (testValue.parent == null ? " does not inherit anything" : " inherits " + testValue.parent.getName()), + testValue.parent, + testValue.summary.getInherited()); + assertEquals("Summary " + testValue.summary.getName() + " has expected fields", testValue.fields, actualFields); + }); + } + + @Test + public void testRedeclaringInheritedFieldFails() throws Exception { + String sd = + "search music {\n" + + "\n" + + " document music {\n" + + " field title type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " field title_short type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " }\n" + + " \n" + + " document-summary title {\n" + + " summary title type string {\n" + + " source: title\n" + + " }\n" + + " }\n" + + " document-summary title2 inherits title {\n" + + " summary title type string {\n" + + " source: title_short\n" + + " }\n" + + " }\n" + + " \n" + + "}"; + var logger = new DeployLoggerStub(); + try { + SearchBuilder.createFromString(sd, logger); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'music', summary class 'title2', summary field 'title': Can not use " + + "source 'title_short' for this summary field, an equally named field in summary class 'title' " + + "uses a different source: 'title'.", e.getMessage()); + } + } + + private static class TestValue { + + private final DocumentSummary summary; + private final DocumentSummary parent; + private final List<String> fields; + + public TestValue(DocumentSummary summary, DocumentSummary parent, List<List<String>> fields) { + this.summary = summary; + this.parent = parent; + this.fields = fields.stream().flatMap(Collection::stream).collect(Collectors.toList());; + } + + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java index 6b3a3cf6a24..36204d6913a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.parser.ParseException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 1b03825eef1..a1231a1418b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -70,7 +70,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -91,7 +91,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -114,7 +114,7 @@ public class RankingExpressionWithTensorTestCase { f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); - f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -135,7 +135,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile"); - f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 25a06a98633..6164b311da0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. */ @@ -306,8 +307,8 @@ public class MetricsProxyContainerClusterTest { } private void assertNodeConfig(MetricsNodesConfig.Node node) { - assertFalse(node.configId().isEmpty()); - assertFalse(node.hostname().isEmpty()); + assertTrue(node.nodeId().startsWith("container/foo/0/")); + assertTrue(node.hostname().startsWith("node-1-3-9-")); assertEquals(MetricsProxyContainer.BASEPORT, node.metricsPort()); assertEquals(MetricsHandler.VALUES_PATH, node.metricsPath()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 80b4680ea82..854b708797e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 8a179f514fb..f3140aafdaf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index a651bbb7772..d2a840d1fbc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -5,13 +5,14 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.search.grouping.GroupingValidator; -import org.apache.commons.io.IOUtils; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -50,7 +51,7 @@ public class BundleInstantiationSpecificationBuilderTest { xml += " bundle=\"" + explicitBundle + "\""; } xml += " />"; - InputStream xmlStream = IOUtils.toInputStream(xml); + InputStream xmlStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index b18ec88b0d4..484709a0c18 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java index e5d6e149c70..fa38b415cd6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.content.ContentSearch; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -52,7 +52,7 @@ public class DomContentSearchBuilderTest { return DomContentSearchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java index d9db6234f1c..107e8968c1a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.SearchCoverage; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -71,7 +71,7 @@ public class DomSearchCoverageBuilderTest { return DomSearchCoverageBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java index 46b5b630f9e..0f909b7a8eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.TuningDispatch; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -93,7 +93,7 @@ public class DomTuningDispatchBuilderTest { return DomTuningDispatchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 4ccdd80805b..177e741937d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -116,13 +116,13 @@ public class DocumentDatabaseTestCase { @Test public void requireThatConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency("index", "", 0.35, 0.35); - verifyConcurrency("streaming", "", 0.7, 0.0); - verifyConcurrency("store-only", "", 0.7, 0.0); + verifyConcurrency("index", "", 0.40, 0.40); + verifyConcurrency("streaming", "", 0.8, 0.0); + verifyConcurrency("store-only", "", 0.8, 0.0); } @Test public void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.7, Arrays.asList(0.35, 0.0)); + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.8, Arrays.asList(0.40, 0.0)); } @Test public void requireThatMixedModeConcurrencyIsReflected() { diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 3cff2bb4da1..2a35dafedbd 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -249,12 +249,15 @@ "methods": [ "public static com.yahoo.config.provision.ClusterSpec$Type[] values()", "public static com.yahoo.config.provision.ClusterSpec$Type valueOf(java.lang.String)", + "public boolean isContent()", + "public boolean isContainer()", "public static com.yahoo.config.provision.ClusterSpec$Type from(java.lang.String)" ], "fields": [ "public static final enum com.yahoo.config.provision.ClusterSpec$Type admin", "public static final enum com.yahoo.config.provision.ClusterSpec$Type container", - "public static final enum com.yahoo.config.provision.ClusterSpec$Type content" + "public static final enum com.yahoo.config.provision.ClusterSpec$Type content", + "public static final enum com.yahoo.config.provision.ClusterSpec$Type combined" ] }, "com.yahoo.config.provision.ClusterSpec": { @@ -277,7 +280,7 @@ "public java.lang.String toString()", "public int hashCode()", "public boolean equals(java.lang.Object)", - "public boolean equalsIgnoringGroupAndVespaVersion(java.lang.Object)" + "public boolean satisfies(com.yahoo.config.provision.ClusterSpec)" ], "fields": [] }, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 496874e978b..f1a8b579866 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -85,14 +85,15 @@ public final class ClusterSpec { return true; } - /** Returns whether this is equal, disregarding the group value and wanted Vespa version */ - public boolean equalsIgnoringGroupAndVespaVersion(Object o) { - if (o == this) return true; - if ( ! (o instanceof ClusterSpec)) return false; - ClusterSpec other = (ClusterSpec)o; - if ( ! other.type.equals(this.type)) return false; - if ( ! other.id.equals(this.id)) return false; - return true; + /** + * Returns whether this satisfies other for allocation purposes. Only considers cluster ID and type, other fields + * are ignored. + */ + public boolean satisfies(ClusterSpec other) { + if ( ! other.id.equals(this.id)) return false; // ID mismatch + // TODO(mpolden): Remove this after January 2019, once all nodes in combined clusters have type combined. + if (other.type.isContent() || this.type.isContent()) return other.type.isContent() == this.type.isContent(); + return other.type.equals(this.type); } /** A cluster type */ @@ -101,13 +102,25 @@ public final class ClusterSpec { // These enum values are stored in ZooKeeper - do not change admin, container, - content; + content, + combined; + + /** Returns whether this runs a content cluster */ + public boolean isContent() { + return this == content || this == combined; + } + + /** Returns whether this runs a container cluster */ + public boolean isContainer() { + return this == container || this == combined; + } public static Type from(String typeName) { switch (typeName) { case "admin" : return admin; case "container" : return container; case "content" : return content; + case "combined" : return combined; default: throw new IllegalArgumentException("Illegal cluster type '" + typeName + "'"); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java index d91c76a4f7d..db90b0ebff9 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java @@ -2,8 +2,14 @@ package com.yahoo.config.provision; import com.google.common.testing.EqualsTester; +import com.yahoo.component.Version; import org.junit.Test; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + /** * @author Ulf Lilleengen */ @@ -27,4 +33,26 @@ public class ClusterSpecTest { .testEquals(); } + @Test + public void testSatisfies() { + var tests = Map.of( + List.of(spec(ClusterSpec.Type.content, "id1"), spec(ClusterSpec.Type.content, "id2")), false, + List.of(spec(ClusterSpec.Type.admin, "id1"), spec(ClusterSpec.Type.container, "id1")), false, + List.of(spec(ClusterSpec.Type.admin, "id1"), spec(ClusterSpec.Type.content, "id1")), false, + List.of(spec(ClusterSpec.Type.combined, "id1"), spec(ClusterSpec.Type.container, "id1")), false, + List.of(spec(ClusterSpec.Type.combined, "id1"), spec(ClusterSpec.Type.content, "id1")), true, + List.of(spec(ClusterSpec.Type.content, "id1"), spec(ClusterSpec.Type.content, "id1")), true + ); + tests.forEach((specs, satisfies) -> { + var s1 = specs.get(0); + var s2 = specs.get(1); + assertEquals(s1 + (satisfies ? " satisfies " : " does not satisfy ") + s2, satisfies, s1.satisfies(s2)); + assertEquals(s2 + (satisfies ? " satisfies " : " does not satisfy ") + s1, satisfies, s2.satisfies(s1)); + }); + } + + private static ClusterSpec spec(ClusterSpec.Type type, String id) { + return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(1), Version.emptyVersion, false); + } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java index 0888004ef02..4da27e8c346 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -25,6 +25,7 @@ import java.util.Stack; * @author Ulf Lilleengen */ public class ConfigVerification { + private final static int port = 19071; private final static String prefix = "http://"; diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java index e4cedfafeea..9473a05332f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java @@ -7,5 +7,7 @@ package com.yahoo.vespa.config.buildergen; * @author Ulf Lilleengen */ public interface ConfigCompiler { + CompiledBuilder compile(ConfigDefinitionClass defClass); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java index 2fc3cd7aa19..2af19d1c6f2 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java @@ -16,9 +16,9 @@ import java.util.Locale; * {@link CompiledBuilder}. * * @author Ulf Lilleengen - * @since 5.2 */ public class LazyConfigCompiler implements ConfigCompiler { + private final File outputDirectory; private final ClassLoader classLoader; private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 49767ab47aa..33b4aa8111d 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <stdlib.h> #include <stdio.h> diff --git a/configserver/pom.xml b/configserver/pom.xml index b39686c31d9..1e242988327 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -206,11 +206,6 @@ <artifactId>commons-compress</artifactId> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>2.4</version> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> <scope>test</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 465344a6d76..43b78212827 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -39,6 +39,7 @@ import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; +import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.v2.MetricsResponse; @@ -325,8 +326,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Deployment deployment = deployFromPreparedSession(localSession, tenant, timeoutBudget.timeLeft()); deployment.setIgnoreSessionStaleFailure(ignoreSessionStaleFailure); deployment.activate(); - ApplicationId applicationId = localSession.getApplicationId(); - return applicationId; + return localSession.getApplicationId(); } private Deployment deployFromPreparedSession(LocalSession session, Tenant tenant, Duration timeout) { @@ -341,7 +341,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return true if the application was found and deleted, false if it was not present * @throws RuntimeException if the delete transaction fails. This method is exception safe. */ - public boolean delete(ApplicationId applicationId) { + boolean delete(ApplicationId applicationId) { + return delete(applicationId, Duration.ofSeconds(60)); + } + + /** + * Deletes an application + * + * @return true if the application was found and deleted, false if it was not present + * @throws RuntimeException if the delete transaction fails. This method is exception safe. + */ + public boolean delete(ApplicationId applicationId, Duration waitTime) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); if (tenant == null) return false; @@ -349,23 +359,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye try (Lock lock = tenantApplications.lock(applicationId)) { if ( ! tenantApplications.exists(applicationId)) return false; + Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId); + if (activeSession.isEmpty()) return false; + // Deleting an application is done by deleting the remote session and waiting // until the config server where the deployment happened picks it up and deletes // the local session - boolean sessionDeleted = tenantApplications.activeSessionOf(applicationId).map(sessionId -> { - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - // TODO: Add support for timeout in request - Duration waitTime = Duration.ofSeconds(60); - if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - return true; - } else { - log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); - return false; - } - }).orElse(true); + long sessionId = activeSession.get(); + RemoteSession remoteSession = getRemoteSession(tenant, sessionId); + remoteSession.createDeleteTransaction().commit(); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + + + if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + throw new InternalServerException("Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + } NestedTransaction transaction = new NestedTransaction(); transaction.add(new ContainerEndpointsCache(tenant.getPath(), tenant.getCurator()).delete(applicationId)); // TODO: Not unit tested @@ -376,7 +386,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); transaction.commit(); - return sessionDeleted; + return true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 5b2f6cab3c4..7ec0f49ed60 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; @@ -24,12 +23,10 @@ import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.stream.Collectors; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 401b6974596..13ef19f5f5d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -17,7 +17,6 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; -import org.apache.commons.io.IOUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -249,7 +248,7 @@ public class ZooKeeperClient { } ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream inputStream = file.createInputStream()) { - IOUtils.copy(inputStream, baos); + inputStream.transferTo(baos); baos.flush(); configCurator.putData(zkPath.append(file.getPath().getName()).getAbsolute(), baos.toByteArray()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java index 595575531d8..07d27ffd5de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; -import org.apache.commons.io.IOUtils; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpResponse.Status.OK; /** * Represents a response for a request to read contents of a file. @@ -28,7 +27,7 @@ public class SessionContentReadResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { try (InputStream inputStream = file.createInputStream()) { - IOUtils.copyLarge(inputStream, outputStream, new byte[1]); + inputStream.transferTo(outputStream); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java index 707ba6e361e..d027d08eda5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import org.apache.commons.io.IOUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,7 +28,7 @@ public class StaticResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { - IOUtils.copy(body, outputStream); + body.transferTo(outputStream); body.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 1e80ef2d883..babfdfa575d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -66,9 +66,10 @@ public class ApplicationHandler extends HttpHandler { @Override public HttpResponse handleDELETE(HttpRequest request) { ApplicationId applicationId = getApplicationIdFromRequest(request); - boolean deleted = applicationRepository.delete(applicationId); + // TODO: Add support for timeout in request + boolean deleted = applicationRepository.delete(applicationId, Duration.ofSeconds(60)); if ( ! deleted) - return HttpErrorResponse.notFoundError("Unable to delete " + applicationId + ": Not found"); + return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found"); return new DeleteApplicationResponse(Response.Status.OK, applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index b31f4501767..26d322665f0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -49,7 +49,7 @@ public class StaticProvisioner implements HostProvisioner { if (requestedCluster.group().isPresent()) // we are requesting a specific group return nodeCluster.equals(requestedCluster); else // we are requesting nodes of all groups in this cluster - return nodeCluster.equalsIgnoringGroupAndVespaVersion(requestedCluster); + return nodeCluster.satisfies(requestedCluster); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java index 20435d96068..8b46a5b40fa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java @@ -6,12 +6,28 @@ package com.yahoo.vespa.config.server.rpc.security; */ class AuthorizationException extends RuntimeException { - AuthorizationException(String message) { + enum Type {WARN, SILENT} + + private final Type type; + + AuthorizationException(Type type, String message) { super(message); + this.type = type; } - AuthorizationException(String message, Throwable cause) { + AuthorizationException(String message) { + this(Type.WARN, message); + } + + AuthorizationException(Type type, String message, Throwable cause) { super(message, cause); + this.type = type; } + + AuthorizationException(String message, Throwable cause) { + this(Type.WARN, message, cause); + } + + Type type() { return type; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index d20f9ed1abc..8d17a32d102 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -31,6 +31,8 @@ import java.util.concurrent.Executors; import java.util.function.BiConsumer; import java.util.logging.Logger; +import static com.yahoo.vespa.config.server.rpc.security.AuthorizationException.*; + /** * A {@link RpcAuthorizer} that perform access control for configserver RPC methods when TLS and multi-tenant mode are enabled. @@ -110,7 +112,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { if (isConfigKeyForSentinelConfig(configKey)) { return; // config processor will return empty sentinel config for unknown nodes } - throw new AuthorizationException(String.format("Host '%s' not found in host registry for [%s]", hostname, configKey)); + throw new AuthorizationException(Type.SILENT, String.format("Host '%s' not found in host registry for [%s]", hostname, configKey)); } RequestHandler tenantHandler = getTenantHandler(tenantName.get()); ApplicationId resolvedApplication = tenantHandler.resolveApplicationId(hostname); @@ -151,10 +153,13 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { } private void handleAuthorizationFailure(Request request, Throwable throwable) { + boolean isAuthorizationException = throwable instanceof AuthorizationException; String errorMessage = String.format("For request '%s' from '%s': %s", request.methodName(), request.target().toString(), throwable.getMessage()); - log.log(LogLevel.INFO, errorMessage); + if (!isAuthorizationException || ((AuthorizationException) throwable).type() != Type.SILENT) { + log.log(LogLevel.INFO, errorMessage); + } log.log(LogLevel.DEBUG, throwable, throwable::getMessage); - JrtErrorCode error = throwable instanceof AuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; + JrtErrorCode error = isAuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; request.setError(error.code, errorMessage); request.returnRequest(); throwUnchecked(throwable); // rethrow exception to ensure that subsequent completion stages are not executed (don't execute implementation of rpc method). diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index d5a87b3c45e..3ac98f9ac4c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -1,15 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import java.util.logging.Logger; /** @@ -40,7 +37,8 @@ public class LocalSessionStateWatcher { // Will delete session if it exists in local session repo private void sessionChanged(Session.Status status) { long sessionId = session.getSessionId(); - log.log(LogLevel.DEBUG, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); + log.log(status == Session.Status.DELETE ? LogLevel.INFO : LogLevel.DEBUG, + session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { log.log(LogLevel.DEBUG, session.logPre() + "Deleting session " + sessionId); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index f25b41bd121..b7eea3eb7fd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -17,7 +17,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; -import com.yahoo.vespa.config.server.http.LogRetriever; +import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; @@ -34,8 +34,6 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -51,6 +49,7 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author hmusum @@ -277,6 +276,16 @@ public class ApplicationRepositoryTest { assertTrue(applicationRepository.delete(applicationId())); } + + { + try { + deployApp(testApp); + applicationRepository.delete(applicationId(), Duration.ZERO); + fail("Should have gotten an exception"); + } catch (InternalServerException e) { + assertEquals("Session 5 was not deleted (waited PT0S)", e.getMessage()); + } + } } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java index 66d113afbe6..beb8abb7be6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 22ce0db89e2..87f4363050b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -123,6 +123,15 @@ public class ApplicationHandlerTest { } @Test + public void testDeleteNonExistent() throws Exception { + deleteAndAssertResponse(applicationId, + Zone.defaultZone(), + Response.Status.NOT_FOUND, + HttpErrorResponse.errorCodes.NOT_FOUND, + "Unable to delete mytenant.default.default: Not found"); + } + + @Test public void testGet() throws Exception { long sessionId = applicationRepository.deploy(testApp, prepareParams(applicationId)).sessionId(); assertApplicationGeneration(applicationId, Zone.defaultZone(), sessionId, true); @@ -234,6 +243,10 @@ public class ApplicationHandlerTest { deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE); } + private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse) throws IOException { + deleteAndAssertResponse(toUrlPath(applicationId, zone, true), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE); + } + private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { ApplicationHandler handler = createApplicationHandler(); HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 01c6631111e..a106f52e4d1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.google.common.io.Files; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -24,6 +23,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertNotNull; @@ -145,8 +145,8 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { } private File createTestApp() throws IOException { - File testApp = Files.createTempDir(); - FileUtils.copyDirectory(new File("src/test/apps/content"), testApp); + File testApp = Files.createTempDirectory("session-content-handler-test-app").toFile(); + IOUtils.copyDirectory(new File("src/test/apps/content"), testApp); return testApp; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java index 8a798570101..492767728e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import com.yahoo.component.Version; @@ -115,4 +116,4 @@ public class ApplicationMetricsRetrieverTest { throw new UnsupportedOperationException(); } } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java index da676663d3b..0894e38ce09 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -97,4 +98,4 @@ public class ClusterMetricsRetrieverTest { if (a.isPresent() != b.isPresent()) throw new AssertionFailedError("Both optionals are not present: " + a + ", " + b); a.ifPresent(x -> b.ifPresent(y -> comparer.accept(x, y))); } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java index 28827cec53e..9e1edb35b8f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc.security;// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.yahoo.cloud.config.LbServicesConfig; @@ -307,4 +308,4 @@ public class MultiTenantRpcAuthorizerTest { } } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java index 053a3f7a15d..ddaf547e413 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.ContainerEndpoint; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java index f20d6b8fcf8..c71c7b8e040 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.provision.ApplicationId; @@ -69,4 +70,4 @@ public class TlsSecretsKeysTest { assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}", new String(curator.getData(tlsSecretsKeysPath).get())); } -}
\ No newline at end of file +} diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index c82cfd000d9..53c17257992 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -51,10 +51,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <scope>test</scope> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index 72eaf129453..b943c03f48f 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -2,7 +2,6 @@ package com.yahoo.container.logging; import com.yahoo.collections.ListMap; -import org.apache.commons.lang.builder.ReflectionToStringBuilder; import javax.security.auth.x500.X500Principal; import java.net.InetAddress; @@ -456,9 +455,34 @@ public class AccessLogEntry { @Override public String toString() { synchronized (monitor) { - return new ReflectionToStringBuilder(this) - .setExcludeFieldNames(FIELDS_EXCLUDED_FROM_TOSTRING) - .toString(); + return "AccessLogEntry{" + + "ipV4AddressInDotDecimalNotation='" + ipV4AddressInDotDecimalNotation + '\'' + + ", timeStampMillis=" + timeStampMillis + + ", durationBetweenRequestResponseMillis=" + durationBetweenRequestResponseMillis + + ", numBytesReturned=" + numBytesReturned + + ", remoteAddress='" + remoteAddress + '\'' + + ", remotePort=" + remotePort + + ", peerAddress='" + peerAddress + '\'' + + ", peerPort=" + peerPort + + ", profile='" + profile + '\'' + + ", errorMessage='" + errorMessage + '\'' + + ", fileName='" + fileName + '\'' + + ", userAgent='" + userAgent + '\'' + + ", referer='" + referer + '\'' + + ", user='" + user + '\'' + + ", hitCounts=" + hitCounts + + ", httpMethod='" + httpMethod + '\'' + + ", httpVersion='" + httpVersion + '\'' + + ", hostString='" + hostString + '\'' + + ", statusCode=" + statusCode + + ", scheme='" + scheme + '\'' + + ", localPort=" + localPort + + ", principal=" + principal + + ", sslPrincipal=" + sslPrincipal + + ", rawPath='" + rawPath + '\'' + + ", rawQuery='" + rawQuery + '\'' + + ", keyValues=" + keyValues + + '}'; } } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java index 266437bb8bc..9d122b90641 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; /** diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java index 3ac1e00b1a0..2099037203f 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; import java.io.File; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 7789cae50aa..bfcecd61fa4 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -2,7 +2,6 @@ package com.yahoo.container.jdisc; import com.google.inject.Inject; -import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.ResourceReference; @@ -19,13 +18,13 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.log.LogLevel; import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.Collections.singletonMap; /** * A request handler which assigns a worker thread to handle each request. @@ -79,19 +78,20 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { this.allowAsyncResponse = allowAsyncResponse; } - private Map<String, Metric.Context> handlerContexts = new CopyOnWriteHashMap<>(); - private Metric.Context contextFor(BindingMatch match) { + private Metric.Context contextFor(Request request) { + BindingMatch match = request.getBindingMatch(); if (match == null) return null; UriPattern matched = match.matched(); if (matched == null) return null; String name = matched.toString(); - Metric.Context context = handlerContexts.get(name); - if (context == null) { - Map<String, String> dimensions = singletonMap("handler", name); - context = this.metric.createContext(dimensions); - handlerContexts.put(name, context); + String endpoint = request.headers().containsKey("Host") ? request.headers().get("Host").get(0) : null; + + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("handler", name); + if (endpoint != null) { + dimensions.put("endpoint", endpoint); } - return context; + return this.metric.createContext(dimensions); } /** @@ -101,7 +101,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { */ @Override public final ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { - metric.add("handled.requests", 1, contextFor(request.getBindingMatch())); + metric.add("handled.requests", 1, contextFor(request)); if (request.getTimeout(TimeUnit.SECONDS) == null) { Duration timeout = getTimeout(); if (timeout != null) { @@ -195,7 +195,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { if ( tryHasResponded()) throw new IllegalStateException("Response already handled"); ContentChannel cc = responseHandler.handleResponse(response); long millis = request.timeElapsed(TimeUnit.MILLISECONDS); - metric.set("handled.latency", millis, contextFor(request.getBindingMatch())); + metric.set("handled.latency", millis, contextFor(request)); return cc; } diff --git a/container-core/src/main/sh/vespa-load-balancer-status b/container-core/src/main/sh/vespa-load-balancer-status index d7383c1614c..e9cadc487a1 100755 --- a/container-core/src/main/sh/vespa-load-balancer-status +++ b/container-core/src/main/sh/vespa-load-balancer-status @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index a806df1883f..78541137db5 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; import com.fasterxml.jackson.databind.JsonNode; diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 05de643a116..34ad2883625 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -89,7 +89,7 @@ <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> + <include>commons-logging:commons-logging:[1.1.3]:jar:provided</include> <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 1acc1050acd..c743c6c2b2e 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -118,7 +118,7 @@ <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> <!-- This version is exported by jdisc via jcl-over-slf4j. --> - <version>1.1.1</version> + <version>1.1.3</version> </dependency> <dependency> <groupId>javax.annotation</groupId> diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java index 3b29fa0a04f..ed9b1350b1c 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di.componentgraph.cycle; import java.util.ArrayList; diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java index e1b110d51ee..946330668bd 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di.componentgraph.cycle; diff --git a/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java b/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java index 219aa6b5e8b..174ca301c59 100644 --- a/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java +++ b/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/CycleFinderTest.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di.componentgraph.cycle; diff --git a/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java b/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java index 588c1e30ffe..069f72ad8e7 100644 --- a/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java +++ b/container-di/src/test/java/com/yahoo/container/di/componentgraph/cycle/GraphTest.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di.componentgraph.cycle; diff --git a/container-search/pom.xml b/container-search/pom.xml index 2f82728179b..84ee5b2bc65 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -78,14 +78,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> <exclusions> @@ -112,10 +104,6 @@ </exclusions> </dependency> <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> </dependency> diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 57d093fb1c1..37b0fd7ebfb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -22,7 +22,6 @@ import com.yahoo.search.query.ParameterParser; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; -import org.apache.commons.lang.StringUtils; import java.util.ArrayList; import java.util.Collection; @@ -222,7 +221,7 @@ public class ClusterSearcher extends Searcher { return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + "' is undefined for document type" + plural + " '" + - StringUtils.join(invalidInDocTypes.iterator(), ", ") + "'")); + String.join(", ", invalidInDocTypes) + "'")); } return null; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java index 0dc9a33e3f1..0dd2f4692d9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java @@ -16,9 +16,10 @@ import java.io.PrintStream; * <li>debug true|false</li> * </ol> * - * @author <a href="mailto:vlarsen@yahoo-inc.com">Vidar Larsen</a> + * @author Vidar Larsen */ class UnicodePropertyDump { + public static void main(String[] arg) { int start = 0; int end = 0xffff; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index ec17e046068..ebea2f95af7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -55,6 +55,7 @@ public class StatisticsSearcher extends Searcher { private static final String FAILED_QUERIES_METRIC = "failed_queries"; private static final String MEAN_QUERY_LATENCY_METRIC = "mean_query_latency"; private static final String QUERY_LATENCY_METRIC = "query_latency"; + private static final String QUERY_OFFSET_METRIC = "query_hit_offset"; private static final String QUERIES_METRIC = "queries"; private static final String ACTIVE_QUERIES_METRIC = "active_queries"; private static final String PEAK_QPS_METRIC = "peak_qps"; @@ -232,7 +233,7 @@ public class StatisticsSearcher extends Searcher { //handle exceptions thrown below in searchers try { result = execution.search(query); // Pass on down the chain - } catch (Exception e) { + } catch (Exception e) { incrErrorCount(null, metricContext); throw e; } @@ -263,6 +264,7 @@ public class StatisticsSearcher extends Searcher { hitsPerQuery.put((double) hitCount); metric.set(HITS_PER_QUERY_METRIC, (double) hitCount, metricContext); metric.set(TOTALHITS_PER_QUERY_METRIC, (double) result.getTotalHitCount(), metricContext); + metric.set(QUERY_OFFSET_METRIC, (double) (query.getHits() + query.getOffset()), metricContext); if (hitCount == 0) { emptyResults.increment(); metric.add(EMPTY_RESULTS_METRIC, 1, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index ce4891d7c25..94c347a6927 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.FastHit; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 0749d8476c2..86f1999d8b4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 7fd41862212..58d7035c5e8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index ec2cc5107ab..f2e22ba86dc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 285a1fcd57e..b16fa941f68 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; import com.yahoo.prelude.Pong; diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java index e26bf153eb3..5f1cfccf549 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java @@ -94,6 +94,7 @@ class FederationResult { public Optional<Result> getIfAvailable(long timeout) { if (availableResult.isPresent()) return availableResult; availableResult = futureResult.getIfAvailable(timeout, TimeUnit.MILLISECONDS); + availableResult.ifPresent(result -> target.modifyTargetResult(result)); return availableResult; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 6e36881ae63..c5573dc8fee 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -27,7 +27,6 @@ import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.query.Properties; -import com.yahoo.search.query.properties.QueryProperties; import com.yahoo.search.query.properties.SubProperties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; @@ -39,7 +38,6 @@ import com.yahoo.search.searchchain.ForkingSearcher; import com.yahoo.search.searchchain.FutureResult; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.searchchain.model.federation.FederationOptions; -import org.apache.commons.lang.StringUtils; import java.time.Clock; import java.util.ArrayList; @@ -228,12 +226,15 @@ public class FederationSearcher extends ForkingSearcher { if (timeout <= 0) return Optional.empty(); Execution newExecution = new Execution(target.getChain(), execution.context()); + Result result; if (strictSearchchain) { query.resetTimeout(); - return Optional.of(newExecution.search(createFederationQuery(query, query, Window.from(query), timeout, target))); + result = newExecution.search(createFederationQuery(query, query, Window.from(query), timeout, target)); } else { - return Optional.of(newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target))); + result = newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target)); } + target.modifyTargetResult(result); + return Optional.of(result); } private FederationResult search(Query query, Execution execution, Collection<Target> targets) { @@ -319,8 +320,8 @@ public class FederationSearcher extends ForkingSearcher { } private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { - String message = StringUtils.join(getMessagesSet(unresolvedSearchChainExceptions), ' ') + - " Valid source refs are " + StringUtils.join(allSourceRefDescriptions().iterator(), ", ") +'.'; + String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) + + " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; return ErrorMessage.createInvalidQueryParameter(message); } @@ -560,7 +561,6 @@ public class FederationSearcher extends ForkingSearcher { } private void mergeResult(Query query, Target target, Result mergedResults, Result result) { - target.modifyTargetResult(result); ComponentId searchChainId = target.getId(); Chain<Searcher> searchChain = target.getChain(); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index 0741e8effac..f2e2e1b034d 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.pagetemplates.result; import com.yahoo.io.ByteWriter; @@ -331,4 +332,4 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu writer.close(); } -}
\ No newline at end of file +} diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index 30a3121902c..b73ef1298ee 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -153,13 +153,12 @@ public class RewriterUtils { * * @param query Query object from the searcher * @param paramName parameter to be retrieved - * * @return parameter value or null if not found */ public static String getUserParam(Query query, String paramName) { log(utilsLogger, query, "Retrieving user param value: " + paramName); - if(paramName==null) { + if (paramName == null) { error(utilsLogger, query, "Parameter name is null"); return null; } @@ -172,8 +171,7 @@ public class RewriterUtils { } /** - * Retrieve metadata passed by previous rewriter - * from query properties + * Retrieve metadata passed by previous rewriter from query properties * Initialize values if this is the first rewriter * * @param query Query object from the searcher @@ -183,10 +181,10 @@ public class RewriterUtils { log(utilsLogger, query, "Retrieving metadata passed by previous rewriter"); @SuppressWarnings("unchecked") - HashMap<String, Object> rewriteMeta = (HashMap<String, Object>) query - .properties().get(RewriterConstants.REWRITE_META); + HashMap<String, Object> rewriteMeta = + (HashMap<String, Object>)query.properties().get(RewriterConstants.REWRITE_META); - if(rewriteMeta==null) { + if (rewriteMeta == null) { log(utilsLogger, query, "No metadata available from previous rewriter"); rewriteMeta = new HashMap<>(); rewriteMeta.put(RewriterConstants.REWRITTEN, false); @@ -326,9 +324,9 @@ public class RewriterUtils { * @param msg Error message */ public static void error(Logger logger, Query query, String msg) { - if(query!=null) { + if (query != null) query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); - } logger.severe(logger.getName() + ": " + msg); } + } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java index 347f8607b59..085a9b24993 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; import org.junit.Test; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index f42185e955f..315a05ce14d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; import com.yahoo.container.QrSearchersConfig; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/CostResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/CostResource.java index c006a89be50..e5e882f4ec1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/CostResource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/CostResource.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4; import com.yahoo.vespa.hosted.controller.api.application.v4.model.CostMonths; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItem.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItem.java index 1d87b2fcfb6..f261dc26fa1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItem.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItem.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItemUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItemUsage.java index 9b8967bc9d8..30e213049ba 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItemUsage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostItemUsage.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import java.math.BigDecimal; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostMonths.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostMonths.java index 2c9470399fa..d404f0dade5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostMonths.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostMonths.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import java.util.List; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostResult.java index c0ca5d91d3f..2905b352d2a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostResult.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/CostResult.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import java.util.List; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java index 766ef71b25c..4ee3e849bc7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; public interface ApplicationIdSource { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java index 606d7add8b0..42c7e3c2590 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java index 59fd84aec74..a7c8a680b73 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.aws; import java.util.Date; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java index b0b06fc6c83..79b332c093a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java index 58bdef6df51..b6ad1701449 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java index 76d848e3d1c..a6be46d1b11 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterId.java index b586b9f3019..eac85d0a7b7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java index 21f38084c22..54caf29e149 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.maven; import static java.util.Objects.requireNonNull; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java index fb133f75654..8877c082d93 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.maven; /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java index fd84a05db6a..ef9c9bcd1f3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.maven; import com.yahoo.component.Version; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingInfo.java index 1bc3764aacd..b07d244dd03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingInfo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ContactRetriever.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ContactRetriever.java index 9b05234d529..f82fcd5e91c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ContactRetriever.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ContactRetriever.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index 7d1369da864..2adc83ff415 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java index 38b9cd4db5a..8d75b375044 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.google.common.collect.ImmutableList; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mailer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mailer.java index d541dd90237..1df81b5ddc1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mailer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mailer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java index 5c4bfcf219d..92f013f85e0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumerMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumerMock.java index c59e13246cd..ce56797731a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumerMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumerMock.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java index 9ef89d53068..e69398c4a77 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java index be1deb3997a..7c31cbfcce9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.component.Version; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java index ee7337b524d..6df44ceaf9d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.vespa.hosted.controller.api.integration.user.User; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java index c8c21c13fc1..77bd589f23b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.user; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java index ac64df7713f..7ad8a8a4197 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.user; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserId.java index 3b138d0ce18..f82aebe3bbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.user; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java index 922f0f11244..8a549b505c7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.user; import com.yahoo.vespa.hosted.controller.api.role.Role; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java index 37ef06d461b..293ee208e13 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Enforcer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Enforcer.java index c83e25c6073..dee22f0b753 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Enforcer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Enforcer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import com.yahoo.config.provision.SystemName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java index 92f4e17017a..6cc726f2ac3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 10df7604667..58d69512feb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import java.util.EnumSet; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java index 11e4552fcb5..3d6f2cd00b8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import java.security.Principal; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java index 8712b436c0d..b53b0d1cc2d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import com.yahoo.config.provision.SystemName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java index 93774b25210..64dc501e666 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import com.yahoo.config.provision.SystemName; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java index 17d0694538c..0c96807b0c7 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java @@ -1,12 +1,9 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.maven; import com.yahoo.component.Version; -import org.junit.Assert; import org.junit.Test; -import java.net.URI; -import java.nio.file.Path; - import static org.junit.Assert.assertEquals; public class MetadataTest { diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java index 19d2d1a6c49..cfb5462e50a 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.user; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/PathGroupTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/PathGroupTest.java index 9d76d055877..916c7d8dcb0 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/PathGroupTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/PathGroupTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.role; import org.junit.Test; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index e23230b8503..638c0e5db83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java index 09fd0b1720a..1e9fe067d6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import org.jetbrains.annotations.NotNull; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java index b4f0d6e2487..bf9b8fce595 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Badges.java index 2576a439be9..f5369406f97 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Badges.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index bc1ad37dae2..e574b994870 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index cfc5a305678..6f4d947a3c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java index 52d60aca388..d5988c9d9b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index dc17e1c17bb..e9c9e111c8b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index c33c4681dd9..a083e2d0210 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index 9141fe30f14..eda3d0fc571 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 4c61b341c20..83173fc32a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import org.jetbrains.annotations.TestOnly; import java.time.Duration; -import java.time.Instant; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -40,7 +39,8 @@ public class JobRunner extends Maintainer { } @TestOnly - public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) { + public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, + StepRunner runner) { super(controller, duration, jobControl); this.jobs = controller.jobController(); this.jobs.setRunner(this::advance); @@ -102,7 +102,7 @@ public class JobRunner extends Maintainer { StepInfo stepInfo = run.stepInfo(lockedStep.get()).orElseThrow(); if (stepInfo.startTime().isEmpty()) { - jobs.setStartTimestamp(run.id(), Instant.now(), lockedStep); + jobs.setStartTimestamp(run.id(), controller().clock().instant(), lockedStep); } runner.run(lockedStep, run.id()).ifPresent(status -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java index 07cf7045e66..567cb664788 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.metric; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 6413be181c3..6e22acd65a1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -445,6 +445,13 @@ class JobControllerApiHandlerHelper { } runLog.lastId().ifPresent(id -> detailsObject.setLong("lastId", id)); + Cursor stepsObject = detailsObject.setObject("steps"); + run.steps().forEach((step, info) -> { + Cursor stepCursor = stepsObject.setObject(step.name()); + stepCursor.setString("status", info.status().name()); + info.startTime().ifPresent(startTime -> stepCursor.setLong("startMillis", startTime.toEpochMilli())); + }); + return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index 8b300c10d1d..8bf3f4a6fd0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.CloudName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index 04c65480dd6..ec01b3817a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.CloudName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 139d5e03f49..1aaecb58a8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 6b6a93934b3..b89a9dd7f5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -5,6 +5,7 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; @@ -98,14 +99,16 @@ class SystemFlagsDeployer { if (currentData != null) { return; // not a new flag } - if (!dryRun) { - try { + try { + if (!dryRun) { client.putFlagData(target, data); - } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); - errors.add(OperationError.createFailed(e.getMessage(), target, data)); - return; + } else { + dryRunFlagDataValidation(data); } + } catch (Exception e) { + log.log(LogLevel.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); + errors.add(OperationError.createFailed(e.getMessage(), target, data)); + return; } results.add(FlagDataChange.created(id, target, data)); }); @@ -122,14 +125,16 @@ class SystemFlagsDeployer { if (currentData == null || isEqual(currentData, wantedData)) { return; // not an flag data update } - if (!dryRun) { - try { + try { + if (!dryRun) { client.putFlagData(target, wantedData); - } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); - errors.add(OperationError.updateFailed(e.getMessage(), target, wantedData)); - return; + } else { + dryRunFlagDataValidation(wantedData); } + } catch (Exception e) { + log.log(LogLevel.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); + errors.add(OperationError.updateFailed(e.getMessage(), target, wantedData)); + return; } results.add(FlagDataChange.updated(id, target, wantedData, currentData)); }); @@ -158,6 +163,11 @@ class SystemFlagsDeployer { }); } + private static void dryRunFlagDataValidation(FlagData data) { + Flags.getFlag(data.id()) + .ifPresent(definition -> data.validate(definition.getUnboundFlag().serializer())); + } + private static Map<FlagId, FlagData> lookupTable(Collection<FlagData> data) { return data.stream().collect(Collectors.toMap(FlagData::id, Function.identity())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java index 8db96801a36..02387213135 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java index bff750db7c3..7db183e8fd4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java index 38faab1483d..6cf0c8f4fb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java index 57c9e037f6a..ebdb70e2c6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.vespa.athenz.api.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java index b122199bf05..2a39b03c2d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 4f3a55dbea0..f34c9c67baa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java index ea931616211..c0f8f585216 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java index eed3fee2c04..a44c13e9ae0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java index 10e47d2f96d..7a25dbfc439 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import java.security.Principal; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java index 358088e9b08..35994e4c5bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index e230daf0c50..8601fb34e8f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tenant; import com.google.common.collect.BiMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java index 9f3addd4992..577db7b6bf3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java index 79e4072c2e7..b2a6a75b937 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 038a2afb306..5b5c6b61357 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; @@ -29,9 +28,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.temporal.TemporalAdjusters; import java.util.Collections; import java.util.logging.Logger; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index 2fc49c3dc21..ce22e0772ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ClusterSpec; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 9867ea4bdc2..ca624281e94 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index e5e124d6647..0a0aa7c9ded 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; @@ -50,4 +51,4 @@ public class ResourceTagMaintainerTest { tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java index 23e86320a78..dad7ea4ec31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.metric; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 9473e88acf8..16447f47ee8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index b53a4862192..07888ec1609 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.component.Version; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 21ef5035481..e5b2fef2120 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -59,5 +59,18 @@ }, "active": true, "lastId": 9, + "steps": { + "deployReal": { + "startMillis": 0, + "status": "succeeded" + }, + "installReal": { + "startMillis": 0, + "status": "unfinished" + }, + "copyVespaLogs": { + "status": "unfinished" + } + }, "status": "running" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index 6f0c5c8a384..3248bfa78a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -50,5 +50,19 @@ }, "active": false, "lastId": 18, + "steps": { + "deployReal": { + "startMillis": 0, + "status": "succeeded" + }, + "installReal": { + "startMillis": 0, + "status": "succeeded" + }, + "copyVespaLogs": { + "startMillis": 0, + "status": "succeeded" + } + }, "status": "success" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 993c53b52d8..986109cd1be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -399,6 +399,48 @@ } ] }, - "lastId": 75 + "lastId": 75, + "steps": { + "deployTester": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "deployReal": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "installTester": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "installReal": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "startTests": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "endTests": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "copyVespaLogs": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "deactivateReal": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "deactivateTester": { + "status": "succeeded", + "startMillis": "(ignore)" + }, + "report": { + "status": "succeeded", + "startMillis": "(ignore)" + } + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json index e344ef07762..4e501410324 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json @@ -1,6 +1,4 @@ { - "active": false, - "status": "deploymentFailed", "log": { "deployTester": [ { @@ -17,5 +15,36 @@ } ] }, - "lastId": 2 + "active": false, + "lastId": 2, + "steps": { + "startTests": { + "status": "unfinished" + }, + "deployTester": { + "startMillis": 1000, + "status": "failed" + }, + "report": { + "startMillis": 1000, + "status": "succeeded" + }, + "installTester": { + "status": "unfinished" + }, + "deployReal": { + "status": "unfinished" + }, + "installReal": { + "status": "unfinished" + }, + "deactivateTester": { + "startMillis": 1000, + "status": "succeeded" + }, + "endTests": { + "status": "unfinished" + } + }, + "status": "deploymentFailed" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java index 34ee160c449..8d3f2d584b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.athenz; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 37cf5511c0b..ff02a1e16be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.vespa.hosted.controller.ControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index d5d635294b5..82b97a5b144 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index 137a0d016dd..343a1f07a21 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 3088bcb2529..d1dd50cfb4c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java index 026d174cb73..3f7b9db1dba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; diff --git a/document/pom.xml b/document/pom.xml index 19f1a9470fc..1696d525f6f 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -27,10 +27,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index f7bb5aecf0b..4a9434d43bc 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -9,8 +9,8 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.json.TokenBuffer; import com.yahoo.document.update.ValueUpdate; -import org.apache.commons.codec.binary.Base64; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -86,7 +86,7 @@ public class SingleValueReader { public static FieldValue readAtomic(String field, DataType expectedType) { if (expectedType.equals(DataType.RAW)) { - return expectedType.createFieldValue(new Base64().decode(field)); + return expectedType.createFieldValue(Base64.getMimeDecoder().decode(field)); } else if (expectedType.equals(PositionDataType.INSTANCE)) { return PositionDataType.fromString(field); } else if (expectedType instanceof ReferenceDataType) { diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index a0e7f81ea73..85bc4d032ff 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -5,8 +5,8 @@ import com.yahoo.document.Document; import com.yahoo.document.Field; import com.yahoo.document.datatypes.*; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -84,7 +84,7 @@ public class XmlSerializationHelper { public static void printRawXml(Raw r, XmlStream xml) { xml.addAttribute("binaryencoding", "base64"); - xml.addContent(new Base64(0).encodeToString(r.getByteBuffer().array())); + xml.addContent(Base64.getEncoder().encodeToString(r.getByteBuffer().array())); } public static void printStringXml(StringFieldValue s, XmlStream xml) { @@ -92,7 +92,7 @@ public class XmlSerializationHelper { if (containsNonPrintableCharactersString(content)) { byte[] bytecontent = Utf8.toBytes(content); xml.addAttribute("binaryencoding", "base64"); - xml.addContent(new Base64(0).encodeToString(bytecontent)); + xml.addContent(Base64.getEncoder().encodeToString(bytecontent)); } else { xml.addContent(content); } diff --git a/document/src/main/java/com/yahoo/document/update/TensorAddUpdate.java b/document/src/main/java/com/yahoo/document/update/TensorAddUpdate.java index f8d2464deb7..b607076e356 100644 --- a/document/src/main/java/com/yahoo/document/update/TensorAddUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/TensorAddUpdate.java @@ -48,7 +48,7 @@ public class TensorAddUpdate extends ValueUpdate<TensorFieldValue> { Tensor old = ((TensorFieldValue) oldValue).getTensor().get(); Tensor update = tensor.getTensor().get(); - Tensor result = old.merge((left, right) -> right, update.cells()); // note this might be slow for large mixed tensor updates + Tensor result = old.merge(update, (left, right) -> right); // note this might be slow for large mixed tensor updates return new TensorFieldValue(result); } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java index 737371f2375..02682fa943f 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java @@ -16,12 +16,12 @@ import com.yahoo.document.serialization.DeserializationException; import com.yahoo.document.serialization.FieldReader; import com.yahoo.text.Utf8; import com.yahoo.vespa.objects.FieldBase; -import org.apache.commons.codec.binary.Base64; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import java.io.InputStream; import java.math.BigInteger; +import java.util.Base64; import java.util.Optional; /** @@ -242,7 +242,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { } private void assignPositionFieldFromStringIfNonEmpty(Struct value, String elementText, boolean base64) { - String str = base64 ? Utf8.toString(new Base64().decode(elementText)) : elementText; + String str = base64 ? Utf8.toString(Base64.getMimeDecoder().decode(elementText)) : elementText; str = str.trim(); if (str.isEmpty()) { return; @@ -409,7 +409,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { public void read(FieldBase field, Raw value) { try { if (isBase64EncodedElement(reader)) { - value.assign(new Base64().decode(reader.getElementText())); + value.assign(Base64.getMimeDecoder().decode(reader.getElementText())); } else { value.assign(reader.getElementText().getBytes()); } @@ -422,7 +422,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { public void read(FieldBase field, PredicateFieldValue value) { try { if (isBase64EncodedElement(reader)) { - value.assign(Predicate.fromBinary(new Base64().decode(reader.getElementText()))); + value.assign(Predicate.fromBinary(Base64.getMimeDecoder().decode(reader.getElementText()))); } else { value.assign(Predicate.fromString(reader.getElementText())); } diff --git a/document/src/test/java/com/yahoo/document/datatypes/BoolFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/BoolFieldValueTestCase.java index 0afce499cb1..01b5fb6870f 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/BoolFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/BoolFieldValueTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.datatypes; import org.junit.Test; diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java index 7293dcdb8eb..9a89da75cd0 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java @@ -52,7 +52,7 @@ public class TensorFieldValueTestCase { @Test public void requireThatToStringWorks() { TensorFieldValue field1 = createFieldValue("{{x:0}:2.0}"); - assertEquals("tensor(x{}):{{x:0}:2.0}", field1.toString()); + assertEquals("tensor(x{}):{0:2.0}", field1.toString()); TensorFieldValue field2 = new TensorFieldValue(TensorType.fromSpec("tensor(x{})")); assertEquals("null", field2.toString()); } diff --git a/document/src/test/java/com/yahoo/document/select/ArithmeticNodeTestCase.java b/document/src/test/java/com/yahoo/document/select/ArithmeticNodeTestCase.java index 20b0e2034cc..abf3bc80dbf 100644 --- a/document/src/test/java/com/yahoo/document/select/ArithmeticNodeTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/ArithmeticNodeTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.select; import com.yahoo.document.select.rule.ArithmeticNode; diff --git a/document/src/test/java/com/yahoo/document/select/LogicalNodeTestCase.java b/document/src/test/java/com/yahoo/document/select/LogicalNodeTestCase.java index 90cde23bf30..aedcb882908 100644 --- a/document/src/test/java/com/yahoo/document/select/LogicalNodeTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/LogicalNodeTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.select; import com.yahoo.document.BucketIdFactory; diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java index 29621426a85..a3db1b06abe 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java @@ -13,13 +13,13 @@ import com.yahoo.document.predicate.FeatureSet; import com.yahoo.document.predicate.Predicate; import com.yahoo.document.serialization.DeserializationException; import com.yahoo.tensor.TensorType; -import org.apache.commons.codec.binary.Base64; import org.junit.Test; import javax.xml.stream.XMLStreamReader; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -161,7 +161,7 @@ public class VespaXmlFieldReaderTestCase { assertRead(predicate, "<document id='id:ns:my_type::' type='my_type'>" + " <my_predicate binaryencoding='base64'>" + - Base64.encodeBase64String(BinaryFormat.encode(predicate)) + + Base64.getMimeEncoder().encodeToString(BinaryFormat.encode(predicate)) + " </my_predicate>" + "</document>"); } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 4cc8703b954..6339416d007 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1007,7 +1007,7 @@ public class DocumentGenPluginTest { Book book = new Book(new DocumentId("id:book:book::0")); assertNull(book.getVector()); book.setVector(Tensor.from("{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}")); - assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}", book.getVector().toString()); + assertEquals("tensor(x{}):{0:1.0,1:2.0,2:3.0}", book.getVector().toString()); } @Test diff --git a/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp index 3e7d234fd54..385da6d1fcd 100644 --- a/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/dense/typed_dense_tensor_builder.cpp @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "typed_dense_tensor_builder.h" diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 14086c2bf7a..532f7696893 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,6 +13,7 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * Definitions of feature flags. @@ -183,6 +184,12 @@ public class Flags { "Whether to disable CM3.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag MIGRATE_AWS_TARGET_GROUP_HEALTH_CHECKS = defineFeatureFlag( + "migrate-aws-health-checks", false, + "Migrate target group health checks to https", + "Takes effect on config server restart", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fsa/src/main/java/com/yahoo/fsa/FSA.java b/fsa/src/main/java/com/yahoo/fsa/FSA.java index f0fc27a15c5..5a4a0e4475f 100644 --- a/fsa/src/main/java/com/yahoo/fsa/FSA.java +++ b/fsa/src/main/java/com/yahoo/fsa/FSA.java @@ -83,7 +83,7 @@ public class FSA implements Closeable { public void delta(String string){ ByteBuffer buf = fsa.encode(string); Maps m = fsa.map(); - while(state >0 && buf.position()<buf.limit()){ + while (state >0 && buf.position()<buf.limit()){ delta(m, buf.get()); } } @@ -106,7 +106,7 @@ public class FSA implements Closeable { /** Jumps ahead by a word - if this is not the first word, it must be preceeded by space. */ public void deltaWord(String string){ - if (state!=fsa.start()) { + if (state != fsa.start()) { delta((byte)' '); } delta(string); diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java index 97453d97c6d..ab2ec75d7f3 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java @@ -4,39 +4,38 @@ package com.yahoo.fsa.segmenter; /** * Class encapsulation of a segment. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segment { - int _beg; - int _end; - int _conn; + final int begin; + final int end; + final int conn; - public Segment(int b, int e, int c) - { - _beg = b; - _end = e; - _conn = c; + public Segment(int b, int e, int c) { + begin = b; + end = e; + conn = c; } public int beg() { - return _beg; + return begin; } public int end() { - return _end; + return end; } public int len() { - return _end-_beg; + return end - begin; } public int conn() { - return _conn; + return conn; } } diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java index 64c20e38b55..505085c46cf 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java @@ -9,58 +9,56 @@ import com.yahoo.fsa.FSA; /** * API for accessing the Segmenter automata. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segmenter { - private FSA _fsa; + private final FSA fsa; public Segmenter(FSA fsa) { - _fsa = fsa; + this.fsa = fsa; } public Segmenter(String filename) { - _fsa = new FSA(filename,"utf-8"); + fsa = new FSA(filename, "utf-8"); } public Segmenter(String filename, String charsetname) { - _fsa = new FSA(filename,charsetname); + fsa = new FSA(filename, charsetname); } - public boolean isOk() - { - return _fsa.isOk(); + public boolean isOk() { + return fsa.isOk(); } - public Segments segment(String input) - { + public Segments segment(String input) { String[] tokens = input.split("\\s"); return segment(tokens); } private class Detector { - FSA.State _state; - int _index; - public Detector(FSA.State s, int i) - { - _state = s; - _index = i; + final FSA.State state; + final int index; + + public Detector(FSA.State s, int i) { + state = s; + index = i; } public FSA.State state() { - return _state; + return state; } public int index() { - return _index; + return index; } + } - public Segments segment(String[] tokens) - { + public Segments segment(String[] tokens) { Segments segments = new Segments(tokens); LinkedList detectors = new LinkedList(); @@ -68,7 +66,7 @@ public class Segmenter { while(i<tokens.length){ - detectors.add(new Detector(_fsa.getState(),i)); + detectors.add(new Detector(fsa.getState(), i)); ListIterator det_it = detectors.listIterator(); while(det_it.hasNext()){ @@ -88,50 +86,5 @@ public class Segmenter { return segments; } - //// test //// - public static void main(String[] args) { - String fsafile = "/home/gv/fsa/automata/segments.fsa"; - - Segmenter segmenter = new Segmenter(fsafile); - - System.out.println("Loading segmenter FSA file "+fsafile+": "+segmenter.isOk()); - - for(int a=0;a<1||a<args.length;a++){ - - String query; - if(a==args.length){ - query = "times square head"; - } - else { - query = args[a]; - } - System.out.println("processing query \""+query+"\""); - - Segments segments = segmenter.segment(query); - System.out.println("all segments:"); - for(int i=0; i<segments.size();i++){ - System.out.println(" "+i+": \""+segments.sgm(i)+"\","+segments.conn(i)); - } - - Segments best; - - best = segments.segmentation(Segments.SEGMENTATION_WEIGHTED); - System.out.print("best segments (weighted): "); - for(int i=0; i<best.size();i++){ - System.out.print("("+best.sgm(i)+")"); - } - System.out.println(); - - best = segments.segmentation(Segments.SEGMENTATION_RIGHTMOST_LONGEST); - System.out.print("best segments (rightmost_longest):"); - for(int i=0; i<best.size();i++){ - System.out.print("("+best.sgm(i)+")"); - } - System.out.println(); - - } - - } - } diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java index 6d7ee7be1a6..e01afa91cd5 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java @@ -6,7 +6,7 @@ import java.util.LinkedList; /** * Contains the segmentation() method. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segments extends LinkedList { diff --git a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java index f29dc38f2ce..cc3ff2e79ac 100644 --- a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java +++ b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java @@ -25,13 +25,13 @@ import com.yahoo.fsa.MetaData; * predicted topics for a term. Each topic has an attached weight and * a term vector (topicSegments). * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> - **/ + * @author Peter Boros + */ public class TopicPredictor extends MetaData { private static final String packageName = "com.yahoo.fsa.topicpredictor"; - private FSA fsa = null; + private final FSA fsa; public TopicPredictor(String fsafile, String datfile){ this(fsafile, datfile, "utf-8"); @@ -154,27 +154,4 @@ public class TopicPredictor extends MetaData { return getStringArrayEntry(user(0) + topicId, 2); } - - //// test //// - public static void main(String[] args) { - String segment = "new york"; - if (args.length >= 1) { - segment = args[0]; - } - - String fsafile = "/home/gv/fsa/automata/dmozPred_2.fsa"; - String datfile = "/home/gv/fsa/automata/dmozPred_2.dat"; - - TopicPredictor predictor = new TopicPredictor(fsafile, datfile); - - List predictedTopics = predictor.getPredictedTopics(segment, 25); - Iterator i = predictedTopics.iterator(); - while (i.hasNext()) { - PredictedTopic topic = (PredictedTopic) i.next(); - System.out.println("\n topic=" + topic.getTopic()); - System.out.println(" weight=" + topic.getWeight()); - System.out.println(" vector=" + topic.getVector()); - } - } - } diff --git a/hosted-api/README.md b/hosted-api/README.md index 28eea5c3f3d..fe88fef51e0 100644 --- a/hosted-api/README.md +++ b/hosted-api/README.md @@ -1 +1,2 @@ -# Hosted Vespa controller API miscellaneous
\ No newline at end of file +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +# Hosted Vespa controller API miscellaneous diff --git a/hosted-api/pom.xml b/hosted-api/pom.xml index 928a173f9d8..2a42f890ba4 100644 --- a/hosted-api/pom.xml +++ b/hosted-api/pom.xml @@ -1,4 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ApiAuthenticator.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ApiAuthenticator.java index 674faaf0ee7..dcbebb37258 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ApiAuthenticator.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ApiAuthenticator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; public interface ApiAuthenticator { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java index e745f555847..d012d27fbd8 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import java.nio.file.Path; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java index ded82a8573d..177c72107e0 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import java.time.Instant; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java index 34ce04b95fd..87a4c21306f 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/EndpointAuthenticator.java b/hosted-api/src/main/java/ai/vespa/hosted/api/EndpointAuthenticator.java index 20e3b18eb56..81813335a63 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/EndpointAuthenticator.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/EndpointAuthenticator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import javax.net.ssl.SSLContext; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java index 39d56408fb8..fd640bcc235 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java index 768b7d9a50c..870afaa4f99 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java index 9ac02cd9130..dda483ef882 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; diff --git a/http-utils/README.md b/http-utils/README.md index 8bbe2cd7d05..714b7581ad4 100644 --- a/http-utils/README.md +++ b/http-utils/README.md @@ -1 +1,2 @@ -# Http utilities for Java
\ No newline at end of file +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +# Http utilities for Java diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index 74c2f701a67..0c6791e05b5 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -44,11 +44,6 @@ <artifactId>predicate-search-core</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.4</version> - </dependency> </dependencies> <build> <plugins> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java index 0b392ebfa03..463c009e0a3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -3,7 +3,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.LongFieldValue; -import org.apache.commons.codec.binary.Base64; + +import java.util.Base64; /** * @author Simon Thoresen Hult @@ -24,7 +25,7 @@ public final class Base64DecodeExpression extends Expression { if (input.length() > 12) { throw new NumberFormatException("Base64 value '" + input + "' is out of range."); } - byte[] decoded = Base64.decodeBase64(input); + byte[] decoded = Base64.getDecoder().decode(input); if (decoded == null || decoded.length == 0) { throw new NumberFormatException("Illegal base64 value '" + input + "'."); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java index ff698bf2dd7..e61ac7051c1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import org.apache.commons.codec.binary.Base64; + +import java.util.Base64; /** * @author Simon Thoresen Hult @@ -23,7 +23,7 @@ public final class Base64EncodeExpression extends Expression { output[i] = (byte)(input & 0xffL); input >>>= 8; } - String encoded = new Base64(0).encodeToString(output); + String encoded = Base64.getEncoder().encodeToString(output); ctx.setValue(new StringFieldValue(encoded)); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java index df4fd981e01..3e547cedc4b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java @@ -58,7 +58,7 @@ public class Base64DecodeTestCase { new Base64DecodeExpression().execute(new StringFieldValue("???")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Illegal base64 value '???'.", e.getMessage()); + assertEquals("Illegal base64 character 3f", e.getMessage()); } } diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java index 914a9d7b42c..9d843af7fb5 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.jaxrs.client; import java.time.Duration; diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/LegacyJaxRsTimeouts.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/LegacyJaxRsTimeouts.java index 3f2139f6bf0..9f97f71987e 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/LegacyJaxRsTimeouts.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/LegacyJaxRsTimeouts.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.jaxrs.client; import java.time.Duration; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java index 43586c3e8cc..0f863b4b5a2 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; import org.osgi.framework.Bundle; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 6e890fd2c2d..5a2608b63be 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -24,6 +24,7 @@ import java.util.logging.Logger; * @author gjoranv */ public class ExportPackages { + private static final Logger log = Logger.getLogger(ExportPackages.class.getName()); public static final String PROPERTIES_FILE = "/exportPackages.properties"; diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BundleCollisionHookIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BundleCollisionHookIntegrationTest.java index 80de2a28322..8943d1c8545 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BundleCollisionHookIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BundleCollisionHookIntegrationTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; import com.yahoo.jdisc.test.TestDriver; diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 979b9418f4a..c5555d5b690 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -81,11 +81,6 @@ <!-- TEST SCOPE --> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpmime</artifactId> <scope>test</scope> diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java index 4d3bb4a280a..006a282e1e0 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java @@ -33,4 +33,5 @@ class JDiscSslContextFactory extends SslContextFactory.Server { Objects.toString(getTrustStoreProvider(), getKeyStoreProvider()), trustStorePassword); } + } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/servlet/ServletTestBase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/servlet/ServletTestBase.java index fe6a1974a35..54bfe8c026d 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/servlet/ServletTestBase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/servlet/ServletTestBase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty.servlet; -import com.google.common.collect.ImmutableMap; import com.google.inject.AbstractModule; import com.google.inject.Module; import com.google.inject.TypeLiteral; @@ -16,8 +15,6 @@ import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.jdisc.http.ServletPathsConfig.Servlets.Builder; import com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.RequestExecutor; import com.yahoo.jdisc.http.server.jetty.TestDriver; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.eclipse.jetty.servlet.ServletHolder; import javax.servlet.ServletException; @@ -27,15 +24,27 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; +import java.util.List; /** * @author Tony Vaagenes * @author bakksjo */ public class ServletTestBase { - private ImmutableMap<Pair<ComponentId, String>, HttpServlet> servlets = ImmutableMap.of( - ImmutablePair.of(TestServlet.ID, TestServlet.PATH), new TestServlet(), - ImmutablePair.of(NoContentTestServlet.ID, NoContentTestServlet.PATH), new NoContentTestServlet()); + + private static class ServletInstance { + final ComponentId componentId; final String path; final HttpServlet instance; + + ServletInstance(ComponentId componentId, String path, HttpServlet instance) { + this.componentId = componentId; + this.path = path; + this.instance = instance; + } + } + + private final List<ServletInstance> servlets = List.of( + new ServletInstance(TestServlet.ID, TestServlet.PATH, new TestServlet()), + new ServletInstance(NoContentTestServlet.ID, NoContentTestServlet.PATH, new NoContentTestServlet())); protected RequestExecutor httpGet(TestDriver testDriver, String path) { return testDriver.client().newGet("/" + path); @@ -44,10 +53,10 @@ public class ServletTestBase { protected ServletPathsConfig createServletPathConfig() { ServletPathsConfig.Builder configBuilder = new ServletPathsConfig.Builder(); - servlets.forEach((idAndPath, servlet) -> + servlets.forEach(servlet -> configBuilder.servlets( - idAndPath.getLeft().stringValue(), - new Builder().path(idAndPath.getRight()))); + servlet.componentId.stringValue(), + new Builder().path(servlet.path))); return new ServletPathsConfig(configBuilder); } @@ -55,8 +64,8 @@ public class ServletTestBase { protected ComponentRegistry<ServletHolder> servlets() { ComponentRegistry<ServletHolder> result = new ComponentRegistry<>(); - servlets.forEach((idAndPath, servlet) -> - result.register(idAndPath.getLeft(), new ServletHolder(servlet))); + servlets.forEach(servlet -> + result.register(servlet.componentId, new ServletHolder(servlet.instance))); result.freeze(); return result; diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 31d7543304a..5a4a79995d8 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -133,9 +133,9 @@ public class Mirror implements IMirror { * and end with '**' to match the rest of the name. * Note that this isn't quite globbing, as there is no backtracking. * - * @return true if the name matches the pattern * @param name the name * @param pattern the pattern + * @return true if the name matches the pattern */ static boolean match(char [] name, char [] pattern) { int ni = 0; diff --git a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java index 7803767e4d0..6c36e8f9604 100644 --- a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java +++ b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java @@ -28,10 +28,10 @@ import java.util.ArrayList; public class RpcInvoker { private Value getArgument(Request request, String parameter) { - if (parameter.length()<=1 || parameter.charAt(1)!=':') + if (parameter.length() <= 1 || parameter.charAt(1) != ':') return new StringValue(parameter); - String value=parameter.substring(2); + String value = parameter.substring(2); switch (parameter.charAt(0)) { case 'b': return new Int8Value(Byte.parseByte(value)); @@ -53,9 +53,9 @@ public class RpcInvoker { "There is no jrt type identified by '" + parameter.charAt(0) + "'"); } - protected Request createRequest(String method,List<String> arguments) { - Request request=new Request(method); - if (arguments!=null) { + protected Request createRequest(String method, List<String> arguments) { + Request request = new Request(method); + if (arguments != null) { for (String argument : arguments) request.parameters().add(getArgument(request,argument)); } @@ -69,37 +69,37 @@ public class RpcInvoker { * @param method the name of the method to invoke * @param arguments the argument to the method, or null or an empty list if there are no arguments */ - public void invoke(String connectspec,String method, List<String> arguments) { - Supervisor supervisor=null; - Target target=null; + public void invoke(String connectspec, String method, List<String> arguments) { + Supervisor supervisor = null; + Target target = null; try { - if (connectspec.indexOf('/')<0) - connectspec="tcp/" + connectspec; + if (connectspec.indexOf('/') < 0) + connectspec = "tcp/" + connectspec; - supervisor=new Supervisor(new Transport()); + supervisor = new Supervisor(new Transport()); target = supervisor.connect(new Spec(connectspec)); - Request request=createRequest(method,arguments); + Request request = createRequest(method,arguments); target.invokeSync(request,10.0); if (request.isError()) { System.err.println("error(" + request.errorCode() + "): " + request.errorMessage()); return; } - Values returned=request.returnValues(); - for (int i=0; i<returned.size(); i++) { + Values returned = request.returnValues(); + for (int i = 0; i < returned.size(); i++) { System.out.println(returned.get(i)); } } finally { - if (target!=null) + if (target != null) target.close(); - if (supervisor!=null) + if (supervisor != null) supervisor.transport().shutdown().join(); } } public static void main(String[] args) { - if (args.length<1) { + if (args.length < 1) { System.err.println("usage: invoke [-h <connectspec>] <method> [arguments]"); System.err.println(" Connectspec: This is on the form hostname:port, or tcp/hostname:port"); System.err.println(" if omitted, localhost:8086 is used"); @@ -107,14 +107,14 @@ public class RpcInvoker { System.err.println(" supported types: {'b','h','i','l','f','d','s'}"); System.exit(0); } - List<String> arguments=new ArrayList<String>(Arrays.asList(args)); - String connectSpec="localhost:8086"; - if ("-h".equals(arguments.get(0)) && arguments.size()>=3) { + List<String> arguments = new ArrayList<String>(Arrays.asList(args)); + String connectSpec = "localhost:8086"; + if ("-h".equals(arguments.get(0)) && arguments.size() >= 3) { arguments.remove(0); // Consume -h - connectSpec=arguments.remove(0); + connectSpec = arguments.remove(0); } - String method=arguments.remove(0); - new RpcInvoker().invoke(connectSpec,method,arguments); + String method = arguments.remove(0); + new RpcInvoker().invoke(connectSpec, method, arguments); } } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java index 7ba061aaef1..ef1d6c966ac 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.opennlp; import com.google.common.base.Optional; diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java index ef3248ee0bb..5dd8d55a19e 100644 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.opennlp; import com.yahoo.language.Language; diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index fd06f886edb..9f55e218140 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms64m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000" +addopts="-server -Xms32m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java index 26021492df2..4d801cd6229 100644 --- a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java +++ b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java @@ -11,19 +11,18 @@ package com.yahoo.plugin; * @author Stig Bakken */ public class SystemPropertyConfig extends Config { + private final String prefix; /** - * @param prefix Prefix string prepended to config keys + * @param prefix prefix string prepended to config keys * as they are looked up as system properties. */ public SystemPropertyConfig(String prefix) { this.prefix = prefix; } - /** - * @return a config value for the specified key - */ + /** Returns a config value for the specified key */ public String get(String key, String defaultValue) { return System.getProperty(prefix + key, defaultValue); } @@ -31,4 +30,5 @@ public class SystemPropertyConfig extends Config { public String toString() { return "Prefix=" + prefix; } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java index 564de0806ca..99d7b9fe8bf 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.core; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 8d0074dca66..4c4015220bc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.core; import ai.vespa.metricsproxy.metric.ExternalMetrics; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index ab9f4f6c9c4..247a5ddeeef 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.core; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java index 617cf0a1525..16dddad76a6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.metricsproxy.core; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java index d3296331a6a..9bd30d287d4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java index a36126a3a24..aa82a921e1a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import com.yahoo.container.jdisc.HttpRequest; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java index 9f31e9356e8..9de5933bd1f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import com.yahoo.container.jdisc.HttpResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java index 86f1260207c..c4360545806 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import ai.vespa.metricsproxy.core.MetricsConsumers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/TextResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/TextResponse.java index 7dfc232d253..c8a8bed0333 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/TextResponse.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/TextResponse.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import com.yahoo.container.jdisc.HttpResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index b47665c2536..768c1beebef 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import ai.vespa.metricsproxy.core.MetricsConsumers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsException.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsException.java index 087959d4c73..bdfbc65882f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsException.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsException.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index 7d1ea3ecd2f..ce2e383f0d2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 7e6285954cc..c51970ce3ae 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index 7cc47b31104..07070027f96 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; @@ -16,7 +13,7 @@ import java.util.Objects; */ public class Node { - final String configId; + final String nodeId; final String host; final int port; final String path; @@ -24,15 +21,15 @@ public class Node { private final String metricsUriBase; public Node(MetricsNodesConfig.Node nodeConfig) { - this(nodeConfig.configId(), nodeConfig.hostname(), nodeConfig.metricsPort() , nodeConfig.metricsPath()); + this(nodeConfig.nodeId(), nodeConfig.hostname(), nodeConfig.metricsPort() , nodeConfig.metricsPath()); } - public Node(String configId, String host, int port, String path) { - Objects.requireNonNull(configId, "Null configId is not allowed"); + public Node(String nodeId, String host, int port, String path) { + Objects.requireNonNull(nodeId, "Null configId is not allowed"); Objects.requireNonNull(host, "Null host is not allowed"); Objects.requireNonNull(path, "Null path is not allowed"); - this.configId = configId; + this.nodeId = nodeId; this.host = host; this.port = port; this.path = path; @@ -40,7 +37,7 @@ public class Node { } public String getName() { - return configId; + return nodeId; } URI metricsUri(ConsumerId consumer) { @@ -53,12 +50,12 @@ public class Node { if (o == null || getClass() != o.getClass()) return false; Node node = (Node) o; return port == node.port && - configId.equals(node.configId) && + nodeId.equals(node.nodeId) && host.equals(node.host); } @Override public int hashCode() { - return Objects.hash(configId, host, port); + return Objects.hash(nodeId, host, port); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index 8b397bc65f8..f2ee326029a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java index 03b349c293b..5f3723df94d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.prometheus; import ai.vespa.metricsproxy.core.MetricsConsumers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java index 9eb1b242535..ae79e891e3d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.metric.model.ConsumerId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java index 017b2c57370..0a2dbc9a910 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.core.MetricsConsumers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java index 4961cc8b2a6..bda0a2537c8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.metric.model.StatusCode; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java index 59fbe301a49..a7ea70495f7 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.metric.model.ConsumerId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java index f1d029d8746..45d76375c07 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import java.util.ArrayList; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java index 8858e21486a..44b8c9f9d84 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.service.VespaService; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java index ae40f672a32..37644489322 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.dimensions; import ai.vespa.metricsproxy.metric.model.DimensionId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java index d2c1799e148..cad4b644903 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.dimensions; import ai.vespa.metricsproxy.metric.model.DimensionId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java index f4e5f74313a..d93a83a44e9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.metricsproxy.metric.dimensions; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java index 0d7acd5f354..62de9649bb0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import java.util.Objects; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java index 03f4c2c01ff..c99e54f2607 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import java.util.Objects; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java index c93735c7fca..9be67607d4d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import java.util.Objects; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 098fd48c8b3..f9cc17f94ea 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java index b61ead75b72..14f1a67bef3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import java.util.Objects; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java index 7f5a7d0e64b..0dbd63f271d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; /** diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java index b3ebd469b13..797e2e9d9fd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java index f1e58f83505..a1df38296ac 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java index c20ded30361..642081e3c8f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.http.application.Node; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java index 46f515bfb47..0c3009482c4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.DimensionId; @@ -10,11 +7,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.google.common.base.Functions; -import com.yahoo.stream.CustomCollectors; import java.util.Map; -import java.util.stream.Collectors; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; import static com.yahoo.stream.CustomCollectors.toLinkedMap; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java index 0cd7e7a2180..5b45ed6cdfc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java index f348bd4beca..73d9c43ce74 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.StatusCode; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java index d9df3e0c0e9..9d12c61291d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import com.fasterxml.jackson.core.JsonGenerator; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonRenderingException.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonRenderingException.java index 82f3c7491fb..ec618c5a04c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonRenderingException.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonRenderingException.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; /** diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java index 86249ba6e31..05021f20c82 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java index 5fdbe9577be..38038fda66a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java index 7c1539d1c55..9487b65a650 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.ConsumerId; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java index a87fa236f18..a3932bd0109 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.prometheus; import io.prometheus.client.Collector; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusRenderingException.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusRenderingException.java index 941810a1261..bdb93701aa4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusRenderingException.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusRenderingException.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.prometheus; /** diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java index d43baa9c9c9..f0894b39e28 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.prometheus; import ai.vespa.metricsproxy.metric.model.MetricsPacket; @@ -11,11 +8,9 @@ import io.prometheus.client.Collector.MetricFamilySamples; import io.prometheus.client.Collector.MetricFamilySamples.Sample; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.util.Collections.emptyList; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java index c72f2484f8c..860df519a59 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.metricsproxy.metric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index edb9fba5307..269b70c8316 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.rpc; import com.yahoo.component.AbstractComponent; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java index b43e7959eff..ba84347c7ea 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.rpc; import ai.vespa.metricsproxy.core.MetricsManager; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index 4372adf34dc..09fb66bffab 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import com.google.inject.Inject; @@ -10,7 +7,6 @@ import com.yahoo.log.LogLevel; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.PrintStream; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java index 312e7d5c0c1..481068f0df2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; class CpuJiffies { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java index c9bfc8b365c..f333aab8f1e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java index f21d125e279..b304e5d74d3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metrics; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 81358041502..922a2a15ffd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.util.http.VespaHttpClientBuilder; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 068a8faade8..ec36ae344bc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 552b4dc4010..442ebc0d38d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -1,7 +1,4 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 653c79ba578..379e5296bb8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java index 6494054c278..05f13051cc0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.core.MonitoringConfig; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java index c59daf65cc2..69558b0c474 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java index 0f95a698393..76a4ae5046b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.core.MonitoringConfig; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java index b478cdf8b5b..f187e255cb3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.metricsproxy.service; diff --git a/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def b/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def index 3f4165c08ec..0de2f21d972 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def +++ b/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def @@ -1,7 +1,7 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package=ai.vespa.metricsproxy.http.application -node[].configId string +node[].nodeId string node[].hostname string node[].metricsPort int node[].metricsPath string diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java index 5e8322c4c01..e067639023d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy; import ai.vespa.metricsproxy.core.MetricsConsumers; @@ -10,16 +7,11 @@ import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.service.VespaService; import ai.vespa.metricsproxy.service.VespaServices; import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.util.List; import java.util.stream.Collectors; /** diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index bc83712ac70..f21774aeb15 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.core; import ai.vespa.metricsproxy.TestUtil; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java index ad2e06da19c..ccf04560906 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import org.junit.Test; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java index bd6fba84e35..77c3a719cd9 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import ai.vespa.metricsproxy.TestUtil; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/MetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/MetricsHandlerTest.java index 83cd97bbeda..1444f5c57f5 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/MetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/MetricsHandlerTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index e3d929823a4..2d12930bd0a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.ConsumersConfig; @@ -174,7 +171,7 @@ public class ApplicationMetricsHandlerTest { private MetricsNodesConfig.Node.Builder nodeConfig(String path) { return new MetricsNodesConfig.Node.Builder() - .configId(path) + .nodeId(path) .hostname("localhost") .metricsPath(path) .metricsPort(port); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java index 48d421cf1a4..5ff6b580988 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -160,7 +157,7 @@ public class ApplicationMetricsRetrieverTest { private MetricsNodesConfig.Node.Builder nodeConfig(String path) { return new MetricsNodesConfig.Node.Builder() - .configId(path) + .nodeId(path) .hostname(HOST) .metricsPath(path) .metricsPort(port); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java index 0c0b0749011..0c800c0ff63 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.http.MetricsHandler; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index 69cdd9954e4..a85f0425b4b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.prometheus; import ai.vespa.metricsproxy.http.HttpHandlerTestBase; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java index 2cce2f66039..9c0a9f0b3d2 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.core.ConsumersConfig; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java index 91eda8f744c..84f40592f46 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; import ai.vespa.metricsproxy.metric.model.StatusCode; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java index d522a56a9ac..9b37a805245 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java index 8a66f98f42a..a4d03703476 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; import org.junit.Test; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java index eac63998402..47d91fb57a3 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java @@ -1,12 +1,8 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.http.application.Node; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; import ai.vespa.metricsproxy.metric.model.StatusCode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java index f56f3531982..4763c9d3495 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java index a73d15e8be3..b20217cdca0 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java index b5b5ba9eaa5..cdef510e73b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java index 316e67c13d5..e2ad0ccd504 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.node; import ai.vespa.metricsproxy.metric.model.MetricId; @@ -44,4 +45,4 @@ public class NodeMetricGathererTest { jsonObject.put("metrics", metrics); return jsonObject; } -}
\ No newline at end of file +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java index 997c02f7768..7275cb737c4 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.rpc; import ai.vespa.metricsproxy.core.ConsumersConfig; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java index 55e14381183..ac5fae9a727 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.rpc; import ai.vespa.metricsproxy.metric.HealthMetric; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index b0b8bcc614c..a363247ff52 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.rpc; import ai.vespa.metricsproxy.metric.Metric; @@ -17,9 +14,7 @@ import com.yahoo.jrt.Transport; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.List; @@ -38,7 +33,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author jobergum diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java index bd61b8443aa..8c6c5ffc3f8 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import org.junit.Test; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java index 108f5c18e1d..5c746521c1f 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; /** diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 5a174412729..0d53f988ac7 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java index f78c496fcd1..d39cabcbf93 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java index cf559628490..7b36e1d4b8b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java index ce33ce9f7bf..be57b24d92f 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.TestUtil; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java index 917c529e63e..6b439093682 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java @@ -1,6 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import com.yahoo.log.LogLevel; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java index f7802fd04fb..68beb7a5f41 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import com.sun.net.httpserver.HttpExchange; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java index c5f2ea6960b..86e67309e5a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import org.junit.Test; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java index ac839e595f9..9400d74fa3a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metrics; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java index bd0b670ca35..90ad9baf7fc 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java @@ -1,7 +1,4 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; import com.google.common.collect.ImmutableList; diff --git a/metrics/pom.xml b/metrics/pom.xml index b99ae535719..4121467985c 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -33,12 +33,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.4</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_core</artifactId> <version>${project.version}</version> diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/MlModelImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/MlModelImporter.java index d294872113b..bb09bd9b928 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/MlModelImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/MlModelImporter.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer.configmodelview; import java.io.File; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/package-info.java index 3258426dac4..1f87a5d1fe7 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/package-info.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/package-info.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * The config models view of imported models. This API cannot be changed without taking earlier config models * into account, not even on major versions. diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/package-info.java index 9599cf8627c..6b76a9305b9 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/package-info.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/package-info.java @@ -1,5 +1,6 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.rankingexpression.importer.onnx; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/package-info.java index 45ac2b16e97..0d77862cc83 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/package-info.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/package-info.java @@ -1 +1,2 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/package-info.java index 0840e584d25..8bb77d9c1aa 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/package-info.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/package-info.java @@ -1,4 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.rankingexpression.importer.tensorflow; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/package-info.java index d310de9041b..2dad2affa2e 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/package-info.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/package-info.java @@ -1,4 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package ai.vespa.rankingexpression.importer.xgboost; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/main/java/org/tensorflow/package-info.java b/model-integration/src/main/java/org/tensorflow/package-info.java index cc6335f0d38..237cb36a77a 100644 --- a/model-integration/src/main/java/org/tensorflow/package-info.java +++ b/model-integration/src/main/java/org/tensorflow/package-info.java @@ -1,6 +1,7 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage @PublicApi package org.tensorflow; import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportEvaluationTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportEvaluationTestCase.java index 4c2d72dfb21..d217ec84afd 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportEvaluationTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportEvaluationTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer.xgboost; import com.yahoo.searchlib.rankingexpression.RankingExpression; diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportTestCase.java index 9a71905aa65..181656f5c6b 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostImportTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer.xgboost; import com.yahoo.searchlib.rankingexpression.RankingExpression; diff --git a/model-integration/src/test/models/onnx/simple/simple.py b/model-integration/src/test/models/onnx/simple/simple.py index 4471ed812b8..878cf1c928e 100755 --- a/model-integration/src/test/models/onnx/simple/simple.py +++ b/model-integration/src/test/models/onnx/simple/simple.py @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import onnx from onnx import helper, TensorProto diff --git a/model-integration/src/test/models/pytorch/pytorch_test.py b/model-integration/src/test/models/pytorch/pytorch_test.py index d2adb6c8974..249b77ad8c7 100755 --- a/model-integration/src/test/models/pytorch/pytorch_test.py +++ b/model-integration/src/test/models/pytorch/pytorch_test.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import torch diff --git a/model-integration/src/test/models/tensorflow/external/train.py b/model-integration/src/test/models/tensorflow/external/train.py index b015f1ff5af..32b14c481c3 100644 --- a/model-integration/src/test/models/tensorflow/external/train.py +++ b/model-integration/src/test/models/tensorflow/external/train.py @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import numpy as np import tensorflow as tf import tensorflow.keras.backend as K diff --git a/model-integration/src/test/models/tensorflow/external/train_embed.py b/model-integration/src/test/models/tensorflow/external/train_embed.py index 7aae61b49a7..2d1ab18a0eb 100644 --- a/model-integration/src/test/models/tensorflow/external/train_embed.py +++ b/model-integration/src/test/models/tensorflow/external/train_embed.py @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import numpy as np import tensorflow as tf import tensorflow.keras.backend as K diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java index e0b761ec816..368b30ffa9a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index ecf451b2555..821765fea20 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.HostName; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 0d3b02c2bd8..02e01264a57 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.CloudName; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java index efb56be56c5..7cc0966337f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java index be94846fb65..e84d8345815 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.vespa.test.file.TestFileSystem; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java index 29b012c1397..f12faf936f1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -23,4 +24,4 @@ public class FileDeleterTest { assertTrue(deleter.converge(context)); assertFalse(deleter.converge(context)); } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index b5068892527..37987f0512d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -30,7 +30,7 @@ public class NodeList implements Iterable<Node> { private final List<Node> nodes; private final boolean negate; - public NodeList(List<Node> nodes) { + NodeList(List<Node> nodes) { this(nodes, true, false); } @@ -62,6 +62,11 @@ public class NodeList implements Iterable<Node> { return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().equals(type)); } + /** Returns the subset of nodes that run containers */ + public NodeList container() { + return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().isContainer()); + } + /** Returns the subset of nodes that are currently changing their Vespa version */ public NodeList changingVersion() { return filter(node -> node.status().vespaVersion().isPresent() && @@ -166,4 +171,8 @@ public class NodeList implements Iterable<Node> { return new NodeList(nodes, false, false); } + public static NodeList copyOf(List<Node> nodes) { + return new NodeList(nodes, true, false); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 682d1419a5c..1c9e1445447 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -183,7 +183,7 @@ public class NodeRepository extends AbstractComponent { /** Returns a filterable list of all nodes in this repository */ public NodeList list() { - return new NodeList(getNodes()); + return NodeList.copyOf(getNodes()); } /** Returns a locked list of all nodes in this repository */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 65fab51932a..ca8399da629 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeResources; @@ -281,7 +282,7 @@ public class CapacityChecker { if (node.allocation().isEmpty()) return false; Allocation nodeAllocation = node.allocation().get(); for (var allocation : containedAllocations.get(host)) { - if (allocation.membership().cluster().equalsIgnoringGroupAndVespaVersion(nodeAllocation.membership().cluster()) + if (allocation.membership().cluster().satisfies(nodeAllocation.membership().cluster()) && allocation.owner().equals(nodeAllocation.owner())) { return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index 6a6fbf3aed0..d08811e9255 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 67e0800a3f0..db79e3164fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -389,7 +389,7 @@ public class NodeFailer extends Maintainer { List<Node> nodes = nodeRepository().getNodes(); NodeList recentlyFailedNodes = nodes.stream() .filter(n -> n.history().hasEventAfter(History.Event.Type.failed, startOfThrottleWindow)) - .collect(collectingAndThen(Collectors.toList(), NodeList::new)); + .collect(collectingAndThen(Collectors.toList(), NodeList::copyOf)); // Allow failing nodes within policy if (recentlyFailedNodes.size() < throttlePolicy.allowedToFailOf(nodes.size())) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java index b1a4b1d7aa9..86d4044b373 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.vespa.curator.Curator; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 0364264bd5b..816396fe9b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import java.util.EnumSet; import java.util.Locale; /** @@ -106,10 +105,14 @@ public class CapacityPolicies { private int ensureRedundancy(int nodeCount, ClusterSpec.Type clusterType, boolean canFail) { if (canFail && nodeCount == 1 && - EnumSet.of(ClusterSpec.Type.container, ClusterSpec.Type.content).contains(clusterType) && + requiresRedundancy(clusterType) && zone.environment().isProduction()) throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy"); return nodeCount; } + private static boolean requiresRedundancy(ClusterSpec.Type clusterType) { + return clusterType.isContent() || clusterType.isContainer(); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index bd28ec59da2..9a06f2a980a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.base.Supplier; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java index 979deb6a8cc..25fcfecc9b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.base.Supplier; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index ee786ea414e..bdd3c31fa38 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -71,7 +71,7 @@ public class LoadBalancerProvisioner { */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type - if (cluster.type() != ClusterSpec.Type.container) return; // Nothing to provision for this cluster type + if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type try (var loadBalancersLock = db.lockLoadBalancers()) { provision(application, cluster.id(), false, loadBalancersLock); } @@ -179,12 +179,12 @@ public class LoadBalancerProvisioner { /** Returns a list of active and reserved nodes of type container in given cluster */ private List<Node> allocatedContainers(ApplicationId application, ClusterSpec.Id clusterId) { - return new NodeList(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) - .owner(application) - .filter(node -> node.state().isAllocated()) - .type(ClusterSpec.Type.container) - .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterId)) - .asList(); + return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) + .owner(application) + .filter(node -> node.state().isAllocated()) + .container() + .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterId)) + .asList(); } /** Find IP addresses reachable by the load balancer service */ @@ -204,7 +204,7 @@ public class LoadBalancerProvisioner { private static Set<ClusterSpec.Id> containerClusterOf(Set<ClusterSpec> clusters) { return clusters.stream() - .filter(c -> c.type() == ClusterSpec.Type.container) + .filter(c -> c.type().isContainer()) .map(ClusterSpec::id) .collect(Collectors.toUnmodifiableSet()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 1c3cd40bc02..a57b9d359f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -102,7 +102,7 @@ class NodeAllocation { if (offered.allocation().isPresent()) { ClusterMembership membership = offered.allocation().get().membership(); if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application - if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type + if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type if ((! node.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) @@ -198,23 +198,24 @@ class NodeAllocation { * Returns whether this node should be accepted into the cluster even if it is not currently desired * (already enough nodes, or wrong flavor). * Such nodes will be marked retired during finalization of the list of accepted nodes. - * The conditions for this are - * <ul> - * <li>This is a content node. These must always be retired before being removed to allow the cluster to + * The conditions for this are: + * + * This is a content or combined node. These must always be retired before being removed to allow the cluster to * migrate away data. - * <li>This is a container node and it is not desired due to having the wrong flavor. In this case this + * + * This is a container node and it is not desired due to having the wrong flavor. In this case this * will (normally) obtain for all the current nodes in the cluster and so retiring before removing must * be used to avoid removing all the current nodes at once, before the newly allocated replacements are * initialized. (In the other case, where a container node is not desired because we have enough nodes we * do want to remove it immediately to get immediate feedback on how the size reduction works out.) - * </ul> */ private boolean acceptToRetire(Node node) { if (node.state() != Node.State.active) return false; if (! node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; - return (cluster.type() == ClusterSpec.Type.content) || - (cluster.type() == ClusterSpec.Type.container && ! hasCompatibleFlavor(node)); + return cluster.type() == ClusterSpec.Type.content || + cluster.type() == ClusterSpec.Type.combined || + (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(node)); } private boolean hasCompatibleFlavor(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java index 5402c22498a..488b11e627b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java index 831520cf383..e7bf76986ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java index 132ab40b3de..9d79af804ce 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index b2bc21d1cf1..da174a8d38c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.fasterxml.jackson.annotation.JsonGetter; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java index e305e2b3c87..c19b54a8cab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.HostName; @@ -29,4 +30,4 @@ public class MaintainerTest { assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval)); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index cd139a1be19..b1c3b23016c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -287,7 +287,7 @@ public class PeriodicApplicationMaintainerTest { } NodeList getNodes(Node.State ... states) { - return new NodeList(nodeRepository.getNodes(NodeType.tenant, states)); + return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, states)); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 9cd95e401a9..7e7338ab9ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -66,7 +66,7 @@ public class AllocationSimulator { initialNodes.add(host("host9", flavors.getFlavorOrThrow("host-small"))); initialNodes.add(host("host10", flavors.getFlavorOrThrow("host-small"))); initialNodes.add(node("node1", flavors.getFlavorOrThrow("d-2"), Optional.of("host1"), Optional.of("test"))); - nodes = new NodeList(initialNodes); + nodes = NodeList.copyOf(initialNodes); visualizer.addStep(nodes.asList(), "Initial state", ""); } @@ -117,7 +117,7 @@ public class AllocationSimulator { public void addCluster(String task, int count, Flavor flavor, String id) { // TODO: Implement - nodes = new NodeList(nodes.asList()); + nodes = NodeList.copyOf(nodes.asList()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java index 23b2a93a67d..fd2374c55b7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -56,7 +56,7 @@ public class AllocationVisualizer extends JPanel { public void addStep(List<Node> nodes, String task, String message) { - steps.add(new AllocationSnapshot(new NodeList(nodes), task, message)); + steps.add(new AllocationSnapshot(NodeList.copyOf(nodes), task, message)); } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 618df82e38b..9bccfb74334 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -436,7 +436,7 @@ public class DynamicDockerAllocationTest { private List<Node> findSpareCapacity(ProvisioningTester tester) { List<Node> nodes = tester.nodeRepository().getNodes(Node.State.values()); - NodeList nl = new NodeList(nodes); + NodeList nl = NodeList.copyOf(nodes); return nodes.stream() .filter(n -> n.type() == NodeType.host) .filter(n -> nl.childrenOf(n).size() == 0) // Nodes without children diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 9c42d0d95a1..6dfca4d2c04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -200,6 +200,18 @@ public class LoadBalancerProvisionerTest { assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList()); } + @Test + public void provision_load_balancer_combined_cluster() { + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); + ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); + + var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster)); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); + tester.activate(app1, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java index 725842db819..b4618efd371 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; @@ -146,4 +147,4 @@ public class PrioritizableNodeTest { ", without: " + node.skewWithoutThis()); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 93802d0d4b2..f2c9c409fdb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -509,8 +509,8 @@ public class ProvisioningTest { new com.yahoo.component.Version(4, 5, 6), false); tester.activate(application, tester.prepare(application, cluster, capacity, 1)); - assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); - assertEquals(0, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); + assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); + assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); @@ -518,16 +518,16 @@ public class ProvisioningTest { tester.activate(application, tester.prepare(application, cluster, capacityFORCED, 1)); // Nodes are not retired since that is unsafe when we cannot fail - assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); - assertEquals(0, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); + assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); + assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // ... but we still want to tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> assertTrue(node.status().wantToRetire())); // redeploy with allowing failing tester.activate(application, tester.prepare(application, cluster, capacity, 1)); // ... old nodes are now retired - assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); - assertEquals(5, new NodeList(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); + assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); + assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); } @Test @@ -651,6 +651,35 @@ public class ProvisioningTest { tester.activate(application, state.allHosts); } + @Test + public void change_to_combined_cluster_does_not_change_node_allocation() { + var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + var application = tester.makeApplicationId(); + + tester.makeReadyNodes(4, defaultResources); + + // Application allocates two content nodes initially. This is the old behaviour where combined clusters has type + // content + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, + ClusterSpec.Id.from("combined"), + Version.fromString("1.2.3"), + false); + var initialNodes = tester.activate(application, tester.prepare(application, cluster, + Capacity.fromCount(2, defaultResources, false, false), + 1)); + + // Application is redeployed with cluster type combined + cluster = ClusterSpec.request(ClusterSpec.Type.combined, + ClusterSpec.Id.from("combined"), + Version.fromString("1.2.3"), + false); + var newNodes = tester.activate(application, tester.prepare(application, cluster, + Capacity.fromCount(2, defaultResources, false, false), + 1)); + + assertEquals("Node allocation remains the same", initialNodes, newNodes); + } + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, container0Size, container1Size, content0Size, content1Size, flavor, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 242b0d8ca9e..8be0da7c83d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -118,7 +118,7 @@ public class ProvisioningTester { public NodeRepositoryProvisioner provisioner() { return provisioner; } public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } public CapacityPolicies capacityPolicies() { return capacityPolicies; } - public NodeList getNodes(ApplicationId id, Node.State ... inState) { return new NodeList(nodeRepository.getNodes(id, inState)); } + public NodeList getNodes(ApplicationId id, Node.State ... inState) { return NodeList.copyOf(nodeRepository.getNodes(id, inState)); } public void patchNode(Node node) { nodeRepository.write(node, () -> {}); } @@ -207,7 +207,7 @@ public class ProvisioningTester { Set<Integer> indices = new HashSet<>(); for (HostSpec host : hosts) { ClusterSpec nodeCluster = host.membership().get().cluster(); - assertTrue(requestedCluster.equalsIgnoringGroupAndVespaVersion(nodeCluster)); + assertTrue(requestedCluster.satisfies(nodeCluster)); if (requestedCluster.group().isPresent()) assertEquals(requestedCluster.group(), nodeCluster.group()); else diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java index 22f2440b408..42ba61f3742 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; import com.google.common.util.concurrent.UncheckedTimeoutException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java index 042eca0040e..5ebad190f55 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java index 63b8e498f16..30a914bffa9 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -88,4 +89,4 @@ public class ClusterControllerClientTimeoutsTest { clock.advance(originalTimeout.minus(MINIMUM_TIME_LEFT)); timeouts.getServerTimeoutOrThrow(); } -}
\ No newline at end of file +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java index fcb056e3d26..95fdd61563b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; import com.yahoo.test.ManualClock; diff --git a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java index be871031101..d10892cd44f 100644 --- a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java +++ b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java @@ -4,7 +4,6 @@ package com.yahoo.search.predicate; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import org.apache.commons.lang.ArrayUtils; import java.io.IOException; import java.util.Arrays; @@ -112,7 +111,7 @@ public class PredicateQueryParser { private static void skipToken(JsonParser parser, JsonToken... expected) throws IOException { JsonToken actual = parser.nextToken(); - if (!ArrayUtils.contains(expected, actual)) { + if (Arrays.stream(expected).noneMatch(e -> e.equals(actual))) { throw new IllegalArgumentException( String.format("Expected a token in %s, got %s (%s).", Arrays.toString(expected), actual, parser.getTokenLocation())); diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java index a23339aa532..982d2488105 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.utils; +import com.google.common.html.HtmlEscapers; import com.yahoo.document.predicate.Predicate; -import org.apache.commons.lang.StringEscapeUtils; import java.io.BufferedWriter; import java.io.IOException; @@ -35,7 +35,7 @@ public class VespaFeedWriter extends BufferedWriter { try { this.append(String.format("<document documenttype=\"%2$s\" documentid=\"id:%1$s:%2$s::%3$d\">\n", namespace, documentType, id)); - this.append("<" + fieldName + ">" + StringEscapeUtils.escapeHtml(predicate.toString()) + "</" + fieldName + ">\n"); + this.append("<" + fieldName + ">" + HtmlEscapers.htmlEscaper().escape(predicate.toString()) + "</" + fieldName + ">\n"); this.append("</document>\n"); } catch (IOException e) { throw new RuntimeException(e); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java index 5d7e668a0de..24aa59955bc 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java @@ -9,9 +9,8 @@ import com.yahoo.document.predicate.PredicateHash; import com.yahoo.document.predicate.RangeEdgePartition; import com.yahoo.document.predicate.RangePartition; import com.yahoo.search.predicate.index.Feature; -import com.yahoo.search.predicate.index.conjunction.IndexableFeatureConjunction; import com.yahoo.search.predicate.index.IntervalWithBounds; -import org.apache.commons.lang.ArrayUtils; +import com.yahoo.search.predicate.index.conjunction.IndexableFeatureConjunction; import org.junit.Test; import java.util.Arrays; @@ -232,7 +231,7 @@ public class PredicateTreeAnnotatorTest { long hash = PredicateHash.hash64(feature); List<Integer> actualIntervals = r.intervalMap.get(hash); assertNotNull(actualIntervals); - assertArrayEquals(ArrayUtils.toPrimitive(expectedIntervals), Ints.toArray(actualIntervals)); + assertArrayEquals(Ints.toArray(Arrays.asList(expectedIntervals)), Ints.toArray(actualIntervals)); } private static void assertBoundsContains(PredicateTreeAnnotations r, String feature, IntervalWithBounds expectedBounds) { diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java index 31777959704..13a15cab52a 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.index; -import org.apache.commons.lang.ArrayUtils; +import com.google.common.primitives.Ints; import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.junit.Test; @@ -109,7 +109,7 @@ public class CachedPostingListCounterTest { private static PostingList postingList(Integer... docIds) { PostingList postingList = mock(PostingList.class); - when(postingList.getDocIds()).thenReturn(ArrayUtils.toPrimitive(docIds)); + when(postingList.getDocIds()).thenReturn(Ints.toArray(Arrays.asList((docIds)))); return postingList; } diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 2d75bb8906b..14b72c9d8f8 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -749,7 +749,7 @@ TEST_F("require that attribute writer spreads write over 2 write contexts", Fixt TEST_DO(putAttributes(f, {0, 1})); } -TEST_F("require that attribute writer spreads write over 3 write contexts", Fixture(3)) +TEST_F("require that attribute writer spreads write over 3 write contexts", Fixture(8)) { TEST_DO(putAttributes(f, {0, 1, 2})); } diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index a953d651a33..d007b030e6b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -9,15 +9,13 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/iattributemanager.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.cachedselect"); - namespace proton { using search::AttributeVector; using search::AttributeGuard; using document::select::FieldValueNode; using search::attribute::CollectionType; +using search::attribute::BasicType; using NodeUP = std::unique_ptr<document::select::Node>; @@ -42,7 +40,7 @@ public: } AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::AttributeVectors &attributes); - ~AttrVisitor(); + ~AttrVisitor() override; /* * Mutate field value nodes representing single value attributes into @@ -62,7 +60,11 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At _complexAttrs(0u) {} -AttrVisitor::~AttrVisitor() { } +AttrVisitor::~AttrVisitor() = default; + +bool isSingleValueThatWEHandle(BasicType type) { + return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE); +} void AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) @@ -83,20 +85,26 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) } std::shared_ptr<search::AttributeVector> av(ag->getSP()); if (av->getCollectionType() == CollectionType::SINGLE) { - ++_svAttrs; - AttrMap::iterator it(_amap.find(name)); - uint32_t idx(invalidIdx()); - if (it == _amap.end()) { - // Allocate new location for guard - idx = _attributes.size(); - _amap[name] = idx; - _attributes.push_back(av); + if (isSingleValueThatWEHandle(av->getBasicType())) { + ++_svAttrs; + auto it(_amap.find(name)); + uint32_t idx(invalidIdx()); + if (it == _amap.end()) { + // Allocate new location for guard + idx = _attributes.size(); + _amap[name] = idx; + _attributes.push_back(av); + } else { + // Already allocated location for guard + idx = it->second; + } + assert(idx != invalidIdx()); + _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, av); } else { - // Already allocated location for guard - idx = it->second; + ++_complexAttrs; + // Don't try to optimize predicate/tensor/reference attributes yet. + _valueNode = expr.clone(); } - assert(idx != invalidIdx()); - _valueNode.reset(new AttributeFieldValueNode(expr.getDocType(), name, av)); } else { // Don't try to optimize multivalue attribute vectors yet ++_mvAttrs; @@ -186,7 +194,7 @@ CachedSelect::CachedSelect() _preDocSelect() { } -CachedSelect::~CachedSelect() { } +CachedSelect::~CachedSelect() = default; void CachedSelect::set(const vespalib::string &selection, diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 54942b4ba11..6397195da93 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -464,7 +464,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) [upd = updOp.getUpdate(), serialNum, lid, onWriteDone, promisedDoc = std::move(promisedDoc), promisedStream = std::move(promisedStream), this]() mutable { - makeUpdatedDocument(serialNum, lid, upd, onWriteDone, + makeUpdatedDocument(serialNum, lid, *upd, onWriteDone, std::move(promisedDoc), std::move(promisedStream)); }); #pragma GCC diagnostic pop @@ -473,12 +473,11 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) } void -StoreOnlyFeedView::makeUpdatedDocument(SerialNum serialNum, Lid lid, DocumentUpdate::SP update, +StoreOnlyFeedView::makeUpdatedDocument(SerialNum serialNum, Lid lid, const DocumentUpdate & update, OnOperationDoneType onWriteDone, PromisedDoc promisedDoc, PromisedStream promisedStream) { Document::UP prevDoc = _summaryAdapter->get(lid, *_repo); - const DocumentUpdate & upd = *update; Document::UP newDoc; vespalib::nbostream newStream(12345); assert(!onWriteDone->hasToken() || useDocumentStore(serialNum)); @@ -492,10 +491,10 @@ StoreOnlyFeedView::makeUpdatedDocument(SerialNum serialNum, Lid lid, DocumentUpd // also check that this operation is marked for ignore by index // proxy. } else { - if (upd.getId() == prevDoc->getId()) { + if (update.getId() == prevDoc->getId()) { newDoc = std::move(prevDoc); if (useDocumentStore(serialNum)) { - upd.applyTo(*newDoc); + update.applyTo(*newDoc); newDoc->serialize(newStream); } } else { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 18b78642814..4c1d50232dd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -189,7 +189,7 @@ private: // Ack token early if visibility delay is nonzero void considerEarlyAck(FeedToken &token); - void makeUpdatedDocument(SerialNum serialNum, Lid lid, DocumentUpdateSP upd, OnOperationDoneType onWriteDone, + void makeUpdatedDocument(SerialNum serialNum, Lid lid, const DocumentUpdate & update, OnOperationDoneType onWriteDone, PromisedDoc promisedDoc, PromisedStream promisedStream); protected: diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index ff6c1ad0b9d..79752c5feaf 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -878,6 +878,7 @@ "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorReduce()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorReduceComposites()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorJoin()", + "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorMerge()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorRename()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorConcat()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorGenerate()", @@ -1026,6 +1027,7 @@ "public static final int MAP", "public static final int REDUCE", "public static final int JOIN", + "public static final int MERGE", "public static final int RENAME", "public static final int CONCAT", "public static final int TENSOR", diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index 3a3410aeebb..2a6e6793bcd 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -11,10 +11,13 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; +import java.util.stream.Collectors; /** * A free, parametrized function @@ -27,7 +30,12 @@ public class LambdaFunctionNode extends CompositeNode { private final ExpressionNode functionExpression; public LambdaFunctionNode(List<String> arguments, ExpressionNode functionExpression) { - // TODO: Verify that the function only accesses the given arguments + if ( ! arguments.containsAll(featuresAccessedIn(functionExpression))) { + throw new IllegalArgumentException("Lambda " + functionExpression + " accesses features outside its scope: " + + featuresAccessedIn(functionExpression).stream() + .filter(f -> ! arguments.contains(f)) + .collect(Collectors.joining(", "))); + } this.arguments = ImmutableList.copyOf(arguments); this.functionExpression = functionExpression; } @@ -134,6 +142,22 @@ public class LambdaFunctionNode extends CompositeNode { }); } + private static Set<String> featuresAccessedIn(ExpressionNode node) { + if (node instanceof ReferenceNode) { + return Set.of(((ReferenceNode) node).reference().toString()); + } + else if (node instanceof NameNode) { // (This clause probably not necessary) + return Set.of(((NameNode) node).getValue()); + } + else if (node instanceof CompositeNode) { + Set<String> features = new HashSet<>(); + ((CompositeNode)node).children().forEach(child -> features.addAll(featuresAccessedIn(child))); + return features; + } + return Set.of(); + } + + private class DoubleUnaryLambda implements DoubleUnaryOperator { @Override diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index c3d597fca46..6fa915a134d 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -126,6 +126,7 @@ TOKEN : <MAP: "map"> | <REDUCE: "reduce"> | <JOIN: "join"> | + <MERGE: "merge"> | <RENAME: "rename"> | <CONCAT: "concat"> | <TENSOR: "tensor"> | @@ -384,6 +385,7 @@ TensorFunctionNode tensorFunction() : tensorExpression = tensorReduce() | tensorExpression = tensorReduceComposites() | tensorExpression = tensorJoin() | + tensorExpression = tensorMerge() | tensorExpression = tensorRename() | tensorExpression = tensorConcat() | tensorExpression = tensorGenerate() | @@ -447,6 +449,18 @@ TensorFunctionNode tensorJoin() : doubleJoiner.asDoubleBinaryOperator())); } } +TensorFunctionNode tensorMerge() : +{ + ExpressionNode tensor1, tensor2; + LambdaFunctionNode doubleMerger; +} +{ + <MERGE> <LBRACE> tensor1 = expression() <COMMA> tensor2 = expression() <COMMA> doubleMerger = lambdaFunction() <RBRACE> + { return new TensorFunctionNode(new Merge(TensorFunctionNode.wrap(tensor1), + TensorFunctionNode.wrap(tensor2), + doubleMerger.asDoubleBinaryOperator())); } +} + TensorFunctionNode tensorRename() : { ExpressionNode tensor; @@ -671,6 +685,7 @@ String tensorFunctionName() : ( <MAP> { return token.image; } ) | ( <REDUCE> { return token.image; } ) | ( <JOIN> { return token.image; } ) | + ( <MERGE> { return token.image; } ) | ( <RENAME> { return token.image; } ) | ( <CONCAT> { return token.image; } ) | ( <TENSOR> { return token.image; } ) | diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index ca2f6c6bbec..6a87e0c6d46 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -299,6 +299,11 @@ public class EvaluationTestCase { "tensor0 in [1,2,3]", "{ {x:0}:3, {x:1}:7 }"); tester.assertEvaluates("{ {x:0}:0.1 }", "join(tensor0, 0.1, f(x,y) (x*y))", "{ {x:0}:1 }"); + // tensor merge + tester.assertEvaluates("{ {x:0}:15, {x:1}:4 }", "merge(tensor0, tensor1, f(x,y) (x*y))", "{ {x:0}:3 }", "{ {x:0}:5, {x:1}:4 }"); + // -- join composites + tester.assertEvaluates("{ }", "merge(tensor0, tensor1, f(x,y) (x*y))", "{}"); + // TODO // argmax // argmin @@ -454,6 +459,22 @@ public class EvaluationTestCase { } @Test + public void testLambdaValidation() { + EvaluationTester tester = new EvaluationTester(); + try { + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:2, {d1:2 }:3 }", + "map(tensor0, f(x) (log10(x+sum(tensor0)))", "{ {d1:0}:10, {d1:1}:100, {d1:2}:1000 }"); + fail("Expected validation failure"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("Lambda log10(x + reduce(tensor0, sum)) accesses features outside its scope: tensor0", + e.getMessage()); + } + + } + + @Test public void testExpand() { EvaluationTester tester = new EvaluationTester(); // Add a dimension using a literal tensor diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java index 2cfc8cb575e..50f37486b90 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java @@ -26,7 +26,6 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[3]", "d0[3]"); assertWillOptimize("d0[1]", "d0[1]", "d0"); assertWillOptimize("d0[2]", "d0[2]", "d0"); - assertWillOptimize("d0[1]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3],d1[2]", "d0"); assertWillOptimize("d0[3],d1[2]", "d0[3]", "d0"); @@ -34,10 +33,10 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[2],d1[3]", "d1[3]", "d1"); assertWillOptimize("d0[2],d2[2]", "d1[3],d2[2]", "d2"); assertWillOptimize("d1[2],d2[2]", "d0[3],d2[2]", "d2"); - assertWillOptimize("d0[1],d2[2]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[2],d2[2]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]"); - assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]", "d0,d1"); + assertWillOptimize("d0[1],d2[4]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[2],d2[4]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]"); + assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]", "d0,d1"); assertWillOptimize("d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d1[2],d2[3]", "d2[3],d3[4],d4[5]", "d2"); diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 77caf535405..fcc7fd7300d 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -19,10 +19,7 @@ class Fixture public: SequencedTaskExecutor _threads; - Fixture() - : _threads(2) - { - } + Fixture() : _threads(2) { } }; @@ -101,7 +98,7 @@ TEST_F("require that task with different component ids are not serialized", Fixt std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); f._threads.execute(0, [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(1, [=]() { tv->modify(14, 42); }); + f._threads.execute(2, [=]() { tv->modify(14, 42); }); tv->wait(2); if (tv->_fail != 1) { continue; @@ -175,12 +172,13 @@ vespalib::string makeAltComponentId(Fixture &f) TEST_F("require that task with different string component ids are not serialized", Fixture) { - int tryCnt = detectSerializeFailure(f, "1", 100); + int tryCnt = detectSerializeFailure(f, "2", 100); EXPECT_TRUE(tryCnt < 100); } -TEST_F("require that task with different string component ids mapping to the same executor id are serialized", Fixture) +TEST_F("require that task with different string component ids mapping to the same executor id are serialized", + Fixture) { vespalib::string altComponentId = makeAltComponentId(f); LOG(info, "second string component id is \"%s\"", altComponentId.c_str()); @@ -236,6 +234,17 @@ TEST("require that you get correct number of executors") { EXPECT_EQUAL(7u, seven.getNumExecutors()); } +TEST("require that you distribute well") { + SequencedTaskExecutor seven(7); + EXPECT_EQUAL(7u, seven.getNumExecutors()); + EXPECT_EQUAL(97u, seven.getComponentHashSize()); + EXPECT_EQUAL(0u, seven.getComponentEffectiveHashSize()); + for (uint32_t id=0; id < 1000; id++) { + EXPECT_EQUAL((id%97)%7, seven.getExecutorId(id).getId()); + } + EXPECT_EQUAL(97u, seven.getComponentHashSize()); + EXPECT_EQUAL(97u, seven.getComponentEffectiveHashSize()); +} } diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 4f0b241e98f..2ee722902c8 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_add_library(searchlib_common OBJECT gatecallback.cpp growablebitvector.cpp indexmetainfo.cpp + isequencedtaskexecutor.cpp location.cpp locationiterators.cpp mapnames.cpp diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.cpp new file mode 100644 index 00000000000..9d1bc99bebb --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.cpp @@ -0,0 +1,45 @@ +// Copyright 2020 Oath inc.. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "isequencedtaskexecutor.h" +#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/hashtable.h> +#include <cassert> + +namespace search { + namespace { + constexpr uint8_t MAGIC = 255; + } + +ISequencedTaskExecutor::ISequencedTaskExecutor(uint32_t numExecutors) + : _component2Id(vespalib::hashtable_base::getModuloStl(numExecutors*8), MAGIC), + _mutex(), + _numExecutors(numExecutors), + _nextId(0) +{ + assert(numExecutors < 256); +} + +ISequencedTaskExecutor::~ISequencedTaskExecutor() = default; + +ISequencedTaskExecutor::ExecutorId +ISequencedTaskExecutor::getExecutorId(vespalib::stringref componentId) const { + vespalib::hash<vespalib::stringref> hashfun; + return getExecutorId(hashfun(componentId)); +} + +ISequencedTaskExecutor::ExecutorId +ISequencedTaskExecutor::getExecutorId(uint64_t componentId) const { + uint32_t shrunkId = componentId % _component2Id.size(); + uint8_t executorId = _component2Id[shrunkId]; + if (executorId == MAGIC) { + std::lock_guard guard(_mutex); + if (_component2Id[shrunkId] == MAGIC) { + _component2Id[shrunkId] = _nextId % getNumExecutors(); + _nextId++; + } + executorId = _component2Id[shrunkId]; + } + return ExecutorId(executorId); +} + +} diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h index 866f9f60423..109e8319148 100644 --- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h @@ -2,8 +2,10 @@ #pragma once #include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vector> +#include <mutex> namespace search { @@ -25,25 +27,19 @@ public: private: uint32_t _id; }; - ISequencedTaskExecutor(uint32_t numExecutors) : _numExecutors(numExecutors) { } - virtual ~ISequencedTaskExecutor() { } + ISequencedTaskExecutor(uint32_t numExecutors); + virtual ~ISequencedTaskExecutor(); /** - * Calculate which executor will handle an component. All callers - * must be in the same thread. + * Calculate which executor will handle an component. * * @param componentId component id * @return executor id */ - ExecutorId getExecutorId(uint64_t componentId) const { - return ExecutorId((componentId * 1099511628211ULL ) % _numExecutors); - } + ExecutorId getExecutorId(uint64_t componentId) const; uint32_t getNumExecutors() const { return _numExecutors; } - ExecutorId getExecutorId(vespalib::stringref componentId) const { - vespalib::hash<vespalib::stringref> hashfun; - return getExecutorId(hashfun(componentId)); - } + ExecutorId getExecutorId(vespalib::stringref componentId) const; /** * Schedule a task to run after all previously scheduled tasks with @@ -75,7 +71,6 @@ public: * Wrap lambda function into a task and schedule it to be run. * Caller must ensure that pointers and references are valid and * call sync before tearing down pointed to/referenced data. - * All tasks must be scheduled from same thread. * * @param componentId component id * @param function function to be wrapped in a task and later executed @@ -90,7 +85,6 @@ public: * Wrap lambda function into a task and schedule it to be run. * Caller must ensure that pointers and references are valid and * call sync before tearing down pointed to/referenced data. - * All tasks must be scheduled from same thread. * * @param id executor id * @param function function to be wrapped in a task and later executed @@ -99,8 +93,16 @@ public: void execute(ExecutorId id, FunctionType &&function) { executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } + /** + * For testing only + */ + uint32_t getComponentHashSize() const { return _component2Id.size(); } + uint32_t getComponentEffectiveHashSize() const { return _nextId; } private: - uint32_t _numExecutors; + mutable std::vector<uint8_t> _component2Id; + mutable std::mutex _mutex; + uint32_t _numExecutors; + mutable uint32_t _nextId; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h index a00e86bd540..7c63b167914 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Copyright 2019 Oath inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once diff --git a/security-tools/README.md b/security-tools/README.md index 077edd04a58..dbfcdac650a 100644 --- a/security-tools/README.md +++ b/security-tools/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # security-tools -Contains the "vespa-security-env" command line tool for Vespa.
\ No newline at end of file +Contains the "vespa-security-env" command line tool for Vespa. diff --git a/security-utils/README.md b/security-utils/README.md index 241d2669ddf..44fb7a2fb76 100644 --- a/security-utils/README.md +++ b/security-utils/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # security-utils -Contains various security utility classes for Java.
\ No newline at end of file +Contains various security utility classes for Java. diff --git a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java index 8bb7e0e5ab9..6e69c36d69c 100644 --- a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java @@ -100,6 +100,9 @@ public class KeyStoreBuilder { } catch (IOException e) { throw new UncheckedIOException(e); } + finally { + inputFilePassword = null; + } } private static class KeyEntry { @@ -125,4 +128,5 @@ public class KeyStoreBuilder { this.certificate = certificate; } } + } diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index 221018122bf..d2b98fd20d9 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -136,6 +136,9 @@ public class SslContextBuilder { } catch (IOException e) { throw new UncheckedIOException(e); } + finally { + keyStorePassword = null; + } } private static KeyStore createTrustStore(List<X509Certificate> caCertificates) { diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 0095092c018..5b7ac5380fd 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -22,7 +22,9 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { @Override public Optional<Boolean> multiTenant() { - return getInstallVariable("multitenant", Boolean::parseBoolean); + return Optional.ofNullable(System.getenv("VESPA_CONFIGSERVER_MULTITENANT")) + .or(() -> getInstallVariable("multitenant")) + .map(Boolean::parseBoolean); } @Override @@ -99,7 +101,9 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { @Override public Optional<Boolean> hostedVespa() { - return getInstallVariable("hosted_vespa", Boolean::parseBoolean); + return Optional.ofNullable(System.getenv("VESPA_CONFIGSERVER_HOSTED")) + .or(() -> getInstallVariable("hosted_vespa")) + .map(Boolean::parseBoolean); } @Override @@ -145,11 +149,7 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { } private static <T> Optional<T> getInstallVariable(String name, Function<String, T> converter) { - return getInstallVariable(name, "cloudconfig_server", converter); - } - - private static <T> Optional<T> getInstallVariable(String name, String installPackage, Function<String, T> converter) { - return getRawInstallVariable(installPackage + "." + name).map(converter); + return getRawInstallVariable("cloudconfig_server." + name).map(converter); } private static Optional<String> getRawInstallVariable(String name) { diff --git a/tenant-auth/README.md b/tenant-auth/README.md index 0514b68400e..c7552bd987c 100644 --- a/tenant-auth/README.md +++ b/tenant-auth/README.md @@ -1 +1,2 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Utilities that authenticate users to the hosted Vespa API, or to hosted Vespa applications. diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java index c2d47622040..82d859b08bd 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.auth; import ai.vespa.hosted.api.ControllerHttpClient; diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java index 37d5106d0c2..a13e2d8ee56 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.auth; import ai.vespa.hosted.api.Properties; diff --git a/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java index ff4bebce3ff..64282c26849 100644 --- a/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java +++ b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.auth; public class AuthenticatorTest { diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java index 0503e553e17..8327916b41d 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; /** diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java index 0453adca7a3..46f5f8ef5fd 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java index 1136233113a..c479bab6e13 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; import ai.vespa.hosted.api.ControllerHttpClient; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java index f6ad4d0acf9..e17589c325d 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java index 63bb9d68523..0c81b483ba0 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java index cc9d5d82b2c..39e9cd5bc75 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.metric; import java.util.HashMap; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java index 3aa5a126745..f1c5fbb0ba5 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.metric; import ai.vespa.hosted.cd.Endpoint; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java index a1ece1e2aa8..561c0f9dee3 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.metric; import java.util.List; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java index f1cb89641b3..62b2528e0a4 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.metric; import java.util.HashMap; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java index d48b4566f6d..d02593e5eb3 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.metric; /** diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index e5810c4e7b2..7ad51a55bff 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -35,8 +35,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; /** - * VespaRecordWriter sends the output <key, value> to one or more Vespa - * endpoints. + * VespaRecordWriter sends the output <key, value> to one or more Vespa endpoints. * * @author lesters */ diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index 7e16979056c..fa463a77923 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -13,6 +13,7 @@ import org.apache.pig.data.DataBag; import org.apache.pig.data.DataByteArray; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.schema.Schema; import org.joda.time.DateTime; import java.io.ByteArrayOutputStream; @@ -60,9 +61,12 @@ public class VespaDocumentOperation extends EvalFunc<String> { } + private static final String PROPERTY_CREATE_IF_NON_EXISTENT = "create-if-non-existent"; private static final String PROPERTY_ID_TEMPLATE = "docid"; private static final String PROPERTY_OPERATION = "operation"; + private static final String BAG_AS_MAP_FIELDS = "bag-as-map-fields"; private static final String SIMPLE_ARRAY_FIELDS = "simple-array-fields"; + private static final String SIMPLE_OBJECT_FIELDS = "simple-object-fields"; private static final String CREATE_TENSOR_FIELDS = "create-tensor-fields"; private static final String EXCLUDE_FIELDS = "exclude-fields"; @@ -99,11 +103,12 @@ public class VespaDocumentOperation extends EvalFunc<String> { reporter.progress(); } - Map<String, Object> fields = TupleTools.tupleMap(getInputSchema(), tuple); + Schema inputSchema = getInputSchema(); + Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); // create json - json = create(operation, docId, fields, properties); + json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1); return null; @@ -134,7 +139,8 @@ public class VespaDocumentOperation extends EvalFunc<String> { * @return A valid JSON Vespa document operation * @throws IOException ... */ - public static String create(Operation op, String docId, Map<String, Object> fields, Properties properties) throws IOException { + public static String create(Operation op, String docId, Map<String, Object> fields, Properties properties, + Schema schema) throws IOException { if (op == null) { return null; } @@ -152,8 +158,13 @@ public class VespaDocumentOperation extends EvalFunc<String> { g.writeStringField(op.toString(), docId); + boolean createIfNonExistent = Boolean.parseBoolean(properties.getProperty(PROPERTY_CREATE_IF_NON_EXISTENT, "false")); + if (op == Operation.UPDATE && createIfNonExistent) { + writeField("create", true, DataType.BOOLEAN, g, properties, schema, op, 0); + } + if (op != Operation.REMOVE) { - writeField("fields", fields, DataType.MAP, g, properties, op, 0); + writeField("fields", fields, DataType.MAP, g, properties, schema, op, 0); } g.writeEndObject(); @@ -164,19 +175,19 @@ public class VespaDocumentOperation extends EvalFunc<String> { @SuppressWarnings("unchecked") - private static void writeField(String name, Object value, Byte type, JsonGenerator g, Properties properties, Operation op, int depth) throws IOException { + private static void writeField(String name, Object value, Byte type, JsonGenerator g, Properties properties, Schema schema, Operation op, int depth) throws IOException { if (shouldWriteField(name, properties, depth)) { g.writeFieldName(name); if (shouldWritePartialUpdate(op, depth)) { - writePartialUpdate(value, type, g, name, properties, op, depth); + writePartialUpdate(value, type, g, name, properties, schema, op, depth); } else { - writeValue(value, type, g, name, properties, op, depth); + writeValue(value, type, g, name, properties, schema, op, depth); } } } @SuppressWarnings("unchecked") - private static void writeValue(Object value, Byte type, JsonGenerator g, String name, Properties properties, Operation op, int depth) throws IOException { + private static void writeValue(Object value, Byte type, JsonGenerator g, String name, Properties properties, Schema schema, Operation op, int depth) throws IOException { switch (type) { case DataType.UNKNOWN: break; @@ -225,31 +236,63 @@ public class VespaDocumentOperation extends EvalFunc<String> { String k = entry.getKey().toString(); Object v = entry.getValue(); Byte t = DataType.findType(v); - writeField(k, v, t, g, properties, op, depth+1); + Schema fieldSchema = (schema != null) ? schema.getField(k).schema : null; + writeField(k, v, t, g, properties, fieldSchema, op, depth+1); } } g.writeEndObject(); break; case DataType.TUPLE: Tuple tuple = (Tuple) value; - boolean writeStartArray = shouldWriteTupleStart(tuple, name, properties); - if (writeStartArray) { - g.writeStartArray(); - } - for (Object v : tuple) { - writeValue(v, DataType.findType(v), g, name, properties, op, depth); - } - if (writeStartArray) { - g.writeEndArray(); + if (shouldWriteTupleAsMap(name, properties)) { + Map<String, Object> fields = TupleTools.tupleMap(schema, tuple); + writeValue(fields, DataType.MAP, g, name, properties, schema, op, depth); + } else { + boolean writeStartArray = shouldWriteTupleStart(tuple, name, properties); + if (writeStartArray) { + g.writeStartArray(); + } + for (Object v : tuple) { + writeValue(v, DataType.findType(v), g, name, properties, schema, op, depth); + } + if (writeStartArray) { + g.writeEndArray(); + } } break; case DataType.BAG: DataBag bag = (DataBag) value; - g.writeStartArray(); - for (Tuple t : bag) { - writeValue(t, DataType.TUPLE, g, name, properties, op, depth); + // get the schema of the tuple in bag + schema = (schema != null) ? schema.getField(0).schema : null; + + if (shouldWriteBagAsMap(name, properties)) { + // when treating bag as map, the schema of bag should be {(key, val)....} + // the size of tuple in bag should be 2. 1st one is key. 2nd one is val. + Schema valueSchema = (schema != null) ? schema.getField(1).schema : null; + + g.writeStartObject(); + for (Tuple element : bag) { + if (element.size() != 2) { + continue; + } + String k = (String) element.get(0); + Object v = element.get(1); + Byte t = DataType.findType(v); + if (t == DataType.TUPLE) { + Map<String, Object> fields = TupleTools.tupleMap(valueSchema, (Tuple) v); + writeField(k, fields, DataType.MAP, g, properties, valueSchema, op, depth); + } else { + writeField(k, v, t, g, properties, valueSchema, op, depth); + } + } + g.writeEndObject(); + } else { + g.writeStartArray(); + for (Tuple t : bag) { + writeValue(t, DataType.TUPLE, g, name, properties, schema, op, depth); + } + g.writeEndArray(); } - g.writeEndArray(); break; } @@ -259,10 +302,10 @@ public class VespaDocumentOperation extends EvalFunc<String> { return op == Operation.UPDATE && depth == 1; } - private static void writePartialUpdate(Object value, Byte type, JsonGenerator g, String name, Properties properties, Operation op, int depth) throws IOException { + private static void writePartialUpdate(Object value, Byte type, JsonGenerator g, String name, Properties properties, Schema schema, Operation op, int depth) throws IOException { g.writeStartObject(); g.writeFieldName(PARTIAL_UPDATE_ASSIGN); // TODO: lookup field name in a property to determine correct operation - writeValue(value, type, g, name, properties, op, depth); + writeValue(value, type, g, name, properties, schema, op, depth); g.writeEndObject(); } @@ -286,6 +329,46 @@ public class VespaDocumentOperation extends EvalFunc<String> { return true; } + private static boolean shouldWriteTupleAsMap(String name, Properties properties) { + if (properties == null) { + return false; + } + String simpleObjectFields = properties.getProperty(SIMPLE_OBJECT_FIELDS); + if (simpleObjectFields == null) { + return false; + } + if (simpleObjectFields.equals("*")) { + return true; + } + String[] fields = simpleObjectFields.split(","); + for (String field : fields) { + if (field.trim().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + private static boolean shouldWriteBagAsMap(String name, Properties properties) { + if (properties == null) { + return false; + } + String bagAsMapFields = properties.getProperty(BAG_AS_MAP_FIELDS); + if (bagAsMapFields == null) { + return false; + } + if (bagAsMapFields.equals("*")) { + return true; + } + String[] fields = bagAsMapFields.split(","); + for (String field : fields) { + if (field.trim().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + private static boolean shouldCreateTensor(Map<Object, Object> map, String name, Properties properties) { if (properties == null) { return false; diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaStorage.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaStorage.java index f0283636725..0ef90214f41 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaStorage.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaStorage.java @@ -11,6 +11,7 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.pig.ResourceSchema; import org.apache.pig.StoreFunc; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.UDFContext; import java.io.*; @@ -160,7 +161,8 @@ public class VespaStorage extends StoreFunc { Map<String, Object> fields = TupleTools.tupleMap(resourceSchema, tuple); String docId = TupleTools.toString(fields, template); - return VespaDocumentOperation.create(operation, docId, fields, properties); + Schema schema = Schema.getPigSchema(resourceSchema); + return VespaDocumentOperation.create(operation, docId, fields, properties, schema); } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 6ff13f5c4fa..7d0fe72fc64 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hadoop.pig; import org.apache.pig.data.*; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; @@ -48,6 +49,22 @@ public class VespaDocumentOperationTest { @Test + public void requireThatUDFSupportsCreateIfNonExistent() throws IOException { + String json = getDocumentOperationJson("docid=id:<application>:metrics::<name>-<date>", "operation=update", + "create-if-non-existent=true"); + ObjectMapper m = new ObjectMapper(); + JsonNode root = m.readTree(json); + JsonNode fields = root.path("fields"); + + assertEquals("id:testapp:metrics::clicks-20160112", root.get("update").getTextValue()); + assertEquals(true, root.get("create").getBooleanValue()); + assertEquals("testapp", fields.get("application").get("assign").getTextValue()); + assertEquals("clicks", fields.get("name").get("assign").getTextValue()); + assertEquals(3, fields.get("value").get("assign").getIntValue()); + } + + + @Test public void requireThatUDFReturnsNullForMissingConfig() throws Exception { String json = getDocumentOperationJson(); assertNull(json); @@ -266,10 +283,76 @@ public class VespaDocumentOperationTest { } + @Test + public void requireThatUDFSupportsSimpleObjectFields() throws IOException { + Schema objectSchema = new Schema(); + Tuple objectTuple = TupleFactory.getInstance().newTuple(); + addToTuple("id", DataType.LONG, 123456789L, objectSchema, objectTuple); + addToTuple("url", DataType.CHARARRAY, "example.com", objectSchema, objectTuple); + addToTuple("value", DataType.INTEGER, 123, objectSchema, objectTuple); + + Schema schema = new Schema(); + Tuple tuple = TupleFactory.getInstance().newTuple(); + addToTuple("object", DataType.TUPLE, objectTuple, objectSchema, schema, tuple); + + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "simple-object-fields=object"); + docOp.setInputSchema(schema); + String json = docOp.exec(tuple); + + ObjectMapper m = new ObjectMapper(); + JsonNode root = m.readTree(json); + JsonNode fields = root.get("fields"); + JsonNode objectNode = fields.get("object"); + + assertEquals(123456789L, objectNode.get("id").asLong()); + assertEquals("example.com", objectNode.get("url").asText()); + assertEquals(123, objectNode.get("value").asInt()); + } + + + @Test + public void requireThatUDFSupportsBagAsMapFields() throws IOException { + DataBag bag = BagFactory.getInstance().newDefaultBag(); + + Schema objectSchema = new Schema(); + Tuple objectTuple = TupleFactory.getInstance().newTuple(); + addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple); + addToTuple("value", DataType.INTEGER, 123456, objectSchema, objectTuple); + bag.add(objectTuple); + + objectSchema = new Schema(); + objectTuple = TupleFactory.getInstance().newTuple(); + addToTuple("key", DataType.CHARARRAY, "234567", objectSchema, objectTuple); + addToTuple("value", DataType.INTEGER, 234567, objectSchema, objectTuple); + bag.add(objectTuple); + + Schema schema = new Schema(); + Tuple tuple = TupleFactory.getInstance().newTuple(); + addToTuple("bag", DataType.BAG, bag, objectSchema, schema, tuple); + + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "bag-as-map-fields=bag"); + docOp.setInputSchema(schema); + String json = docOp.exec(tuple); + + ObjectMapper m = new ObjectMapper(); + JsonNode root = m.readTree(json); + JsonNode fields = root.get("fields"); + JsonNode bagNode = fields.get("bag"); + + assertEquals(123456, bagNode.get("123456").asInt()); + assertEquals(234567, bagNode.get("234567").asInt()); + } + + private void addToTuple(String alias, byte type, Object value, Schema schema, Tuple tuple) { schema.add(new Schema.FieldSchema(alias, type)); tuple.append(value); } + private void addToTuple(String alias, byte type, Object value, Schema schemaInField, Schema schema, Tuple tuple) + throws FrontendException { + schema.add(new Schema.FieldSchema(alias, schemaInField, type)); + tuple.append(value); + } } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java index 8acdec334d0..66ef71fb730 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java @@ -1,6 +1,6 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client; -import com.google.common.annotations.Beta; import com.yahoo.vespa.http.client.config.SessionParams; import java.math.BigInteger; @@ -20,7 +20,6 @@ import java.util.concurrent.ThreadLocalRandom; * * @author bratseth */ -@Beta public class SyncFeedClient implements AutoCloseable { private final FeedClient wrappedClient; @@ -127,23 +126,6 @@ public class SyncFeedClient implements AutoCloseable { */ private LinkedHashMap<String, Result> results = null; - void resetExpectedResults() { - synchronized (monitor) { - if (results != null) - throw new ConcurrentModificationException("A SyncFeedClient instance is used by multiple threads"); - - resultsReceived = 0; - exception = null; - results = new LinkedHashMap<>(); - } - } - - void addExpectationOfResultFor(String operationId) { - synchronized (monitor) { - results.put(operationId, null); - } - } - void expectResultsOf(List<SyncOperation> operations) { synchronized (monitor) { if (results != null) @@ -156,6 +138,7 @@ public class SyncFeedClient implements AutoCloseable { results.put(operation.operationId, null); } } + SyncResult waitForResults() { try { synchronized (monitor) { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index fce7afdb436..0e202d1f348 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -21,6 +21,7 @@ public class Runner { /** * Feed data from inputFile to session. + * * @param feedClient where to send data to * @param inputStream source of data * @param isJson if input stream is of json formatted data diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java index 388c71087ec..5a93fdbf9e1 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client; import com.yahoo.vespa.http.client.config.Cluster; diff --git a/vespa-maven-plugin/README.md b/vespa-maven-plugin/README.md index ee98d721981..c3dbcb9cb05 100644 --- a/vespa-maven-plugin/README.md +++ b/vespa-maven-plugin/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Vespa application plugin Maven Plugin for deploying a Vespa application package. diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java index 1dbe306d549..d9ac88c6ff4 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import com.yahoo.config.provision.Environment; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index dabbaa351f9..1dae2b76711 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.ControllerHttpClient; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java index 715bd0e3009..bc7a3483e2d 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import org.apache.maven.plugins.annotations.Mojo; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java index c7062899678..eae681009cf 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import com.yahoo.config.provision.Environment; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java index 210151c450d..851b1fa214c 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.Deployment; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java index 905198c1f0d..91c7809dc76 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import com.yahoo.config.application.XmlPreProcessor; diff --git a/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java b/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java index 17c233fa71a..35f3425706d 100644 --- a/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java +++ b/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; import com.yahoo.config.provision.zone.ZoneId; diff --git a/vespa-maven-plugin/src/test/resources/effective-services/services.xml b/vespa-maven-plugin/src/test/resources/effective-services/services.xml index bf183f474e4..184eb2ab48c 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/services.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/services.xml @@ -8,4 +8,4 @@ </config> </component> </container> -</services>
\ No newline at end of file +</services> diff --git a/vespa-testrunner-components/CMakeLists.txt b/vespa-testrunner-components/CMakeLists.txt index 35ddb17cbc6..e2eeb2feecb 100644 --- a/vespa-testrunner-components/CMakeLists.txt +++ b/vespa-testrunner-components/CMakeLists.txt @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_java_artifact(vespa-testrunner-components) install_fat_java_artifact(vespa-testrunner-components) install_config_definition(src/main/resources/configdefinitions/test-runner.def com.yahoo.vespa.hosted.testrunner.test-runner.def) diff --git a/vespa-testrunner-components/README.md b/vespa-testrunner-components/README.md index 034ad95ac25..2f8bf5fe01b 100644 --- a/vespa-testrunner-components/README.md +++ b/vespa-testrunner-components/README.md @@ -1,3 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Vespa-testrunner-components Defines handler and component used by the vespa application that is deployed by the controller to diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml index 66bcd92df50..7c9b8aa99f1 100644 --- a/vespa-testrunner-components/pom.xml +++ b/vespa-testrunner-components/pom.xml @@ -1,4 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java index 0233e4db2bb..e6f402ba563 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import com.yahoo.vespa.defaults.Defaults; diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java index 1507891e69b..e49650e35e3 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; /** diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index 420b37441a5..5b143238730 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import com.google.inject.Inject; diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index d3393ce8dbe..b5366a97e86 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def index 3686aa972a5..257b73d7f4f 100644 --- a/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def +++ b/vespa-testrunner-components/src/main/resources/configdefinitions/test-runner.def @@ -1,3 +1,4 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package=com.yahoo.vespa.hosted.testrunner artifactsPath path diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java index ef8df2b748b..ca3da385dfe 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import org.apache.commons.io.IOUtils; @@ -31,4 +32,4 @@ public class PomXmlGeneratorTest { assertEquals(resourceFile, expected, actual); } -}
\ No newline at end of file +} diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java index a91b1308080..3f6fecc8249 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import com.yahoo.vespa.config.SlimeUtils; diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java index 49c95fa4b6f..22fd7fddf31 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; import org.fusesource.jansi.Ansi; diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 53e708601d7..9c4b81da806 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -62,11 +62,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>3.4</version> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java index ae8413f226e..325c5492776 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java @@ -5,7 +5,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.json.JsonWriter; import com.yahoo.documentapi.DumpVisitorDataHandler; -import org.apache.commons.lang3.exception.ExceptionUtils; +import com.yahoo.exception.ExceptionUtils; import java.nio.charset.StandardCharsets; @@ -43,7 +43,7 @@ class LocalDataVisitorHandler extends DumpVisitorDataHandler { } } catch (Exception e) { synchronized (monitor) { - errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); } } } @@ -62,7 +62,7 @@ class LocalDataVisitorHandler extends DumpVisitorDataHandler { } } catch (Exception e) { synchronized (monitor) { - errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 94a38a9d3db..0485689b15d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -21,13 +21,13 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.documentapi.metrics.DocumentOperationStatus; import com.yahoo.documentapi.metrics.DocumentOperationType; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import com.yahoo.yolean.concurrent.ResourceFactory; -import org.apache.commons.lang3.exception.ExceptionUtils; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -146,7 +146,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (Exception e) { throw new RestApiException(Response.createErrorResponse( 500, - "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTrace(e), + "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.VISITOR_ERROR)); } @@ -175,7 +175,7 @@ public class OperationHandlerImpl implements OperationHandler { visitorControlHandler.waitUntilDone(); // VisitorParameters' session timeout implicitly triggers timeout failures. throwIfFatalVisitingError(visitorControlHandler, restUri); } catch (InterruptedException e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERRUPTED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERRUPTED)); } if (localDataVisitorHandler.getErrors().isEmpty()) { Optional<String> continuationToken; @@ -214,7 +214,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (DocumentAccessException documentException) { response = createErrorResponse(documentException, restUri); } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } @@ -236,7 +236,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (DocumentAccessException documentException) { response = createErrorResponse(documentException, restUri); } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } @@ -268,7 +268,7 @@ public class OperationHandlerImpl implements OperationHandler { response = Response.createErrorResponse(400, documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCEPTION); } } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); } finally { syncSessions.free(syncSession); } @@ -297,7 +297,7 @@ public class OperationHandlerImpl implements OperationHandler { return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name())); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); } finally { syncSessions.free(syncSession); } @@ -441,7 +441,7 @@ public class OperationHandlerImpl implements OperationHandler { try { params.setResumeToken(ProgressToken.fromSerializedString(options.continuation.get())); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); } } return params; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java index 3ad3e0b7f42..14c45ca625b 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedhandler; import com.yahoo.concurrent.ThreadFactoryFactory; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java index 57f3ab20186..d3f05e18841 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java @@ -35,4 +35,5 @@ public class Main { private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() { return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable)); } + } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index 88a9a783f37..50f5c9d484c 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -27,9 +27,10 @@ import java.util.logging.Logger; /** * Example client using visiting * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + * @author Einar M R Rosenvinge */ public class VdsVisitTarget { + private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName()); private boolean printIds = false; @@ -275,4 +276,5 @@ public class VdsVisitTarget { } } } + } diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index a4a9a1e1b24..d0c1abe061a 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -896,7 +896,6 @@ "public abstract com.yahoo.tensor.IndexedTensor withType(com.yahoo.tensor.TensorType)", "public com.yahoo.tensor.DimensionSizes dimensionSizes()", "public java.util.Map cells()", - "public com.yahoo.tensor.Tensor merge(java.util.function.DoubleBinaryOperator, java.util.Map)", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public java.lang.String toString()", "public boolean equals(java.lang.Object)", @@ -945,7 +944,6 @@ "public java.util.Iterator valueIterator()", "public java.util.Map cells()", "public com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)", - "public com.yahoo.tensor.Tensor merge(java.util.function.DoubleBinaryOperator, java.util.Map)", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", @@ -1036,7 +1034,6 @@ "public java.util.Iterator valueIterator()", "public java.util.Map cells()", "public com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)", - "public com.yahoo.tensor.Tensor merge(java.util.function.DoubleBinaryOperator, java.util.Map)", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", @@ -1157,12 +1154,12 @@ "public double asDouble()", "public abstract com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)", "public com.yahoo.tensor.Tensor modify(java.util.function.DoubleBinaryOperator, java.util.Map)", - "public abstract com.yahoo.tensor.Tensor merge(java.util.function.DoubleBinaryOperator, java.util.Map)", "public abstract com.yahoo.tensor.Tensor remove(java.util.Set)", "public com.yahoo.tensor.Tensor map(java.util.function.DoubleUnaryOperator)", "public varargs com.yahoo.tensor.Tensor reduce(com.yahoo.tensor.functions.Reduce$Aggregator, java.lang.String[])", "public com.yahoo.tensor.Tensor reduce(com.yahoo.tensor.functions.Reduce$Aggregator, java.util.List)", "public com.yahoo.tensor.Tensor join(com.yahoo.tensor.Tensor, java.util.function.DoubleBinaryOperator)", + "public com.yahoo.tensor.Tensor merge(com.yahoo.tensor.Tensor, java.util.function.DoubleBinaryOperator)", "public com.yahoo.tensor.Tensor rename(java.lang.String, java.lang.String)", "public com.yahoo.tensor.Tensor concat(double, java.lang.String)", "public com.yahoo.tensor.Tensor concat(com.yahoo.tensor.Tensor, java.lang.String)", @@ -1266,6 +1263,7 @@ "public int hashCode()", "public boolean equals(java.lang.Object)", "public final java.lang.String toString(com.yahoo.tensor.TensorType)", + "public static java.lang.String labelToString(java.lang.String)", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [] @@ -1280,6 +1278,7 @@ "public void <init>()", "public void <init>(com.yahoo.tensor.TensorType$Value)", "public varargs void <init>(com.yahoo.tensor.TensorType[])", + "public varargs void <init>(boolean, com.yahoo.tensor.TensorType[])", "public void <init>(java.lang.Iterable)", "public void <init>(com.yahoo.tensor.TensorType$Value, java.lang.Iterable)", "public int rank()", @@ -1327,6 +1326,7 @@ "public abstract com.yahoo.tensor.TensorType$Dimension$Type type()", "public abstract com.yahoo.tensor.TensorType$Dimension withName(java.lang.String)", "public boolean isIndexed()", + "public boolean isMapped()", "public abstract java.lang.String toString()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -1746,6 +1746,25 @@ ], "fields": [] }, + "com.yahoo.tensor.functions.Merge": { + "superClass": "com.yahoo.tensor.functions.PrimitiveTensorFunction", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.tensor.functions.TensorFunction, com.yahoo.tensor.functions.TensorFunction, java.util.function.DoubleBinaryOperator)", + "public static com.yahoo.tensor.TensorType outputType(com.yahoo.tensor.TensorType, com.yahoo.tensor.TensorType)", + "public java.util.function.DoubleBinaryOperator merger()", + "public java.util.List arguments()", + "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", + "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)" + ], + "fields": [] + }, "com.yahoo.tensor.functions.PrimitiveTensorFunction": { "superClass": "com.yahoo.tensor.functions.TensorFunction", "interfaces": [], diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 32040785fc2..a8380e9513c 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -24,10 +24,6 @@ <artifactId>lz4</artifactId> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-exec</artifactId> </dependency> diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index c64a9ada410..95e3f7fcf39 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.collections; import java.util.Collection; diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java index 479850beb1a..86ff9eacb8e 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java @@ -3,10 +3,14 @@ package com.yahoo.collections; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang.builder.ToStringBuilder; import java.lang.reflect.InvocationTargetException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * A map holding multiple items at each key (using ArrayList and HashMap). @@ -140,7 +144,10 @@ public class ListMap<K, V> { @Override public String toString() { - return ToStringBuilder.reflectionToString(this); + return "ListMap{" + + "frozen=" + frozen + + ", map=" + map + + '}'; } /** Returns the number of keys in this map */ diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java b/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java index c4b293f19a1..bd5a2a1fdc0 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; import java.util.Collections; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java b/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java index 43a170230ad..3702ed40be0 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; import com.yahoo.yolean.Exceptions; diff --git a/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java new file mode 100644 index 00000000000..4d4a2d02a1a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java @@ -0,0 +1,40 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.exception; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.UncheckedIOException; + +/** + * Misc exception utility methods - replacement for Apache commons-lang's ExceptionUtils + * + * @author bjorncs + */ +public class ExceptionUtils { + + private ExceptionUtils() {} + + public static String getStackTraceAsString(Throwable throwable) { + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + throwable.printStackTrace(printWriter); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static String getStackTraceRecursivelyAsString(Throwable throwable) { + Throwable cause = throwable; + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + do { + cause.printStackTrace(printWriter); + } while ((cause = cause.getCause()) != null); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/exception/package-info.java b/vespajlib/src/main/java/com/yahoo/exception/package-info.java new file mode 100644 index 00000000000..8254f90651f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/exception/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.exception; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/net/Url.java b/vespajlib/src/main/java/com/yahoo/net/Url.java index 60a9c98bce8..73584b4bc4e 100644 --- a/vespajlib/src/main/java/com/yahoo/net/Url.java +++ b/vespajlib/src/main/java/com/yahoo/net/Url.java @@ -250,4 +250,5 @@ public class Url { public String toString() { return image; } + } diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java index 77d931b5278..860107a4cc3 100644 --- a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java +++ b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java @@ -175,4 +175,5 @@ public class UrlTokenizer { schemeToPort.put(scheme, str); portToScheme.put(str, scheme); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java index 81fb6a275cb..fe1d892ee60 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; import java.util.function.Function; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java index 202817ece42..632501c7d08 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java @@ -38,7 +38,7 @@ public final class DimensionSizes { * @throws IllegalArgumentException if the index is larger than the number of dimensions in this tensor minus one */ public long size(int dimensionIndex) { - if (dimensionIndex <0 || dimensionIndex >= sizes.length) + if (dimensionIndex < 0 || dimensionIndex >= sizes.length) throw new IllegalArgumentException("Illegal dimension index " + dimensionIndex + ": This has " + sizes.length + " dimensions"); return sizes[dimensionIndex]; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index ba3a35e8eda..ad82dd6c3ac 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -197,11 +197,6 @@ public abstract class IndexedTensor implements Tensor { } @Override - public Tensor merge(DoubleBinaryOperator op, Map<TensorAddress, Double> cells) { - throw new IllegalArgumentException("Merge is not supported for indexed tensors"); - } - - @Override public Tensor remove(Set<TensorAddress> addresses) { throw new IllegalArgumentException("Remove is not supported for indexed tensors"); } @@ -209,12 +204,18 @@ public abstract class IndexedTensor implements Tensor { @Override public String toString() { if (type.rank() == 0) return Tensor.toStandardString(this); - if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) return Tensor.toStandardString(this); + if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this); Indexes indexes = Indexes.of(dimensionSizes); StringBuilder b = new StringBuilder(type.toString()).append(":"); - for (int index = 0; index < size(); index++) { + indexedBlockToString(this, indexes, b); + return b.toString(); + } + + static void indexedBlockToString(IndexedTensor tensor, Indexes indexes, StringBuilder b) { + for (int index = 0; index < tensor.size(); index++) { indexes.next(); // start brackets @@ -222,20 +223,19 @@ public abstract class IndexedTensor implements Tensor { b.append("["); // value - if (type.valueType() == TensorType.Value.DOUBLE) - b.append(get(index)); - else if (type.valueType() == TensorType.Value.FLOAT) - b.append(get(index)); // TODO: Use getFloat + if (tensor.type().valueType() == TensorType.Value.DOUBLE) + b.append(tensor.get(index)); + else if (tensor.type().valueType() == TensorType.Value.FLOAT) + b.append(tensor.getFloat(index)); else - throw new IllegalStateException("Unexpected value type " + type.valueType()); + throw new IllegalStateException("Unexpected value type " + tensor.type().valueType()); // end bracket and comma for (int i = 0; i < indexes.nextDimensionsAtEnd(); i++) b.append("]"); - if (index < size() - 1) + if (index < tensor.size() - 1) b.append(", "); } - return b.toString(); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index 693c4b5f2b0..33f904efd42 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -53,25 +53,6 @@ public class MappedTensor implements Tensor { } @Override - public Tensor merge(DoubleBinaryOperator op, Map<TensorAddress, Double> addCells) { - - // currently, underlying implementation disallows multiple entries with the same key - - Tensor.Builder builder = Tensor.Builder.of(type()); - for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { - TensorAddress address = cell.getKey(); - double value = cell.getValue(); - builder.cell(address, addCells.containsKey(address) ? op.applyAsDouble(value, addCells.get(address)) : value); - } - for (Map.Entry<TensorAddress, Double> addCell : addCells.entrySet()) { - if ( ! cells.containsKey(addCell.getKey())) { - builder.cell(addCell.getKey(), addCell.getValue()); - } - } - return builder.build(); - } - - @Override public Tensor remove(Set<TensorAddress> addresses) { Tensor.Builder builder = Tensor.Builder.of(type()); for (Iterator<Tensor.Cell> i = cellIterator(); i.hasNext(); ) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 0c4efe78113..67c6930ce35 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -53,9 +53,11 @@ public class MixedTensor implements Tensor { @Override public double get(TensorAddress address) { long cellIndex = index.indexOf(address); + if (cellIndex < 0) + return Double.NaN; Cell cell = cells.get((int)cellIndex); if ( ! address.equals(cell.getKey())) - throw new IllegalStateException("Unable to find correct cell in " + this + " by direct index " + address); + return Double.NaN; return cell.getValue(); } @@ -71,10 +73,6 @@ public class MixedTensor implements Tensor { return cells.iterator(); } - private Iterable<Cell> cellIterable() { - return this::cellIterator; - } - /** * Returns an iterator over the values of this tensor. * The iteration order is the same as for cellIterator. @@ -113,20 +111,6 @@ public class MixedTensor implements Tensor { } @Override - public Tensor merge(DoubleBinaryOperator op, Map<TensorAddress, Double> addCells) { - Tensor.Builder builder = Tensor.Builder.of(type()); - for (Cell cell : cellIterable()) { - TensorAddress address = cell.getKey(); - double value = cell.getValue(); - builder.cell(address, addCells.containsKey(address) ? op.applyAsDouble(value, addCells.get(address)) : value); - } - for (Map.Entry<TensorAddress, Double> addCell : addCells.entrySet()) { - builder.cell(addCell.getKey(), addCell.getValue()); - } - return builder.build(); - } - - @Override public Tensor remove(Set<TensorAddress> addresses) { Tensor.Builder builder = Tensor.Builder.of(type()); @@ -148,7 +132,14 @@ public class MixedTensor implements Tensor { public int hashCode() { return cells.hashCode(); } @Override - public String toString() { return Tensor.toStandardString(this); } + public String toString() { + if (type.rank() == 0) return Tensor.toStandardString(this); + if (type.rank() > 1 && type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this); + if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) return Tensor.toStandardString(this); + + return type.toString() + ":" + index.contentToString(this); + } @Override public boolean equals(Object other) { @@ -380,10 +371,11 @@ public class MixedTensor implements Tensor { this.denseType = createPartialType(type.valueType(), indexedDimensions); } + /** Returns the index of the given address, or -1 if it is not present */ public long indexOf(TensorAddress address) { TensorAddress sparsePart = sparsePartialAddress(address); if ( ! sparseMap.containsKey(sparsePart)) - throw new IllegalArgumentException("Address subspace " + sparsePart + " not found in " + this); + return -1; long base = sparseMap.get(sparsePart); long offset = denseOffset(address); return base + offset; @@ -494,7 +486,63 @@ public class MixedTensor implements Tensor { @Override public String toString() { - return "indexes into " + type; + return "index into " + type; + } + + private String contentToString(MixedTensor tensor) { + if (mappedDimensions.size() > 1) throw new IllegalStateException("Should be ensured by caller"); + if (mappedDimensions.size() == 0) { + StringBuilder b = new StringBuilder(); + denseSubspaceToString(tensor, 0, b); + return b.toString(); + } + + // Exactly 1 mapped dimension + StringBuilder b = new StringBuilder("{"); + sparseMap.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { + b.append(TensorAddress.labelToString(entry.getKey().label(0 ))); + b.append(":"); + denseSubspaceToString(tensor, entry.getValue(), b); + b.append(","); + }); + if (b.length() > 1) + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + private void denseSubspaceToString(MixedTensor tensor, long subspaceIndex, StringBuilder b) { + if (denseSubspaceSize == 1) { + b.append(getDouble(subspaceIndex, 0, tensor)); + return; + } + + IndexedTensor.Indexes indexes = IndexedTensor.Indexes.of(denseType); + for (int index = 0; index < denseSubspaceSize; index++) { + indexes.next(); + + // start brackets + for (int i = 0; i < indexes.nextDimensionsAtStart(); i++) + b.append("["); + + // value + if (type.valueType() == TensorType.Value.DOUBLE) + b.append(getDouble(subspaceIndex, index, tensor)); + else if (tensor.type().valueType() == TensorType.Value.FLOAT) + b.append(getDouble(subspaceIndex, index, tensor)); // TODO: Really use floats + else + throw new IllegalStateException("Unexpected value type " + type.valueType()); + + // end bracket and comma + for (int i = 0; i < indexes.nextDimensionsAtEnd(); i++) + b.append("]"); + if (index < denseSubspaceSize - 1) + b.append(", "); + } + } + + private double getDouble(long indexedSubspaceIndex, long indexInIndexedSubspace, MixedTensor tensor) { + return tensor.cells.get((int)(indexedSubspaceIndex + indexInIndexedSubspace)).getDoubleValue(); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index cffd41905a1..08d4f1c08b7 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -12,6 +12,7 @@ import com.yahoo.tensor.functions.Join; import com.yahoo.tensor.functions.L1Normalize; import com.yahoo.tensor.functions.L2Normalize; import com.yahoo.tensor.functions.Matmul; +import com.yahoo.tensor.functions.Merge; import com.yahoo.tensor.functions.Random; import com.yahoo.tensor.functions.Range; import com.yahoo.tensor.functions.Reduce; @@ -31,6 +32,7 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; +import java.util.stream.Collectors; import static com.yahoo.text.Ascii7BitMatcher.charsAndNumbers; @@ -124,18 +126,6 @@ public interface Tensor { /** * Returns a new tensor where existing cells in this tensor have been - * modified according to the given operation and cells in the given map. - * In contrast to {@link #modify}, previously non-existing cells are added - * to this tensor. Only valid for sparse or mixed tensors. - * - * @param op how to update overlapping cells - * @param cells cells to merge with this tensor - * @return a new tensor where this tensor is merged with the other - */ - Tensor merge(DoubleBinaryOperator op, Map<TensorAddress, Double> cells); - - /** - * Returns a new tensor where existing cells in this tensor have been * removed according to the given set of addresses. Only valid for sparse * or mixed tensors. For mixed tensors, addresses are assumed to only * contain the sparse dimensions, as the entire dense subspace is removed. @@ -164,6 +154,10 @@ public interface Tensor { return new Join<>(new ConstantTensor<>(this), new ConstantTensor<>(argument), combinator).evaluate(); } + default Tensor merge(Tensor argument, DoubleBinaryOperator combinator) { + return new Merge<>(new ConstantTensor<>(this), new ConstantTensor<>(argument), combinator).evaluate(); + } + default Tensor rename(String fromDimension, String toDimension) { return new Rename<>(new ConstantTensor<>(this), Collections.singletonList(fromDimension), Collections.singletonList(toDimension)).evaluate(); @@ -319,23 +313,21 @@ public interface Tensor { } static String contentToString(Tensor tensor) { - List<java.util.Map.Entry<TensorAddress, Double>> cellEntries = new ArrayList<>(tensor.cells().entrySet()); + var cellEntries = new ArrayList<>(tensor.cells().entrySet()); if (tensor.type().dimensions().isEmpty()) { if (cellEntries.isEmpty()) return "{}"; return "{" + cellEntries.get(0).getValue() +"}"; } + return "{" + cellEntries.stream().sorted(Map.Entry.comparingByKey()) + .map(cell -> cellToString(cell, tensor.type())) + .collect(Collectors.joining(",")) + + "}"; + } - Collections.sort(cellEntries, java.util.Map.Entry.<TensorAddress, Double>comparingByKey()); - - StringBuilder b = new StringBuilder("{"); - for (java.util.Map.Entry<TensorAddress, Double> cell : cellEntries) { - b.append(cell.getKey().toString(tensor.type())).append(":").append(cell.getValue()); - b.append(","); - } - if (b.length() > 1) - b.setLength(b.length() - 1); - b.append("}"); - return b.toString(); + private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) { + return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) + + ":" + + cell.getValue(); } // ----------------- equality diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index a3805fb789a..4a076199846 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -91,7 +91,8 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return b.toString(); } - private String labelToString(String label) { + /** Returns a label as a string with approriate quoting/escaping when necessary */ + public static String labelToString(String label) { if (TensorType.labelMatcher.matches(label)) return label; // no quoting if (label.contains("'")) return "\"" + label + "\""; return "'" + label + "'"; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index 9aa764a0b36..becec1a4493 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -99,7 +99,7 @@ class TensorParser { if (type.isEmpty()) throw new IllegalArgumentException("The mixed tensor form requires an explicit tensor type " + "on the form 'tensor(dimensions):..."); - if (type.get().dimensions().stream().filter(d -> ! d.isIndexed()).count() != 1) + if (type.get().dimensions().stream().filter(d -> ! d.isIndexed()).count() > 1) throw new IllegalArgumentException("The mixed tensor form requires a type with a single mapped dimension, " + "but got " + type.get()); @@ -310,7 +310,7 @@ class TensorParser { } private void parse() { - TensorType.Dimension mappedDimension = builder.type().dimensions().stream().filter(d -> ! d.isIndexed()).findAny().get(); + TensorType.Dimension mappedDimension = findMappedDimension(); TensorType mappedSubtype = MixedTensor.createPartialType(builder.type().valueType(), List.of(mappedDimension)); if (dimensionOrder != null) dimensionOrder.remove(mappedDimension.name()); @@ -332,6 +332,15 @@ class TensorParser { } } + private TensorType.Dimension findMappedDimension() { + Optional<TensorType.Dimension> mappedDimension = builder.type().dimensions().stream().filter(d -> d.isMapped()).findAny(); + if (mappedDimension.isPresent()) return mappedDimension.get(); + if (builder.type().rank() == 1 && builder.type().dimensions().get(0).size().isEmpty()) + return builder.type().dimensions().get(0); + throw new IllegalStateException("No suitable dimension in " + builder.type() + + " for parsing as a mixed tensor. This is a bug."); + } + private void parseDenseSubspace(TensorAddress mappedAddress, List<String> denseDimensionOrder) { DenseValueParser denseParser = new DenseValueParser(string.substring(position), denseDimensionOrder, diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 58cb151875e..d8959147ee0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -312,14 +312,18 @@ public class TensorType { /** Returns true if this is an indexed bound or unbound type */ public boolean isIndexed() { return type() == Type.indexedBound || type() == Type.indexedUnbound; } + /** Returns true if this is of the mapped type */ + public boolean isMapped() { return type() == Type.mapped; } + /** * Returns the dimension resulting from combining two dimensions having the same name but possibly different - * types. This works by degrading to the type making the fewer promises. - * [N] + [M] = [min(N, M)] + * types: + * + * [N] + [M] = [ min(N, M) ] * [N] + [] = [] * [] + {} = {} */ - Dimension combineWith(Optional<Dimension> other) { + Dimension combineWith(Optional<Dimension> other, boolean allowDifferentSizes) { if ( ! other.isPresent()) return this; if (this instanceof MappedDimension) return this; if (other.get() instanceof MappedDimension) return other.get(); @@ -329,7 +333,11 @@ public class TensorType { // both are indexed bound IndexedBoundDimension thisIb = (IndexedBoundDimension)this; IndexedBoundDimension otherIb = (IndexedBoundDimension)other.get(); - return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; + if (allowDifferentSizes) + return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; + if ( ! thisIb.size().equals(otherIb.size())) + throw new IllegalArgumentException("Unequal dimension sizes in " + thisIb + " and " + otherIb); + return thisIb; } @Override @@ -483,16 +491,20 @@ public class TensorType { /** * Creates a builder containing a combination of the dimensions of the given types * - * If the same dimension is indexed with different size restrictions the largest size will be used. + * If the same dimension is indexed with different size restrictions the smallest size will be used. * If it is size restricted in one argument but not the other it will not be size restricted. * If it is indexed in one and mapped in the other it will become mapped. * * The value type will be the largest of the value types of the input types */ public Builder(TensorType ... types) { + this(true, types); + } + + public Builder(boolean allowDifferentSizes, TensorType ... types) { this.valueType = TensorType.combinedValueType(types); for (TensorType type : types) - addDimensionsOf(type); + addDimensionsOf(type, allowDifferentSizes); } /** Creates a builder from the given dimensions, having double as the value type */ @@ -510,17 +522,17 @@ public class TensorType { private static final boolean supportsMixedTypes = false; - private void addDimensionsOf(TensorType type) { + private void addDimensionsOf(TensorType type, boolean allowDifferentSizes) { if ( ! supportsMixedTypes) { // TODO: Support it - addDimensionsOfAndDisallowMixedDimensions(type); + addDimensionsOfAndDisallowMixedDimensions(type, allowDifferentSizes); } else { for (Dimension dimension : type.dimensions) - set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())))); + set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())), allowDifferentSizes)); } } - private void addDimensionsOfAndDisallowMixedDimensions(TensorType type) { + private void addDimensionsOfAndDisallowMixedDimensions(TensorType type, boolean allowDifferentSizes) { boolean containsMapped = dimensions.values().stream().anyMatch(d -> ! d.isIndexed()); containsMapped = containsMapped || type.dimensions().stream().anyMatch(d -> ! d.isIndexed()); @@ -528,7 +540,7 @@ public class TensorType { if (containsMapped) dimension = new MappedDimension(dimension.name()); Dimension existing = dimensions.get(dimension.name()); - set(dimension.combineWith(Optional.ofNullable(existing))); + set(dimension.combineWith(Optional.ofNullable(existing), allowDifferentSizes)); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 1e0eaa7fad3..5419d04a4fb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -48,7 +48,12 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP /** Returns the type resulting from applying Join to the two given types */ public static TensorType outputType(TensorType a, TensorType b) { - return new TensorType.Builder(a, b).build(); + try { + return new TensorType.Builder(false, a, b).build(); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Can not join " + a + " and " + b, e); + } } public DoubleBinaryOperator combinator() { return combinator; } @@ -75,14 +80,14 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP @Override public TensorType type(TypeContext<NAMETYPE> context) { - return new TensorType.Builder(argumentA.type(context), argumentB.type(context)).build(); + return outputType(argumentA.type(context), argumentB.type(context)); } @Override public Tensor evaluate(EvaluationContext<NAMETYPE> context) { Tensor a = argumentA.evaluate(context); Tensor b = argumentB.evaluate(context); - TensorType joinedType = new TensorType.Builder(a.type(), b.type()).build(); + TensorType joinedType = outputType(a.type(), b.type()); return evaluate(a, b, joinedType, combinator); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java new file mode 100644 index 00000000000..d5633bde36c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java @@ -0,0 +1,138 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.yahoo.tensor.DimensionSizes; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.PartialAddress; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.EvaluationContext; +import com.yahoo.tensor.evaluation.Name; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.DoubleBinaryOperator; + +/** + * The <i>merge</i> tensor operation produces from two argument tensors having equal types + * a tensor having the same type where the values are the union of the values of both tensors. In the cases where both + * tensors contain a value for a given cell, and only then, the lambda scalar expression is evaluated to produce + * the resulting cell value. + * + * @author bratseth + */ +public class Merge<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE> { + + private final TensorFunction<NAMETYPE> argumentA, argumentB; + private final DoubleBinaryOperator merger; + + public Merge(TensorFunction<NAMETYPE> argumentA, TensorFunction<NAMETYPE> argumentB, DoubleBinaryOperator merger) { + Objects.requireNonNull(argumentA, "The first argument tensor cannot be null"); + Objects.requireNonNull(argumentB, "The second argument tensor cannot be null"); + Objects.requireNonNull(merger, "The merger function cannot be null"); + this.argumentA = argumentA; + this.argumentB = argumentB; + this.merger = merger; + } + + /** Returns the type resulting from applying Merge to the two given types */ + public static TensorType outputType(TensorType a, TensorType b) { + Optional<TensorType> outputType = a.dimensionwiseGeneralizationWith(b); + if (outputType.isPresent()) return outputType.get(); + throw new IllegalArgumentException("Cannot merge " + a + " and " + b + ": Arguments must have compatible types"); + } + + public DoubleBinaryOperator merger() { return merger; } + + @Override + public List<TensorFunction<NAMETYPE>> arguments() { return ImmutableList.of(argumentA, argumentB); } + + @Override + public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { + if ( arguments.size() != 2) + throw new IllegalArgumentException("Merge must have 2 arguments, got " + arguments.size()); + return new Merge<>(arguments.get(0), arguments.get(1), merger); + } + + @Override + public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { + return new Merge<>(argumentA.toPrimitive(), argumentB.toPrimitive(), merger); + } + + @Override + public String toString(ToStringContext context) { + return "merge(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + merger + ")"; + } + + @Override + public TensorType type(TypeContext<NAMETYPE> context) { + return outputType(argumentA.type(context), argumentB.type(context)); + } + + @Override + public Tensor evaluate(EvaluationContext<NAMETYPE> context) { + Tensor a = argumentA.evaluate(context); + Tensor b = argumentB.evaluate(context); + TensorType mergedType = outputType(a.type(), b.type()); + return evaluate(a, b, mergedType, merger); + } + + static Tensor evaluate(Tensor a, Tensor b, TensorType mergedType, DoubleBinaryOperator combinator) { + // Choose merge algorithm + if (hasSingleIndexedDimension(a) && hasSingleIndexedDimension(b) && a.type().dimensions().get(0).name().equals(b.type().dimensions().get(0).name())) + return indexedVectorMerge((IndexedTensor)a, (IndexedTensor)b, mergedType, combinator); + else + return generalMerge(a, b, mergedType, combinator); + } + + private static boolean hasSingleIndexedDimension(Tensor tensor) { + return tensor.type().dimensions().size() == 1 && tensor.type().dimensions().get(0).isIndexed(); + } + + private static Tensor indexedVectorMerge(IndexedTensor a, IndexedTensor b, TensorType type, DoubleBinaryOperator combinator) { + long aSize = a.dimensionSizes().size(0); + long bSize = b.dimensionSizes().size(0); + long mergedSize = Math.max(aSize, bSize); + long sharedSize = Math.min(aSize, bSize); + Iterator<Double> aIterator = a.valueIterator(); + Iterator<Double> bIterator = b.valueIterator(); + IndexedTensor.Builder builder = IndexedTensor.Builder.of(type); + for (long i = 0; i < sharedSize; i++) + builder.cell(combinator.applyAsDouble(aIterator.next(), bIterator.next()), i); + Iterator<Double> largestIterator = aSize > bSize ? aIterator : bIterator; + for (long i = sharedSize; i < mergedSize; i++) + builder.cell(largestIterator.next(), i); + return builder.build(); + } + + private static Tensor generalMerge(Tensor a, Tensor b, TensorType mergedType, DoubleBinaryOperator combinator) { + Tensor.Builder builder = Tensor.Builder.of(mergedType); + addCellsOf(a, b, builder, combinator); + addCellsOf(b, a, builder, null); + return builder.build(); + } + + private static void addCellsOf(Tensor a, Tensor b, Tensor.Builder builder, DoubleBinaryOperator combinator) { + for (Iterator<Tensor.Cell> i = a.cellIterator(); i.hasNext(); ) { + Map.Entry<TensorAddress, Double> aCell = i.next(); + double bCellValue = b.get(aCell.getKey()); + if (Double.isNaN(bCellValue)) + builder.cell(aCell.getKey(), aCell.getValue()); + else if (combinator != null) + builder.cell(aCell.getKey(), combinator.applyAsDouble(aCell.getValue(), bCellValue)); + } + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java index 73b40222f55..4d8b5012c81 100755 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java @@ -131,4 +131,5 @@ public class ObjectDumper extends ObjectVisitor { addLine(name + ": " + obj); } } + } diff --git a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java index 3f9342c5f45..9386bf7256f 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java +++ b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.collections; import org.junit.Test; diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/StripedExecutorTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/StripedExecutorTest.java index 712f24c79d4..e3113a13fa4 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/StripedExecutorTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/StripedExecutorTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; import org.junit.Test; diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeStreamTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeStreamTest.java index 116551cea59..2f3d452ae52 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/SlimeStreamTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeStreamTest.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; import org.junit.Test; diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java index 502f0270831..2699e4642e2 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java @@ -35,7 +35,7 @@ public class MappedTensorTestCase { cell().label("x", "0").value(1). cell().label("x", "1").value(2).build(); assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames()); - assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0}", tensor.toString()); + assertEquals("tensor(x{}):{0:1.0,1:2.0}", tensor.toString()); } @Test diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java index 22b97bff52a..5d4417ec928 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java @@ -41,7 +41,7 @@ public class MixedTensorTestCase { // {y:2} should be 0.0 and non NaN since we specify indexed size build(); assertEquals(Sets.newHashSet("y"), tensor.type().dimensionNames()); - assertEquals("tensor(y[3]):{{y:0}:1.0,{y:1}:2.0,{y:2}:0.0}", + assertEquals("tensor(y[3]):[1.0, 2.0, 0.0]", tensor.toString()); } @@ -57,8 +57,8 @@ public class MixedTensorTestCase { cell().label("x", 1).label("y", 2).value(6). build(); assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); - assertEquals("tensor(x[2],y[3]):{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:0,y:2}:0.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", - tensor.toString()); + assertEquals("tensor(x[2],y[3]):[[1.0, 2.0, 0.0], [4.0, 5.0, 6.0]]", + tensor.toString()); } @Test @@ -69,8 +69,8 @@ public class MixedTensorTestCase { cell().label("x", "1").value(2). build(); assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames()); - assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0}", - tensor.toString()); + assertEquals("tensor(x{}):{0:1.0,1:2.0}", + tensor.toString()); } @Test diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 5a68df6c7df..431e4b06263 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -43,7 +43,7 @@ public class TensorParserTestCase { assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor()")).cell(1.3).build(), "tensor():{1.3}"); assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[])")).cell(1.0, 0).build(), - "tensor(x[]):{{x:0}:1.0}"); + "tensor(x[]):{0:1.0}"); assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1])")).cell(1.0, 0).build(), "tensor(x[1]):[1.0]"); assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2])")).cell(1.0, 0).cell(2.0, 1).build(), diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 7932f90d797..5bd1bbdba37 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -37,7 +37,7 @@ public class TensorTestCase { assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}", Tensor.from("{ {d1:l1,d2:l1}: 5, {d2:l2, d1:l1}:6.0} ").toString()); assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:-5.3,{d1:l1,d2:l2}:0.0}", Tensor.from("{ {d1:l1,d2:l1}:-5.3, {d2:l2, d1:l1}:0}").toString()); assertEquals("Labels are quoted when necessary", - "tensor(d1{}):{{d1:\"'''\"}:6.0,{d1:'[[\":\"]]'}:5.0}", + "tensor(d1{}):{\"'''\":6.0,'[[\":\"]]':5.0}", Tensor.from("{ {d1:'[[\":\"]]'}: 5, {d1:\"'''\"}:6.0 }").toString()); } @@ -130,13 +130,11 @@ public class TensorTestCase { assertEquals("Mapped vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.mapped, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(3, Type.indexedUnbound), vectors(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(5, Type.indexedUnbound), vectors(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(5, Type.indexedBound, 2))); - assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(5, Type.indexedBound), vectors(3, Type.indexedBound, 2))); + assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(3, Type.indexedBound, 2))); assertEquals("Mapped matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.mapped, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(3, Type.indexedUnbound), matrix(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(5, Type.indexedUnbound), matrix(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(5, Type.indexedBound, 2))); - assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(5, Type.indexedBound), matrix(3, Type.indexedBound, 2))); + assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(3, Type.indexedBound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.indexedUnbound, 2))); @@ -216,6 +214,22 @@ public class TensorTestCase { Tensor.from("tensor(x{},y[3])", "{{x:0,y:0}:1,{x:0,y:1}:2}"), Tensor.from("tensor(x{},y[3])", "{}"), Tensor.from("tensor(x{},y[3])", "{{x:0,y:0}:1,{x:0,y:1}:2}")); + assertTensorMerge( + Tensor.from("tensor(x[4]):[5,6,7,8]"), + Tensor.from("tensor(x[4]):[1,2,3,4]"), + Tensor.from("tensor(x[4]):[1,2,3,4]")); + assertTensorMerge( + Tensor.from("tensor(x[]):{{x:0}:1,{x:1}:2,{x:2}:3,{x:3}:4}"), + Tensor.from("tensor(x[]):{{x:0}:5,{x:1}:6}"), + Tensor.from("tensor(x[4]):[5,6,3,4]")); + assertTensorMerge( + Tensor.from("tensor(x{}):{a:1,b:2}"), + Tensor.from("tensor(x{}):{b:3,c:4}"), + Tensor.from("tensor(x{}):{a:1,b:3,c:4}")); + assertTensorMerge( + Tensor.from("tensor(key{},x[4]):{a:[1,2,3,4],c:[5,6,7,8]}"), + Tensor.from("tensor(key{},x[4]):{a:[9,10,11,12],b:[13,14,15,16]}"), + Tensor.from("tensor(key{},x[4]):{a:[9,10,11,12],b:[13,14,15,16],c:[5,6,7,8]}")); } @Test @@ -302,7 +316,7 @@ public class TensorTestCase { private void assertTensorMerge(Tensor init, Tensor update, Tensor expected) { DoubleBinaryOperator op = (left, right) -> right; - assertEquals(expected, init.merge(op, update.cells())); + assertEquals(expected, init.merge(update, op)); } private void assertTensorRemove(Tensor init, Tensor update, Tensor expected) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java index e6560242d5c..625d5d44b19 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java @@ -15,11 +15,11 @@ public class TensorFunctionTestCase { @Test public void testTranslation() { - assertTranslated("join(tensor(x{}):{{x:1}:1.0}, reduce(tensor(x{}):{{x:1}:1.0}, sum, x), f(a,b)(a / b))", + assertTranslated("join(tensor(x{}):{1:1.0}, reduce(tensor(x{}):{1:1.0}, sum, x), f(a,b)(a / b))", new L1Normalize<>(new ConstantTensor<>("{{x:1}:1.0}"), "x")); assertTranslated("tensor(x[2],y[3],z[4])((x==y)*(y==z))", new Diag<>(new TensorType.Builder().indexed("y",3).indexed("x",2).indexed("z",4).build())); - assertTranslated("join(tensor(x{}):{{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, reduce(tensor(x{}):{{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, max, x), f(a,b)(a==b))", + assertTranslated("join(tensor(x{}):{1:1.0,3:5.0,9:3.0}, reduce(tensor(x{}):{1:1.0,3:5.0,9:3.0}, max, x), f(a,b)(a==b))", new Argmax<>(new ConstantTensor<>("{ {x:1}:1, {x:3}:5, {x:9}:3 }"), "x")); } diff --git a/vespajlib/src/test/java/com/yahoo/text/Ascii7BitMatcherTestCase.java b/vespajlib/src/test/java/com/yahoo/text/Ascii7BitMatcherTestCase.java index 3f628b109f5..8ab85e53aca 100644 --- a/vespajlib/src/test/java/com/yahoo/text/Ascii7BitMatcherTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/Ascii7BitMatcherTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; import org.junit.Assert; diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index fa418cbad02..4b425c4bded 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -79,6 +79,8 @@ public: private: next_t _mask; }; + static size_t getModuloStl(size_t newSize); + static size_t getModuloSimple(size_t newSize); protected: struct DefaultMoveHandler { @@ -88,8 +90,6 @@ protected: } }; private: - static size_t getModuloStl(size_t newSize); - static size_t getModuloSimple(size_t newSize); static size_t getModulo(size_t newSize, const unsigned long * list, size_t sz); }; |