diff options
384 files changed, 4049 insertions, 4806 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b1934a8a08..3598d259144 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,7 +65,6 @@ add_subdirectory(configutil) add_subdirectory(container-apache-http-client-bundle) add_subdirectory(container-core) add_subdirectory(container-disc) -add_subdirectory(container-jersey2) add_subdirectory(container-messagebus) add_subdirectory(container-search) add_subdirectory(container-search-gui) diff --git a/application/pom.xml b/application/pom.xml index c0bef61f57d..f671c16c5a6 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -83,12 +83,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>jersey-resources</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> <!-- All dependencies that should be visible in test classpath, but not compile classpath, for user projects must be added in compile scope here. @@ -183,31 +177,6 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy</id> - <phase>process-test-resources</phase> - <goals> - <goal>copy</goal> - </goals> - <configuration> - <artifactItems> - <artifactItem> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>jersey-resources</artifactId> - <version>${project.version}</version> - <type>jar</type> - <outputDirectory>target/test-jars</outputDirectory> - <destFileName>jersey-resources.jar</destFileName> - </artifactItem> - </artifactItems> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkCount>2</forkCount> diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java index 63347c2475a..29f895ce176 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java @@ -35,7 +35,7 @@ import static org.junit.Assert.fail; public class ContainerTest { @Test - public void jdisc_can_be_used_as_top_level_element() { + public void container_can_be_used_as_top_level_element() { try (JDisc container = fromServicesXml("<container version=\"1.0\">" + // "<search />" + // "</container>", Networking.disable)) { @@ -44,7 +44,7 @@ public class ContainerTest { } @Test - public void jdisc_id_can_be_set() { + public void container_id_can_be_set() { try (JDisc container = fromServicesXml("<container version=\"1.0\" id=\"my-service-id\">" + // "<search />" + // "</container>", Networking.disable)) { @@ -53,7 +53,7 @@ public class ContainerTest { } @Test - public void jdisc_can_be_embedded_in_services_tag() { + public void container_can_be_embedded_in_services_tag() { try (JDisc container = fromServicesXml("<services>" + // "<container version=\"1.0\" id=\"my-service-id\">" + // "<search />" + // @@ -65,7 +65,7 @@ public class ContainerTest { @Test @SuppressWarnings("try") // container is unused inside the try block - public void multiple_jdisc_elements_gives_exception() { + public void multiple_container_elements_gives_exception() { try (JDisc container = fromServicesXml("<services>" + // "<container version=\"1.0\" id=\"id1\" />" + // "<container version=\"1.0\" />" + // diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java deleted file mode 100644 index 18e206c7add..00000000000 --- a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.application.container.ContainerTest; -import com.yahoo.application.container.jersey.resources.TestResource; -import com.yahoo.application.container.jersey.resources.nestedpackage1.NestedTestResource1; -import com.yahoo.application.container.jersey.resources.nestedpackage2.NestedTestResource2; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.osgi.maven.ProjectBundleClassPaths; -import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; -import org.junit.Ignore; -import org.junit.Test; - -import javax.ws.rs.core.UriBuilder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static java.util.Collections.emptyList; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@Ignore // TODO: remove test -public class JerseyTest { - private final Path testJar = Paths.get("target/test-jars/jersey-resources.jar"); - private final String testClassesDirectory = "target/test-classes"; - private final String bundleSymbolicName = "myBundle"; - - private final Set<Class<? extends TestResourceBase>> classPathResources; - private final Set<Class<? extends TestResourceBase>> jarFileResources; - - public JerseyTest() { - classPathResources = new HashSet<>(); - classPathResources.add(TestResource.class); - classPathResources.add(NestedTestResource1.class); - classPathResources.add(NestedTestResource2.class); - - jarFileResources = new HashSet<>(); - jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.TestResource.class); - jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.nestedpackage1.NestedTestResource1.class); - jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2.class); - } - - @Test - public void jersey_resources_on_classpath_can_be_invoked_from_application() throws Exception { - saveMainBundleClassPathMappings(testClassesDirectory); - - with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); - } - - @Test - public void jersey_resources_in_provided_dependencies_can_be_invoked_from_application() throws Exception { - BundleClasspathMapping providedDependency = - new BundleClasspathMapping(bundleSymbolicName, List.of(testClassesDirectory)); - - save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), List.of(providedDependency))); - with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); - } - - @Test - public void jersey_resource_on_classpath_can_be_filtered_using_packages() throws Exception { - saveMainBundleClassPathMappings(testClassesDirectory); - - with_jersey_resources(Arrays.asList("com.yahoo.application.container.jersey.resources", - "com.yahoo.application.container.jersey.resources.nestedpackage1"), httpGetter -> { - Class<NestedTestResource2> nestedResource2 = NestedTestResource2.class; - assertDoesNotRespond(nestedResource2, httpGetter); - assertResourcesResponds(copySetExcept(classPathResources, nestedResource2), httpGetter); - }); - } - - @Test - public void jersey_resource_in_jar_can_be_invoked_from_application() throws Exception { - saveMainBundleJarClassPathMappings(testJar); - - with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(jarFileResources, httpGetter)); - } - - @Test - public void jersey_resource_in_jar_can_be_filtered_using_packages() throws Exception { - saveMainBundleJarClassPathMappings(testJar); - - with_jersey_resources(Arrays.asList("com.yahoo.container.test.jars.jersey.resources", - "com.yahoo.container.test.jars.jersey.resources.nestedpackage1"), httpGetter -> { - Class<com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2> nestedResource2 = com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2.class; - - assertDoesNotRespond(nestedResource2, httpGetter); - assertResourcesResponds(copySetExcept(jarFileResources, nestedResource2), httpGetter); - }); - } - - private static <T> Set<T> copySetExcept(Set<T> in, T except) { - Set<T> ret = new HashSet<>(in); - ret.remove(except); - return ret; - } - - private interface ThrowingConsumer<T> { - void accept(T arg) throws Exception; - } - - private interface HttpGetter { - HttpResponse get(String path) throws Exception; - } - - @SuppressWarnings("try") // jdisc unreferenced inside try - private void with_jersey_resources(List<String> packagesToScan, ThrowingConsumer<HttpGetter> f) throws Exception { - StringBuilder packageElements = new StringBuilder(); - for (String p : packagesToScan) { - packageElements.append("<package>"); - packageElements.append(p); - packageElements.append("</package>"); - } - - try (JDisc jdisc = JDisc.fromServicesXml( - "<services>" + // - "<container version=\"1.0\" id=\"default\" jetty=\"true\">" + // - "<rest-api path=\"rest-api\" jersey2=\"true\">" + // - "<components bundle=\"" + bundleSymbolicName + "\">" + // - packageElements + // - "</components>" + // - "</rest-api>" + // - "<http>" + // - "<server id=\"mainServer\" port=\"0\" />" + // - "</http>" + // - "<accesslog type=\"disabled\" />" + - "</container>" + // - "</services>", // - Networking.enable)) { - final int port = ContainerTest.getListenPort(); - f.accept(path -> { - String p = path.startsWith("/") ? path.substring(1) : path; - CloseableHttpClient client = HttpClientBuilder.create().build(); - return client.execute(new HttpGet("http://localhost:" + port + "/rest-api/" + p)); - }); - } - } - - public void assertResourcesResponds(Collection<Class<? extends TestResourceBase>> resourceClasses, - HttpGetter httpGetter) throws Exception { - for (Class<? extends TestResourceBase> resource : resourceClasses) { - HttpResponse response = httpGetter.get(path(resource)); - assertThat("Failed sending response to " + resource, response.getStatusLine().getStatusCode(), is(200)); - - String content = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); - assertThat(content, is(TestResourceBase.content(resource))); - } - } - - public void assertDoesNotRespond(Class<? extends TestResourceBase> resourceClass, HttpGetter httpGetter) - throws Exception { - HttpResponse response = httpGetter.get(path(resourceClass)); - assertThat(response.getStatusLine().getStatusCode(), is(404)); - EntityUtils.consume(response.getEntity()); - } - - public void saveMainBundleJarClassPathMappings(Path jarFile) throws Exception { - assertTrue("Couldn't find file " + jarFile + ", please remember to run mvn process-test-resources first.", - Files.isRegularFile(jarFile)); - saveMainBundleClassPathMappings(jarFile.toAbsolutePath().toString()); - } - - public void saveMainBundleClassPathMappings(String classPathElement) throws Exception { - BundleClasspathMapping mainBundleClassPathMappings = - new BundleClasspathMapping(bundleSymbolicName, List.of(classPathElement)); - save(new ProjectBundleClassPaths(mainBundleClassPathMappings, emptyList())); - } - - public void save(ProjectBundleClassPaths projectBundleClassPaths) throws Exception { - Path path = Paths.get(testClassesDirectory).resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME); - ProjectBundleClassPaths.save(path, projectBundleClassPaths); - } - - public String path(Class<?> resourceClass) { - return UriBuilder.fromResource(resourceClass).build().toString(); - } -} diff --git a/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java b/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java deleted file mode 100644 index 5b6f1fa9c35..00000000000 --- a/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources; - -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@Path("/test-resource") -public class TestResource extends TestResourceBase { -} diff --git a/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.java b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.java deleted file mode 100644 index d4901995152..00000000000 --- a/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources.nestedpackage1; - -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@Path("/nested-test-resource1") -public class NestedTestResource1 extends TestResourceBase { -} diff --git a/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.java b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.java deleted file mode 100644 index 1763023a533..00000000000 --- a/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources.nestedpackage2; - -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@Path("/nested-test-resource2") -public class NestedTestResource2 extends TestResourceBase { -} diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 855b3afafaf..3c5f96a1fec 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -28,6 +28,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -79,16 +85,6 @@ <scope>provided</scope> </dependency> - <!-- COMPILE --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <!-- TEST --> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/build_settings.cmake b/build_settings.cmake index 10f4c7ff926..8dcef41caff 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -55,6 +55,9 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST else() set(CXX_SPECIFIC_WARN_OPTS "-Wnoexcept -Wsuggest-override -Wnon-virtual-dtor -Wformat-security") if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR + (VESPA_OS_DISTRO STREQUAL "rocky" AND + VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND + VESPA_OS_DISTRO_VERSION VERSION_LESS "9") OR (VESPA_OS_DISTRO STREQUAL "rhel" AND VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND VESPA_OS_DISTRO_VERSION VERSION_LESS "9")) diff --git a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml index 31e21eb18fd..f305d02c4ae 100644 --- a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml +++ b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml @@ -16,8 +16,7 @@ <process-resources>org.apache.maven.plugins:maven-resources-plugin:resources</process-resources> <compile>org.apache.maven.plugins:maven-compiler-plugin:compile</compile> <process-test-resources> - org.apache.maven.plugins:maven-resources-plugin:testResources, - com.yahoo.vespa:bundle-plugin:generate-bundle-classpath-mappings + org.apache.maven.plugins:maven-resources-plugin:testResources </process-test-resources> <test-compile>org.apache.maven.plugins:maven-compiler-plugin:testCompile</test-compile> <test>org.apache.maven.plugins:maven-surefire-plugin:test</test> diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 340123ae659..481211390e0 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -152,7 +152,6 @@ <include>com.yahoo.vespa:container-dev:*:jar:provided</include> <include>com.yahoo.vespa:container-disc:*:jar:provided</include> <include>com.yahoo.vespa:container-documentapi:*:jar:provided</include> - <include>com.yahoo.vespa:container-jersey2:*:jar:provided</include> <include>com.yahoo.vespa:container-messagebus:*:jar:provided</include> <include>com.yahoo.vespa:container-search-and-docproc:*:jar:provided</include> <include>com.yahoo.vespa:container-search:*:jar:provided</include> diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java index 93b21c8166b..b056ba962fa 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java @@ -97,11 +97,10 @@ public class Reindexer { // Keep metrics in sync across cluster controller containers. AtomicReference<Reindexing> reindexing = new AtomicReference<>(database.readReindexing(cluster.name())); - database.writeReindexing(reindexing.get(), cluster.name()); metrics.dump(reindexing.get()); try (Lock lock = database.lockReindexing(cluster.name())) { - reindexing.set(updateWithReady(ready, reindexing.get(), clock.instant())); + reindexing.set(updateWithReady(ready, database.readReindexing(cluster.name()), clock.instant())); database.writeReindexing(reindexing.get(), cluster.name()); metrics.dump(reindexing.get()); @@ -178,8 +177,7 @@ public class Reindexer { sessionShutdown.run(); // Shutdown aborts the session unless already complete, then waits for it to terminate normally. // Only as a last resort will we be interrupted here, and the wait for outstanding replies terminate. - CompletionCode result = control.getResult() != null ? control.getResult().getCode() - : CompletionCode.ABORTED; + CompletionCode result = control.getResult() != null ? control.getResult().getCode() : CompletionCode.ABORTED; switch (result) { default: log.log(WARNING, "Unexpected visitor result '" + control.getResult().getCode() + "'"); diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java index 1b5a685b69c..896b9dfc26e 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java @@ -72,11 +72,11 @@ public class Reindexing { private final Instant startedAt; private final Instant endedAt; - private final ProgressToken progress; + private final String progress; private final State state; private final String message; - Status(Instant startedAt, Instant endedAt, ProgressToken progress, State state, String message) { + Status(Instant startedAt, Instant endedAt, String progress, State state, String message) { this.startedAt = startedAt; this.endedAt = endedAt; this.progress = progress; @@ -100,7 +100,9 @@ public class Reindexing { public Status progressed(ProgressToken progress) { if (state != State.RUNNING) throw new IllegalStateException("Current state must be RUNNING when updating progress"); - return new Status(startedAt, null, requireNonNull(progress), state, null); + synchronized (progress) { + return new Status(startedAt, null, progress.serializeToString(), state, null); + } } /** Returns a copy of this in state HALTED. */ @@ -133,7 +135,7 @@ public class Reindexing { } public Optional<ProgressToken> progress() { - return Optional.ofNullable(progress); + return Optional.ofNullable(progress).map(ProgressToken::fromSerializedString); } public State state() { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index d46ab812aca..67816ad0be1 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -5,7 +5,6 @@ import ai.vespa.reindexing.Reindexing.Status; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.ProgressToken; import com.yahoo.path.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -17,9 +16,10 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; -import java.util.Collection; import java.util.Map; import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toUnmodifiableMap; @@ -31,6 +31,8 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class ReindexingCurator { + private static final Logger log = Logger.getLogger(ReindexingCurator.class.getName()); + private final Curator curator; private final ReindexingSerializer serializer; private final Duration lockTimeout; @@ -57,6 +59,7 @@ public class ReindexingCurator { if (ready.get(type).isBefore(now)) reindexing = reindexing.with(type, Status.ready(now).running().successful(now)); + log.log(Level.INFO, "Creating initial reindexing status at '" + statusPath(cluster) + "'"); writeReindexing(reindexing, cluster); } catch (ReindexingLockException ignored) { @@ -66,11 +69,14 @@ public class ReindexingCurator { } public Reindexing readReindexing(String cluster) { - return curator.getData(statusPath(cluster)).map(serializer::deserialize) - .orElse(Reindexing.empty()); + Reindexing reindexing = curator.getData(statusPath(cluster)).map(serializer::deserialize) + .orElse(Reindexing.empty()); + log.log(Level.FINE, () -> "Read reindexing status '" + reindexing + "' from '" + statusPath(cluster) + "'"); + return reindexing; } public void writeReindexing(Reindexing reindexing, String cluster) { + log.log(Level.FINE, () -> "Writing reindexing status '" + reindexing + "' to '" + statusPath(cluster) + "'"); curator.set(statusPath(cluster), serializer.serialize(reindexing)); } @@ -126,7 +132,7 @@ public class ReindexingCurator { .collect(toUnmodifiableMap(object -> require(TYPE, object, field -> types.getDocumentType(field.asString())), object -> new Status(require(STARTED_MILLIS, object, field -> Instant.ofEpochMilli(field.asLong())), get(ENDED_MILLIS, object, field -> Instant.ofEpochMilli(field.asLong())), - get(PROGRESS, object, field -> ProgressToken.fromSerializedString(field.asString())), + get(PROGRESS, object, field -> field.asString()), require(STATE, object, field -> toState(field.asString())), get(MESSAGE, object, field -> field.asString()))))); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 869be883661..4b0e30e04da 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -99,6 +99,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean throwIfResourceLimitsSpecified() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 308dcf25d2a..62c192e2c99 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -64,6 +64,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowDisableMtls = true; private boolean dryRunOnnxOnSetup = false; private List<X509Certificate> operatorCertificates = Collections.emptyList(); + private double resourceLimitDisk = 0.8; + private double resourceLimitMemory = 0.8; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -107,6 +109,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } + @Override public double resourceLimitDisk() { return resourceLimitDisk; } + @Override public double resourceLimitMemory() { return resourceLimitMemory; } public TestProperties setDryRunOnnxOnSetup(boolean value) { dryRunOnnxOnSetup = value; @@ -260,6 +264,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setResourceLimitDisk(double value) { + this.resourceLimitDisk = value; + return this; + } + + public TestProperties setResourceLimitMemory(double value) { + this.resourceLimitMemory = value; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index af2168545dc..c173b2c11d9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -28,6 +28,7 @@ public class SummaryClass extends Derived { /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; private final boolean rawAsBase64; + private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); @@ -44,6 +45,7 @@ public class SummaryClass extends Derived { public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) { this.deployLogger = deployLogger; this.rawAsBase64 = search.isRawAsBase64(); + this.omitSummaryFeatures = summary.omitSummaryFeatures(); deriveName(summary); deriveFields(search,summary); deriveImplicitFields(summary); @@ -128,7 +130,8 @@ public class SummaryClass extends Derived { } classBuilder. id(id). - name(getName()); + name(getName()). + omitsummaryfeatures(omitSummaryFeatures); for (SummaryClassField field : fields.values() ) { classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). name(field.getName()). diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java new file mode 100644 index 00000000000..7ee22c3fd23 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validates attribute fields using bool type, ensuring the collection type is supported. + * + * Currently, only the single value bool type is supported. + * + * @author geirst + */ +public class BoolAttributeValidator extends Processor { + + public BoolAttributeValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var field : search.allConcreteFields()) { + var attribute = field.getAttribute(); + if (attribute == null) { + continue; + } + if (attribute.getType().equals(Attribute.Type.BOOL) && + !attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE)) { + fail(search, field, "Only single value bool attribute fields are supported"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 136d352ece7..750842d398c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -78,6 +78,7 @@ public class Processing { OnnxModelConfigGenerator::new, OnnxModelTypeResolver::new, RankingExpressionTypeResolver::new, + BoolAttributeValidator::new, // These should be last: IndexingValidation::new, IndexingValues::new); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 3c6aa881af4..454d0f08ff0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -14,6 +14,7 @@ import java.util.List; public class DocumentSummary extends FieldView { private boolean fromDisk = false; + private boolean omitSummaryFeatures = false; private DocumentSummary inherited; /** @@ -30,6 +31,14 @@ public class DocumentSummary extends FieldView { /** Returns whether the user has noted explicitly that this summary accesses disk */ public boolean isFromDisk() { return fromDisk; } + public void setOmitSummaryFeatures(boolean value) { + omitSummaryFeatures = value; + } + + public boolean omitSummaryFeatures() { + return omitSummaryFeatures; + } + /** * The model is constrained to ensure that summary fields of the same name * in different classes have the same summary transform, because this is diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 1743d5f4432..65b70bdce33 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -322,6 +322,7 @@ public class VespaMetricSet { metrics.add(new Metric("documents_total.count")); metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); + addMetric(metrics, "jdisc.search.render_latency", Set.of("min", "max", "count", "sum", "last")); metrics.add(new Metric("totalhits_per_query.max")); metrics.add(new Metric("totalhits_per_query.sum")); @@ -783,7 +784,7 @@ public class VespaMetricSet { return metrics; } - private static void addMetric(Set<Metric> metrics, String metricName, List<String> aggregateSuffices) { + private static void addMetric(Set<Metric> metrics, String metricName, Iterable<String> aggregateSuffices) { for (String suffix : aggregateSuffices) { metrics.add(new Metric(metricName + "." + suffix)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 4cc8439566e..a6704481734 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -48,7 +48,8 @@ public class ComplexAttributeFieldsValidator extends Validator { if (!unsupportedFields.isEmpty()) { throw new IllegalArgumentException( String.format("For cluster '%s', search '%s': The following complex fields do not support using struct field attributes: %s. " + - "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types", + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types. " + + "The supported primitive types are: byte, int, long, float, double and string", clusterName, search.getName(), unsupportedFields)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 317ed0f66c7..59e9b1a2b4d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -34,8 +34,6 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; -import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; -import com.yahoo.vespa.model.container.jersey.RestApi; import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.utils.FileSender; @@ -80,7 +78,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final ConfigProducerGroup<Servlet> servletGroup; - private final ConfigProducerGroup<RestApi> restApiGroup; private final Set<String> previousHosts; private ContainerModelEvaluation modelEvaluation; @@ -95,7 +92,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true); this.tlsClientAuthority = deployState.tlsClientAuthority(); - restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); previousHosts = deployState.getPreviousModel().stream() .map(Model::allocatedHosts) @@ -121,8 +117,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (modelEvaluation != null) modelEvaluation.prepare(containers); sendUserConfiguredFiles(deployState); - for (RestApi restApi : restApiGroup.getComponents()) - restApi.prepare(); } private void addAndSendApplicationBundles(DeployState deployState) { @@ -166,15 +160,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat this.modelEvaluation = modelEvaluation; } - public final void addRestApi(RestApi restApi) { - restApiGroup.addComponent(ComponentId.fromString(restApi.getBindingPath()), restApi); - } - - public Map<ComponentId, RestApi> getRestApiMap() { - return restApiGroup.getComponentMap(); - } - - public Map<ComponentId, Servlet> getServletMap() { return servletGroup.getComponentMap(); } @@ -183,18 +168,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat servletGroup.addComponent(servlet.getGlobalComponentId(), servlet); } - // Returns all servlets, including rest-api/jersey servlets. public Collection<Servlet> getAllServlets() { return allServlets().collect(Collectors.toCollection(ArrayList::new)); } private Stream<Servlet> allServlets() { - return Stream.concat(allJersey2Servlets(), - servletGroup.getComponents().stream()); - } - - private Stream<Jersey2Servlet> allJersey2Servlets() { - return restApiGroup.getComponents().stream().map(RestApi::getJersey2Servlet); + return servletGroup.getComponents().stream(); } public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 37bfb8821c3..65247f29281 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -10,7 +10,9 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.xml.PlatformBundles; +import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -26,17 +28,21 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String BUNDLE_NAME = "model-evaluation"; + private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; private final static String REST_BINDING_PATH = "/model-evaluation/v1"; + public static final Path MODEL_EVALUATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(EVALUATION_BUNDLE_NAME); + public static final Path MODEL_INTEGRATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(INTEGRATION_BUNDLE_NAME); + /** Global rank profiles, aka models */ private final RankProfileList rankProfileList; public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); - cluster.addSimpleComponent(EVALUATOR_NAME, null, BUNDLE_NAME); + cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME); cluster.addComponent(ContainerModelEvaluation.getHandler()); } @@ -64,7 +70,7 @@ public class ContainerModelEvaluation implements } public static Handler<?> getHandler() { - Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); + Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); handler.addServerBindings( SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 89f200698fa..b25463b8547 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -91,11 +90,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { if (!tlsCiphersOverride.isEmpty()) { connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride); } else { - // Add TLS_RSA_WITH_AES_256_GCM_SHA384 cipher to list of default allowed ciphers - // TODO Remove TLS_RSA_WITH_AES_256_GCM_SHA384 as it's weak and incompatible with HTTP/2 - Set<String> ciphers = new HashSet<>(TlsContext.ALLOWED_CIPHER_SUITES); - ciphers.add("TLS_RSA_WITH_AES_256_GCM_SHA384"); - connectorBuilder.ssl.enabledCipherSuites(Set.copyOf(ciphers)); + connectorBuilder.ssl.enabledCipherSuites(Set.copyOf(TlsContext.ALLOWED_CIPHER_SUITES)); } connectorBuilder diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java deleted file mode 100644 index 06775d59654..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.jersey; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.VersionSpecification; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.container.component.Servlet; - -/** - * @author Tony Vaagenes - */ -public class Jersey2Servlet extends Servlet { - - public static final String BUNDLE = "container-jersey2"; - public static final String CLASS = "com.yahoo.container.servlet.jersey.JerseyServletProvider"; - public static final String BINDING_SUFFIX = "/*"; - - private static final ComponentId REST_API_NAMESPACE = ComponentId.fromString("rest-api"); - - public Jersey2Servlet(String bindingPath) { - super(new ComponentModel( - new BundleInstantiationSpecification(idSpecFromPath(bindingPath), - ComponentSpecification.fromString(CLASS), - ComponentSpecification.fromString(BUNDLE))), - bindingPath + BINDING_SUFFIX); - } - - private static ComponentSpecification idSpecFromPath(String path) { - return new ComponentSpecification(RestApi.idFromPath(path), - VersionSpecification.emptyVersionSpecification, - REST_API_NAMESPACE); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java deleted file mode 100644 index be8209bcc4e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.jersey; - -import com.yahoo.config.model.producer.AbstractConfigProducer; - -/** - * Represents a rest-api - * - * @author gjoranv - */ -public class RestApi extends AbstractConfigProducer<AbstractConfigProducer<?>> { - - private final String bindingPath; - private final Jersey2Servlet jerseyServlet; - private RestApiContext restApiContext; - - public RestApi(String bindingPath) { - super(idFromPath(bindingPath)); - this.bindingPath = bindingPath; - - jerseyServlet = createJersey2Servlet(this.bindingPath); - addChild(jerseyServlet); - } - - public static String idFromPath(String path) { - return path.replaceAll("/", "|"); - } - - private Jersey2Servlet createJersey2Servlet(String bindingPath) { - return new Jersey2Servlet(bindingPath); - } - - public String getBindingPath() { - return bindingPath; - } - - public void setRestApiContext(RestApiContext restApiContext) { - this.restApiContext = restApiContext; - addChild(restApiContext); - jerseyServlet.inject(restApiContext); - } - - public RestApiContext getContext() { return restApiContext; } - - public Jersey2Servlet getJersey2Servlet() { - return jerseyServlet; - } - - public void prepare() { - restApiContext.prepare(); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java deleted file mode 100644 index 390539b643a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.jersey; - -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.di.config.JerseyBundlesConfig; -import com.yahoo.container.di.config.JerseyInjectionConfig; -import com.yahoo.container.di.config.JerseyInjectionConfig.Inject; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; -import java.util.logging.Logger; - -/** - * @author gjoranv - */ -public class RestApiContext extends SimpleComponent implements - JerseyBundlesConfig.Producer, - JerseyInjectionConfig.Producer -{ - private static final Logger log = Logger.getLogger(RestApi.class.getName()); - public static final String CONTAINER_CLASS = "com.yahoo.container.di.config.RestApiContext"; - - private final List<BundleInfo> bundles = new ArrayList<>(); - - // class name -> componentId - private final Map<String, String> injectComponentForClass = new LinkedHashMap<>(); - - private final String bindingPath; - - private ApplicationContainerCluster containerCluster; - - public RestApiContext(AbstractConfigProducer<?> ancestor, String bindingPath) { - super(componentModel(bindingPath)); - this.bindingPath = bindingPath; - - if (ancestor instanceof ApplicationContainerCluster) - containerCluster = (ApplicationContainerCluster)ancestor; - - } - - private static ComponentModel componentModel(String bindingPath) { - return new ComponentModel(BundleInstantiationSpecification.getFromStrings( - CONTAINER_CLASS + "-" + RestApi.idFromPath(bindingPath), - CONTAINER_CLASS, - null)); - } - - @Override - public void getConfig(JerseyBundlesConfig.Builder builder) { - builder.bundles(createBundlesConfig(bundles)); - } - - private List<JerseyBundlesConfig.Bundles.Builder> createBundlesConfig(List<BundleInfo> bundles) { - List<JerseyBundlesConfig.Bundles.Builder> builders = new ArrayList<>(); - for (BundleInfo b : bundles) { - builders.add( - new JerseyBundlesConfig.Bundles.Builder() - .spec(b.spec) - .packages(b.getPackagesToScan()) - ); - } - return builders; - } - - public void addBundles(Collection<BundleInfo> newBundles) { - bundles.addAll(newBundles); - } - - @Override - public void getConfig(JerseyInjectionConfig.Builder builder) { - for (Map.Entry<String, String> i : injectComponentForClass.entrySet()) { - builder.inject(new Inject.Builder() - .forClass(i.getKey()) - .instance(i.getValue())); - } - } - - @Override - public void validate() throws Exception { - super.validate(); - - if (bundles.isEmpty()) - log.warning("No bundles in rest-api '" + bindingPath + - "' - components will only be loaded from classpath."); - } - - public void prepare() { - if (containerCluster == null) return; - - containerCluster.getAllComponents().stream(). - filter(isCycleGeneratingComponent.negate()). - forEach(this::inject); - } - - - /* - * Example problem - * - * RestApiContext -> ApplicationStatusHandler -> ComponentRegistry<HttpServer> -> JettyHttpServer -> ComponentRegistry<Jersey2Servlet> -> RestApiContext - */ - private Predicate<Component> isCycleGeneratingComponent = component -> { - switch (component.getClassId().getName()) { - case CONTAINER_CLASS: - case Jersey2Servlet.CLASS: - case "com.yahoo.jdisc.http.server.jetty.JettyHttpServer": - case "com.yahoo.container.handler.observability.ApplicationStatusHandler": - return true; - default: - return false; - } - }; - - public static class BundleInfo { - // SymbolicName[:Version] - public final String spec; - - private final List<String> packagesToScan = new ArrayList<>(); - - public BundleInfo(String spec) { - this.spec = spec; - } - - public List<String> getPackagesToScan() { - return packagesToScan; - } - - public void addPackageToScan(String pkg) { - packagesToScan.add(pkg); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java deleted file mode 100644 index 4aa5882119f..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.jersey.xml; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.container.jersey.RestApi; -import com.yahoo.vespa.model.container.jersey.RestApiContext; -import org.w3c.dom.Element; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * @author gjoranv - * @since 5.6 - */ -public class RestApiBuilder extends VespaDomBuilder.DomConfigProducerBuilder<RestApi> { - - @Override - protected RestApi doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { - String bindingPath = spec.getAttribute("path"); - RestApi restApi = new RestApi(bindingPath); - - restApi.setRestApiContext(createRestApiContext(ancestor, spec, bindingPath)); - return restApi; - } - - private RestApiContext createRestApiContext(AbstractConfigProducer ancestor, Element spec, String bindingPath) { - RestApiContext restApiContext = new RestApiContext(ancestor, bindingPath); - - restApiContext.addBundles(getBundles(spec)); - - return restApiContext; - } - - private List<RestApiContext.BundleInfo> getBundles(Element spec) { - List<RestApiContext.BundleInfo> bundles = new ArrayList<>(); - for (Element bundleElement : XML.getChildren(spec, "components")) { - bundles.add(getBundle(bundleElement)); - } - return bundles; - } - - private RestApiContext.BundleInfo getBundle(Element bundleElement) { - RestApiContext.BundleInfo bundle = new RestApiContext.BundleInfo(bundleElement.getAttribute("bundle")); - - for (Element packageElement : XML.getChildren(bundleElement, "package")) - bundle.addPackageToScan(XML.getValue(packageElement)); - - return bundle; - } - - // TODO: use for naming injected components instead - private Map<String, String> getInjections(Element spec) { - Map<String, String> injectForClass = new LinkedHashMap<>(); - for (Element injectElement : XML.getChildren(spec, "inject")) { - injectForClass.put(injectElement.getAttribute("for-class"), - injectElement.getAttribute("component")); - } - return injectForClass; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index a37d0ef416f..a1f52cca9fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; @@ -61,6 +62,11 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); } + @Override + protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + // Model evaluation bundles are pre-installed in the standalone container. + } + /** Note: using {@link CloudConfigOptions} as {@link DeployState#isHosted()} returns <em>false</em> for hosted configserver/controller */ private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3b04d536300..4b45979c698 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -184,9 +184,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(deployState, cluster, spec); addSecretStore(cluster, spec, deployState); - throwUponRestApi(spec); // TODO: remove addServlets(deployState, spec, cluster); addModelEvaluation(spec, cluster, context); + addModelEvaluationBundles(cluster); addProcessing(deployState, spec, cluster); addSearch(deployState, spec, cluster); @@ -516,12 +516,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return http; } - private void throwUponRestApi(Element spec) { - if(! XML.getChildren(spec, "rest-api").isEmpty()) { - throw new IllegalArgumentException("The 'rest-api' element is no longer allowed in services.xml."); - } - } - private void addServlets(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { for (Element servletElem : XML.getChildren(spec, "servlet")) cluster.addServlet(new ServletBuilder().build(deployState, cluster, servletElem)); @@ -565,6 +559,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles)); } + protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + /* These bundles are added to all application container clusters, even if they haven't + * declared 'model-evaluation' in services.xml, because there are many public API packages + * in the model-evaluation bundle that could be used by customer code. */ + cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE); + cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE); + } + private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 638864d85bb..dc5e6c9baee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -39,15 +39,21 @@ public class ClusterResourceLimits { private final boolean hostedVespa; private final boolean throwIfSpecified; private final DeployLogger deployLogger; + private final double resourceLimitDisk; + private final double resourceLimitMemory; private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified, DeployLogger deployLogger) { + public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified, + DeployLogger deployLogger, double resourceLimitDisk, double resourceLimitMemory) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; this.throwIfSpecified = throwIfSpecified; this.deployLogger = deployLogger; + this.resourceLimitDisk = resourceLimitDisk; + this.resourceLimitMemory = resourceLimitMemory; + verifyLimits(resourceLimitDisk, resourceLimitMemory); } public ClusterResourceLimits build(ModelElement clusterElem) { @@ -80,8 +86,14 @@ public class ClusterResourceLimits { private void deriveLimits() { if (enableFeedBlockInDistributor) { // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), + nodeBuilder.getDiskLimit(), + ctrlBuilder::setDiskLimit, + resourceLimitDisk); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), + nodeBuilder.getMemoryLimit(), + ctrlBuilder::setMemoryLimit, + resourceLimitMemory); } deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); @@ -93,10 +105,11 @@ public class ClusterResourceLimits { private void considerSettingDefaultClusterControllerLimit(Optional<Double> clusterControllerLimit, Optional<Double> contentNodeLimit, - Consumer<Double> setter) { + Consumer<Double> setter, + double resourceLimit) { // TODO: remove this when feed block in distributor is default enabled. if (clusterControllerLimit.isEmpty() && contentNodeLimit.isEmpty()) { - setter.accept(0.8); + setter.accept(resourceLimit); } } @@ -120,10 +133,25 @@ public class ClusterResourceLimits { } private double calcContentNodeLimit(double clusterControllerLimit) { - // Note that validation in the range [0.0-1.0] is handled by the rnc schema. return clusterControllerLimit + ((1.0 - clusterControllerLimit) / 2); } + + private void verifyLimits(double resourceLimitDisk, double resourceLimitMemory) { + verifyLimitInRange(resourceLimitDisk, "disk"); + verifyLimitInRange(resourceLimitMemory, "memory"); + } + + private void verifyLimitInRange(double limit, String type) { + String message = "Resource limit for " + type + " is set to illegal value " + limit + + ", but must be in the range [0.0, 1.0]"; + if (limit < 0.0) + throw new IllegalArgumentException(message); + + if (limit > 1.0) + throw new IllegalArgumentException(message); + } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index c298b7f5f5a..fabcbbcc9ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -124,7 +124,9 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, stateIsHosted(deployState), deployState.featureFlags().throwIfResourceLimitsSpecified(), - deployState.getDeployLogger()) + deployState.getDeployLogger(), + deployState.featureFlags().resourceLimitDisk(), + deployState.featureFlags().resourceLimitMemory()) .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 37adb73bc15..32b0f5b6477 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -24,10 +24,12 @@ public class DomResourceLimitsBuilder { if (hostedVespa) { String message = "Element '" + resourceLimits + "' is not allowed to be set"; - if (throwIfSpecified) - throw new IllegalArgumentException(message); - else - deployLogger.logApplicationPackage(Level.WARNING, message); + if (throwIfSpecified) throw new IllegalArgumentException(message); + + + deployLogger.logApplicationPackage(Level.WARNING, message); + // TODO: return (default values will then be used). Cannot be done now as an app needs current behavior + //return builder; } if (resourceLimits.child("disk") != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java index 2e316bfd2c1..b86a4b6fbff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -12,6 +12,7 @@ import java.util.TreeMap; * @author Thomas Gundersen */ public class SearchDefinitionBuilder { + public Map<String, NewDocumentType> build(DocumentTypeRepo repo, ModelElement elem) { Map<String, NewDocumentType> docTypes = new TreeMap<>(); diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index c247100db76..fb24b50aa99 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -291,6 +291,7 @@ TOKEN : | < TO: "to" > | < DIRECT: "direct" > | < FROMDISK: "from-disk" > +| < OMITSUMMARYFEATURES: "omit-summary-features" > | < ALWAYS: "always" > | < ONDEMAND: "on-demand" > | < NEVER: "never" > @@ -1767,6 +1768,7 @@ Object documentSummary(Search search) : lbrace() ( <FROMDISK> { summary.setFromDisk(true); } | + <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } | documentSummaryItem(summary) | <NL> )* diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 3679e53e257..992689a2189 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -12,7 +12,6 @@ ContainerCluster = element container | jdisc { } ContainerServices = - RestApi* & Servlet* & SearchInContainer? & DocprocInContainer? & @@ -128,22 +127,6 @@ Threadpool = element threadpool { element queue-size { xsd:nonNegativeInteger } } -# REST-API: - -RestApi = element rest-api { - attribute path { xsd:string { pattern = "\w[\w_/\-\.]*" } } & - attribute jersey2 { xsd:boolean }? & - element components { - attribute bundle { xsd:Name } & - element package { xsd:NCName }* - }* - -# element inject { -# attribute component { xsd:Name } & -# attribute for-name { xsd:Name } -# }* -} - # Servlet: Servlet = element servlet { diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg index a56d0e6aa4f..f497461b460 100644 --- a/config-model/src/test/derived/advanced/summary.cfg +++ b/config-model/src/test/derived/advanced/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1271952241 classes[].id 1271952241 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "debug" classes[].fields[].type "longstring" classes[].fields[].name "attributes" @@ -25,6 +26,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 472092010 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "location_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index c1679c57d1a..965c875d5ce 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 252850086 classes[].id 252850086 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "elem_array" classes[].fields[].type "jsonstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg index 13b17464946..f0189f9a3c7 100644 --- a/config-model/src/test/derived/attributeprefetch/summary.cfg +++ b/config-model/src/test/derived/attributeprefetch/summary.cfg @@ -1,27 +1,29 @@ defaultsummaryid 1151071433 -classes[0].id 1151071433 -classes[0].name "default" -classes[0].fields[0].name "rankfeatures" -classes[0].fields[0].type "featuredata" -classes[0].fields[1].name "summaryfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "documentid" -classes[0].fields[2].type "longstring" -classes[1].id 1980470965 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "singlebyte" -classes[1].fields[0].type "byte" -classes[1].fields[1].name "singleint" -classes[1].fields[1].type "integer" -classes[1].fields[2].name "singlelong" -classes[1].fields[2].type "int64" -classes[1].fields[3].name "singlefloat" -classes[1].fields[3].type "float" -classes[1].fields[4].name "singledouble" -classes[1].fields[4].type "double" -classes[1].fields[5].name "singlestring" -classes[1].fields[5].type "longstring" -classes[1].fields[6].name "rankfeatures" -classes[1].fields[6].type "featuredata" -classes[1].fields[7].name "summaryfeatures" -classes[1].fields[7].type "featuredata"
\ No newline at end of file +classes[].id 1151071433 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 1980470965 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "singlebyte" +classes[].fields[].type "byte" +classes[].fields[].name "singleint" +classes[].fields[].type "integer" +classes[].fields[].name "singlelong" +classes[].fields[].type "int64" +classes[].fields[].name "singlefloat" +classes[].fields[].type "float" +classes[].fields[].name "singledouble" +classes[].fields[].type "double" +classes[].fields[].name "singlestring" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg index 5bb472b5f41..2dac4736d23 100644 --- a/config-model/src/test/derived/complex/summary.cfg +++ b/config-model/src/test/derived/complex/summary.cfg @@ -1,41 +1,43 @@ defaultsummaryid 1506848752 -classes[0].id 1506848752 -classes[0].name "default" -classes[0].fields[0].name "woe" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "exact" -classes[0].fields[1].type "longstring" -classes[0].fields[2].name "title" -classes[0].fields[2].type "longstring" -classes[0].fields[3].name "dyntitle" -classes[0].fields[3].type "longstring" -classes[0].fields[4].name "source" -classes[0].fields[4].type "longstring" -classes[0].fields[5].name "stringfield" -classes[0].fields[5].type "longstring" -classes[0].fields[6].name "rankfeatures" -classes[0].fields[6].type "featuredata" -classes[0].fields[7].name "summaryfeatures" -classes[0].fields[7].type "featuredata" -classes[0].fields[8].name "documentid" -classes[0].fields[8].type "longstring" -classes[1].id 128090024 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "year_sub" -classes[1].fields[0].type "integer" -classes[1].fields[1].name "prefixenabled" -classes[1].fields[1].type "longstring" -classes[1].fields[2].name "fleeting2" -classes[1].fields[2].type "float" -classes[1].fields[3].name "foundat" -classes[1].fields[3].type "int64" -classes[1].fields[4].name "collapseby" -classes[1].fields[4].type "integer" -classes[1].fields[5].name "ts" -classes[1].fields[5].type "int64" -classes[1].fields[6].name "combineda" -classes[1].fields[6].type "integer" -classes[1].fields[7].name "rankfeatures" -classes[1].fields[7].type "featuredata" -classes[1].fields[8].name "summaryfeatures" -classes[1].fields[8].type "featuredata"
\ No newline at end of file +classes[].id 1506848752 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "woe" +classes[].fields[].type "longstring" +classes[].fields[].name "exact" +classes[].fields[].type "longstring" +classes[].fields[].name "title" +classes[].fields[].type "longstring" +classes[].fields[].name "dyntitle" +classes[].fields[].type "longstring" +classes[].fields[].name "source" +classes[].fields[].type "longstring" +classes[].fields[].name "stringfield" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 128090024 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "year_sub" +classes[].fields[].type "integer" +classes[].fields[].name "prefixenabled" +classes[].fields[].type "longstring" +classes[].fields[].name "fleeting2" +classes[].fields[].type "float" +classes[].fields[].name "foundat" +classes[].fields[].type "int64" +classes[].fields[].name "collapseby" +classes[].fields[].type "integer" +classes[].fields[].name "ts" +classes[].fields[].type "int64" +classes[].fields[].name "combineda" +classes[].fields[].type "integer" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg index ed3a61a5de5..82bed7fd55e 100644 --- a/config-model/src/test/derived/emptychild/summary.cfg +++ b/config-model/src/test/derived/emptychild/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1814603381 classes[].id 1814603381 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1490368133 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg index e47f24b21c3..61294d97b4c 100644 --- a/config-model/src/test/derived/emptydefault/summary.cfg +++ b/config-model/src/test/derived/emptydefault/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1151071433 classes[].id 1151071433 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg index dbc9a90ebce..b50b970afe2 100644 --- a/config-model/src/test/derived/id/summary.cfg +++ b/config-model/src/test/derived/id/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1814716401 classes[].id 1814716401 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "uri" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg index 8da2598a5fc..3ab8e7e29e5 100644 --- a/config-model/src/test/derived/imported_position_field/summary.cfg +++ b/config-model/src/test/derived/imported_position_field/summary.cfg @@ -1,17 +1,19 @@ defaultsummaryid 1570252291 -classes[0].id 1570252291 -classes[0].name "default" -classes[0].fields[0].name "parent_ref" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "rankfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "summaryfeatures" -classes[0].fields[2].type "featuredata" -classes[0].fields[3].name "documentid" -classes[0].fields[3].type "longstring" -classes[1].id 1274088866 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "rankfeatures" -classes[1].fields[0].type "featuredata" -classes[1].fields[1].name "summaryfeatures" -classes[1].fields[1].type "featuredata"
\ No newline at end of file +classes[].id 1570252291 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "parent_ref" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 1274088866 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg index b607786ca36..76faac23170 100644 --- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1194448774 classes[].id 1194448774 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "parent_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 890647799 classes[].name "mysummary" +classes[].omitsummaryfeatures false classes[].fields[].name "my_pos" classes[].fields[].type "jsonstring" classes[].fields[].name "rankfeatures" @@ -29,6 +31,7 @@ classes[].fields[].name "my_pos.distance" classes[].fields[].type "integer" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg index 3a9bf4f5e0a..ab6c6853925 100644 --- a/config-model/src/test/derived/imported_struct_fields/summary.cfg +++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg @@ -1,43 +1,47 @@ defaultsummaryid 1570252291 -classes[0].id 1570252291 -classes[0].name "default" -classes[0].fields[0].name "parent_ref" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "rankfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "summaryfeatures" -classes[0].fields[2].type "featuredata" -classes[0].fields[3].name "documentid" -classes[0].fields[3].type "longstring" -classes[1].id 2126652894 -classes[1].name "mysummary" -classes[1].fields[0].name "documentid" -classes[1].fields[0].type "longstring" -classes[1].fields[1].name "my_elem_array" -classes[1].fields[1].type "jsonstring" -classes[1].fields[2].name "my_elem_map" -classes[1].fields[2].type "jsonstring" -classes[1].fields[3].name "my_str_int_map" -classes[1].fields[3].type "jsonstring" -classes[1].fields[4].name "rankfeatures" -classes[1].fields[4].type "featuredata" -classes[1].fields[5].name "summaryfeatures" -classes[1].fields[5].type "featuredata" -classes[2].id 1629947863 -classes[2].name "filtered" -classes[2].fields[0].name "elem_array_filtered" -classes[2].fields[0].type "jsonstring" -classes[2].fields[1].name "elem_map_filtered" -classes[2].fields[1].type "jsonstring" -classes[2].fields[2].name "str_int_map_filtered" -classes[2].fields[2].type "jsonstring" -classes[2].fields[3].name "rankfeatures" -classes[2].fields[3].type "featuredata" -classes[2].fields[4].name "summaryfeatures" -classes[2].fields[4].type "featuredata" -classes[3].id 1274088866 -classes[3].name "attributeprefetch" -classes[3].fields[0].name "rankfeatures" -classes[3].fields[0].type "featuredata" -classes[3].fields[1].name "summaryfeatures" -classes[3].fields[1].type "featuredata"
\ No newline at end of file +classes[].id 1570252291 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "parent_ref" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 2126652894 +classes[].name "mysummary" +classes[].omitsummaryfeatures false +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].fields[].name "my_elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "my_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "my_str_int_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1629947863 +classes[].name "filtered" +classes[].omitsummaryfeatures false +classes[].fields[].name "elem_array_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "elem_map_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "str_int_map_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1274088866 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg index f95949cfa62..74b5b44214e 100644 --- a/config-model/src/test/derived/importedfields/summary.cfg +++ b/config-model/src/test/derived/importedfields/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1294344677 classes[].id 1294344677 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "b_ref_with_summary" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 159551552 classes[].name "mysummary" +classes[].omitsummaryfeatures false classes[].fields[].name "a_ref" classes[].fields[].type "longstring" classes[].fields[].name "b_ref_with_summary" @@ -33,6 +35,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg index 5bdc8fcdef4..d04bc4eb167 100644 --- a/config-model/src/test/derived/indexswitches/summary.cfg +++ b/config-model/src/test/derived/indexswitches/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1698765342 classes[].id 1698765342 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "source" classes[].fields[].type "longstring" classes[].fields[].name "title" diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg index dde71a1378c..dde9f95ecbe 100644 --- a/config-model/src/test/derived/inheritance/summary.cfg +++ b/config-model/src/test/derived/inheritance/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1797992819 classes[].id 1797992819 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "onlyfather" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1608562186 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "onlygrandparent" classes[].fields[].type "integer" classes[].fields[].name "overridden" diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg index 267585e3fda..d5eb316ff01 100644 --- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg +++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1195656216 classes[].id 1195656216 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" classes[].fields[].name "artist" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1706878063 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" classes[].fields[].name "artist" diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg index 24d6cab7697..b465bdfa541 100644 --- a/config-model/src/test/derived/map_attribute/summary.cfg +++ b/config-model/src/test/derived/map_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1376056200 classes[].id 1376056200 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "str_map" classes[].fields[].type "jsonstring" classes[].fields[].name "int_map" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index f70025c8f02..67988dbf30e 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1131098132 classes[].id 1131098132 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "str_elem_map" classes[].fields[].type "jsonstring" classes[].fields[].name "int_elem_map" diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg index cb5bf17df84..b6a53a9a1d9 100644 --- a/config-model/src/test/derived/mlr/summary.cfg +++ b/config-model/src/test/derived/mlr/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1868876861 classes[].id 1868876861 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" classes[].fields[].name "b" @@ -13,6 +14,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1944325986 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" classes[].fields[].name "ranklog" diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg index 3eca077dbc8..bc55727b407 100644 --- a/config-model/src/test/derived/music/summary.cfg +++ b/config-model/src/test/derived/music/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 2086497905 classes[].id 2086497905 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" classes[].fields[].name "sales" @@ -79,6 +80,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 2060710706 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" classes[].fields[].name "pto" diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg index 7cd92c26e02..0b98b20c342 100644 --- a/config-model/src/test/derived/newrank/summary.cfg +++ b/config-model/src/test/derived/newrank/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 912980235 classes[].id 912980235 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" classes[].fields[].name "sales" @@ -71,6 +72,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1606815285 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" classes[].fields[].name "pto" diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index fad012393ef..4222e88cc2f 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1727020212 classes[].id 1727020212 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "pos.position" classes[].fields[].type "xmlstring" classes[].fields[].name "pos.distance" @@ -13,6 +14,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1530141163 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index af801f43cc0..f54066d865e 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 230670304 classes[].id 230670304 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "pos" classes[].fields[].type "jsonstring" classes[].fields[].name "pos.position" @@ -15,6 +16,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1530141163 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg index 6e33bd4e567..9cc613107e0 100644 --- a/config-model/src/test/derived/predicate_attribute/summary.cfg +++ b/config-model/src/test/derived/predicate_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1391971216 classes[].id 1391971216 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "some_predicate_field" classes[].fields[].type "string" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg index f8b56baf8f2..4f417a848a7 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankexpression/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1753207254 classes[].id 1753207254 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "artist" classes[].fields[].type "longstring" classes[].fields[].name "title" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1736696699 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "year" classes[].fields[].type "integer" classes[].fields[].name "foo1" diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg index 9644eb878ea..49b668e9edf 100644 --- a/config-model/src/test/derived/ranktypes/summary.cfg +++ b/config-model/src/test/derived/ranktypes/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1567556360 classes[].id 1567556360 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "title" classes[].fields[].type "longstring" classes[].fields[].name "descr" diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg index 49037473d88..410bccff7b3 100644 --- a/config-model/src/test/derived/reference_fields/summary.cfg +++ b/config-model/src/test/derived/reference_fields/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1987541865 classes[].id 1987541865 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "campaign_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 428144659 classes[].name "explicit_summary" +classes[].omitsummaryfeatures false classes[].fields[].name "yet_another_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -19,6 +21,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg index 28f19e6fe25..655499a88be 100644 --- a/config-model/src/test/derived/streamingstruct/summary.cfg +++ b/config-model/src/test/derived/streamingstruct/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 569269436 classes[].id 569269436 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "coupleof" classes[].fields[].type "longstring" classes[].fields[].name "anothersummaryfield" @@ -41,6 +42,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 109252281 classes[].name "summ" +classes[].omitsummaryfeatures false classes[].fields[].name "snippet" classes[].fields[].type "longstring" classes[].fields[].name "snippet2" diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg index caa44931c6a..a52b34925dc 100644 --- a/config-model/src/test/derived/streamingstructdefault/summary.cfg +++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 718801936 classes[].id 718801936 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "sum1" classes[].fields[].type "longstring" classes[].fields[].name "f1" diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg index fb32eacbb4c..355cba0e561 100644 --- a/config-model/src/test/derived/tensor/summary.cfg +++ b/config-model/src/test/derived/tensor/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 898020074 classes[].id 898020074 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "f1" classes[].fields[].type "tensor" classes[].fields[].name "f3" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1476352352 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "f2" classes[].fields[].type "tensor" classes[].fields[].name "f3" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg new file mode 100644 index 00000000000..ead74110db3 --- /dev/null +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "tokenization" +ilscript[].docfield[] "text" +ilscript[].docfield[] "text_array" +ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }" +ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }" +ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }" +ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }"
\ No newline at end of file diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd new file mode 100644 index 00000000000..4510a574d60 --- /dev/null +++ b/config-model/src/test/derived/tokenization/tokenization.sd @@ -0,0 +1,23 @@ +schema tokenization { + + document tokenization { + + field text type string { + indexing: index | summary + } + + field text_array type array<string> { + indexing: index | summary + } + + } + + field text_derived type string { + indexing: input text | normalize | index | summary + } + + field text_array_derived type array<string> { + indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index e5485a24c8c..e0e67a5669d 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1131946680 classes[].id 1131946680 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "abyte" classes[].fields[].type "byte" classes[].fields[].name "along" @@ -25,6 +26,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1027812395 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "other" classes[].fields[].type "int64" classes[].fields[].name "abyte" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index bfc738a4f87..ef7da4f23d0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.SummaryConfig; import org.junit.Test; import java.io.IOException; @@ -14,6 +15,7 @@ import java.util.Iterator; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Tests summary extraction @@ -151,4 +153,30 @@ public class SummaryTestCase extends SchemaTestCase { return builder.getSearch("ad"); } + @Test + public void omit_summary_features_specified_for_document_summary() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field foo type string { indexing: summary }", + " }", + " document-summary bar {", + " summary foo type string {}", + " omit-summary-features", + " }", + " document-summary baz {", + " summary foo type string {}", + " }", + "}"); + var search = SearchBuilder.createFromString(sd).getSearch(); + assertOmitSummaryFeatures(true, search, "bar"); + assertOmitSummaryFeatures(false, search, "baz"); + } + + private void assertOmitSummaryFeatures(boolean expected, Search search, String summaryName) { + var summary = new SummaryClass(search, search.getSummary(summaryName), new BaseDeployLogger()); + var config = new SummaryConfig.Classes(summary.getSummaryClassConfig()); + assertEquals(expected, config.omitsummaryfeatures()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java new file mode 100755 index 00000000000..6fe367ef6d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java @@ -0,0 +1,19 @@ +// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseh + */ +public class TokenizationTestCase extends AbstractExportingTestCase { + + @Test + public void testTokenizationScripts() throws IOException, ParseException { + assertCorrectDeriving("tokenization"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java new file mode 100644 index 00000000000..663aace7b79 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import static com.yahoo.searchdefinition.SearchBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class BoolAttributeValidatorTestCase { + + @Test + public void array_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type array<bool> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + @Test + public void weigtedset_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type weightedset<bool> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + private String getSd(String field) { + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 2b55d7a3948..2144c3c9a66 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.text.XML; @@ -16,6 +17,7 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; import org.junit.Test; @@ -27,6 +29,8 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -124,6 +128,14 @@ public class ConfigserverClusterTest { assertTrue(config.zookeeperLocalhostAffinity()); } + @Test + public void model_evaluation_bundles_are_not_installed_via_config() { + // These bundles must be pre-installed because they are used by config-model. + PlatformBundlesConfig config = getConfig(PlatformBundlesConfig.class); + assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString()))); + assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString()))); + } + @SuppressWarnings("varargs") private static <T> void assertZookeeperServerProperty( List<ZookeeperServerConfig.Server> zkServers, Function<ZookeeperServerConfig.Server, T> propertyMapper, T... expectedProperties) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/jersey/xml/RestApiTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/jersey/xml/RestApiTest.java deleted file mode 100644 index c858024f749..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/jersey/xml/RestApiTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.jersey.xml; - -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.test.TestUtil; -import com.yahoo.container.ComponentsConfig; -import com.yahoo.container.di.config.JerseyBundlesConfig; -import com.yahoo.jdisc.http.ServletPathsConfig; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; -import com.yahoo.vespa.model.container.jersey.RestApi; -import com.yahoo.vespa.model.container.jersey.RestApiContext; -import com.yahoo.vespa.model.container.xml.ContainerModelBuilderTestBase; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.w3c.dom.Element; - -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; - -/** - * @author gjoranv - * @author bjorncs - */ -@Ignore // TODO: remove test -public class RestApiTest extends ContainerModelBuilderTestBase { - private static final String PATH = "rest/api"; - private static final String REST_API_CONTEXT_ID = RestApiContext.CONTAINER_CLASS + "-" + RestApi.idFromPath(PATH); - private static final String INJECTED_COMPONENT_ID = "injectedHandler"; - private static final String CLUSTER_ID = "container"; - - private static final Element restApiXml = TestUtil.parse( - "<container version=\"1.0\" id=\"" + CLUSTER_ID + "\">", - " <rest-api path=\"" + PATH + "\">", - " <components bundle=\"my-jersey-bundle:1.0\">", - " <package>com.yahoo.foo</package>", - " </components>", - " </rest-api>", - " <handler id=\"" + INJECTED_COMPONENT_ID + "\" />", - "</container>"); - - private RestApi restApi; - private Jersey2Servlet servlet; - private RestApiContext context; - - @Before - public void setup() throws Exception { - createModel(root, restApiXml); - root.validate(); - getContainerCluster(CLUSTER_ID).prepare(root.getDeployState()); - restApi = getContainerCluster(CLUSTER_ID).getRestApiMap().values().iterator().next(); - servlet = restApi.getJersey2Servlet(); - context = restApi.getContext(); - } - - @Test - public void jersey2_servlet_has_correct_binding_path() { - assertThat(servlet, not(nullValue())); - assertThat(servlet.bindingPath, is(PATH + "/*")); - } - - @Test - public void jersey2_servlet_has_correct_bundle_spec() { - assertThat(servlet.model.bundleInstantiationSpec.bundle.stringValue(), is(Jersey2Servlet.BUNDLE)); - } - - @Test - public void rest_api_path_is_included_in_servlet_config() { - ServletPathsConfig config = root.getConfig(ServletPathsConfig.class, servlet.getConfigId()); - assertThat(config.servlets(servlet.getComponentId().stringValue()).path(), is(PATH + "/*")); - } - - @Test - public void resource_bundles_are_included_in_config() { - JerseyBundlesConfig config = root.getConfig(JerseyBundlesConfig.class, context.getConfigId()); - assertThat(config.bundles().size(), is(1)); - assertThat(config.bundles(0).spec(), is("my-jersey-bundle:1.0")); - } - - @Test - public void packages_to_scan_are_included_in_config() { - JerseyBundlesConfig config = root.getConfig(JerseyBundlesConfig.class, context.getConfigId()); - assertThat(config.bundles(0).packages(), contains("com.yahoo.foo")); - } - - @Test - public void jersey2_servlet_is_included_in_components_config() { - ComponentsConfig config = root.getConfig(ComponentsConfig.class, CLUSTER_ID); - assertThat(config.toString(), containsString(".id \"" + servlet.getComponentId().stringValue() + "\"")); - } - - @Test - public void restApiContext_is_included_in_components_config() { - ComponentsConfig config = root.getConfig(ComponentsConfig.class, CLUSTER_ID); - assertThat(config.toString(), containsString(".id \"" + REST_API_CONTEXT_ID + "\"")); - } - - @Test - public void all_non_restApi_components_are_injected_to_RestApiContext() { - ComponentsConfig componentsConfig = root.getConfig(ComponentsConfig.class, CLUSTER_ID); - - Set<ComponentId> clusterChildrenComponentIds = getContainerCluster(CLUSTER_ID).getAllComponents().stream() - .map(Component::getComponentId) - .collect(Collectors.toSet()); - - Set<ComponentId> restApiChildrenComponentIds = restApi.getChildren().values().stream() - .map(child -> ((Component<?, ?>) child).getComponentId()) - .collect(Collectors.toSet()); - - //TODO: try replacing with filtering against RestApiContext.isCycleGeneratingComponent - ComponentId cycleInducingComponents = ComponentId.fromString("com.yahoo.container.handler.observability.ApplicationStatusHandler"); - - Set<ComponentId> expectedInjectedConfigIds = new HashSet<>(clusterChildrenComponentIds); - expectedInjectedConfigIds.removeAll(restApiChildrenComponentIds); - expectedInjectedConfigIds.remove(cycleInducingComponents); - - Set<ComponentId> injectedConfigIds = restApiContextConfig(componentsConfig).inject().stream() - .map(inject -> ComponentId.fromString(inject.id())) - .collect(Collectors.toSet()); - - // Verify that the two sets are equal. Split in two asserts to get decent failure messages. - assertThat( - "Not all required components are injected", - injectedConfigIds, - containsInAnyOrder(expectedInjectedConfigIds.toArray())); - assertThat( - "We inject some components that should not be injected", - expectedInjectedConfigIds, - containsInAnyOrder(injectedConfigIds.toArray())); - } - - private static ComponentsConfig.Components restApiContextConfig(ComponentsConfig config) { - return config.components().stream() - .filter(component -> component.classId().equals(RestApiContext.CONTAINER_CLASS)) - .findFirst() - .get(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 543318f9224..1af89626199 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -16,7 +16,6 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; @@ -27,6 +26,7 @@ import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.VipStatusHandler; import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; @@ -40,10 +40,6 @@ import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.defaults.Defaults; @@ -52,32 +48,23 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; -import org.hamcrest.core.IsEqual; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.w3c.dom.Element; import org.xml.sax.SAXException; -import javax.security.auth.x500.X500Principal; import java.io.IOException; import java.io.StringReader; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -95,7 +82,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -106,7 +92,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -124,6 +109,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { public TemporaryFolder applicationFolder = new TemporaryFolder(); @Test + public void model_evaluation_bundles_are_deployed() { + createBasicContainerModel(); + PlatformBundlesConfig config = root.getConfig(PlatformBundlesConfig.class, "default"); + assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())); + assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())); + } + + @Test public void deprecated_jdisc_tag_is_allowed() { Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", @@ -244,10 +237,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void verify_bindings_for_builtin_handlers() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' />" - ); - createModel(root, clusterElem); + createBasicContainerModel(); JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); JdiscBindingsConfig.Handlers defaultRootHandler = config.handlers(BindingsOverviewHandler.class.getName()); @@ -1014,11 +1004,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { tlsPort.getConfig(builder); ConnectorConfig connectorConfig = new ConnectorConfig(builder); - Set<String> expectedCiphers = new HashSet<>(); - expectedCiphers.add("TLS_RSA_WITH_AES_256_GCM_SHA384"); - expectedCiphers.addAll(TlsContext.ALLOWED_CIPHER_SUITES); - assertThat(connectorConfig.ssl().enabledCipherSuites(), containsInAnyOrder(expectedCiphers.toArray())); + assertThat(connectorConfig.ssl().enabledCipherSuites(), containsInAnyOrder(TlsContext.ALLOWED_CIPHER_SUITES.toArray())); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index 9e02572737e..7034176da14 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.ComponentsConfig; @@ -51,6 +52,11 @@ public abstract class ContainerModelBuilderTestBase { protected MockRoot root; + protected void createBasicContainerModel() { + Element clusterElem = DomBuilderTest.parse("<container id='default' version='1.0' />"); + createModel(root, clusterElem); + } + public static void createModel(MockRoot root, DeployState deployState, VespaModel vespaModel, Element... containerElems) { for (Element containerElem : containerElems) { ContainerModel model = new ContainerModelBuilder(false, ContainerModelBuilder.Networking.enable) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 4324f257922..c0b1a64bace 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.derived.TestableDeployLogger; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -24,15 +27,23 @@ public class ClusterResourceLimitsTest { private static class Fixture { private final boolean enableFeedBlockInDistributor; + private final boolean hostedVespa; + private final boolean throwIfSpecified; private final ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private final ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); public Fixture() { - this.enableFeedBlockInDistributor = false; + this(false); } public Fixture(boolean enableFeedBlockInDistributor) { + this(enableFeedBlockInDistributor, false, false); + } + + public Fixture(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + this.hostedVespa = hostedVespa; + this.throwIfSpecified = throwIfSpecified; } public Fixture ctrlDisk(double limit) { @@ -52,10 +63,13 @@ public class ClusterResourceLimitsTest { return this; } public ClusterResourceLimits build() { + ModelContext.FeatureFlags featureFlags = new TestProperties(); var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, - false, - new BaseDeployLogger()); + hostedVespa, + throwIfSpecified, + new BaseDeployLogger(), + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()); builder.setClusterControllerBuilder(ctrlBuilder); builder.setContentNodeBuilder(nodeBuilder); return builder.build(); @@ -130,27 +144,86 @@ public class ClusterResourceLimitsTest { } @Test - public void exception_is_thrown_when_resource_limits_are_specified() { + @Ignore // TODO: Remove hosted_limits_are_used_if_app_is_allowed_to_set_limits and enable this when code is fixed to do so + public void hosted_log_when_resource_limits_are_specified() { TestableDeployLogger logger = new TestableDeployLogger(); - buildClusterResourceLimitsAndLogIfSpecified(logger); + var limits = hostedBuildAndLogIfSpecified(logger); assertEquals(1, logger.warnings.size()); assertEquals("Element 'resource-limits' is not allowed to be set", logger.warnings.get(0)); + // Verify that default limits are used + assertLimits(0.8, 0.8, limits.getClusterControllerLimits()); + assertLimits(0.9, 0.9, limits.getContentNodeLimits()); + } + + @Test + public void hosted_exception_is_thrown_when_resource_limits_are_specified() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage(containsString("Element 'resource-limits' is not allowed to be set")); - buildClusterResourceLimitsAndThrowIfSpecified(logger); + hostedBuildAndThrowIfSpecified(); + } + + @Test + // TODO: Remove this and enable hosted_log_when_resource_limits_are_specified when code is fixed to do so + public void hosted_limits_are_used_if_app_is_allowed_to_set_limits() { + TestableDeployLogger logger = new TestableDeployLogger(); + + var limits = hostedBuildAndLogIfSpecified(logger); + assertEquals(1, logger.warnings.size()); + assertEquals("Element 'resource-limits' is not allowed to be set", logger.warnings.get(0)); + + // Verify that limits in XML are used + assertLimits(0.8, 0.92, limits.getClusterControllerLimits()); + assertLimits(0.9, 0.96, limits.getContentNodeLimits()); + } + + @Test + public void hosted_limits_from_feature_flag_are_used() { + TestableDeployLogger logger = new TestableDeployLogger(); + + TestProperties featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(0.85); + featureFlags.setResourceLimitMemory(0.90); + var limits = hostedBuild(false, logger, featureFlags, false); + + // Verify that limits from feature flags are used + assertLimits(0.85, 0.90, limits.getClusterControllerLimits()); + assertLimits(0.925, 0.95, limits.getContentNodeLimits()); } - private void buildClusterResourceLimitsAndThrowIfSpecified(DeployLogger deployLogger) { - buildClusterResourceLimits(true, deployLogger); + @Test + public void exception_is_thrown_when_resource_limits_are_out_of_range() { + TestableDeployLogger logger = new TestableDeployLogger(); + + TestProperties featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(1.1); + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value 1.1, but must be in the range [0.0, 1.0]")); + hostedBuild(false, logger, featureFlags, false); + + featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(-0.1); + expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value -0.1, but must be in the range [0.0, 1.0]")); + hostedBuild(false, logger, featureFlags, false); + } + + private void hostedBuildAndThrowIfSpecified() { + hostedBuild(true, new TestableDeployLogger(), new TestProperties(), true); } - private void buildClusterResourceLimitsAndLogIfSpecified(DeployLogger deployLogger) { - buildClusterResourceLimits(false, deployLogger); + private ClusterResourceLimits hostedBuildAndLogIfSpecified(DeployLogger deployLogger) { + return hostedBuild(false, deployLogger); } - private void buildClusterResourceLimits(boolean throwIfSpecified, DeployLogger deployLogger) { + private ClusterResourceLimits hostedBuild(boolean throwIfSpecified, DeployLogger deployLogger) { + return hostedBuild(throwIfSpecified, deployLogger, new TestProperties(), true); + } + + private ClusterResourceLimits hostedBuild(boolean throwIfSpecified, + DeployLogger deployLogger, + ModelContext.FeatureFlags featureFlags, + boolean limitsInXml) { Document clusterXml = XML.getDocument("<cluster id=\"test\">" + " <tuning>\n" + " <resource-limits>\n" + @@ -159,11 +232,16 @@ public class ClusterResourceLimitsTest { " </tuning>\n" + "</cluster>"); + Document noLimitsXml = XML.getDocument("<cluster id=\"test\">" + + "</cluster>"); + ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, true, throwIfSpecified, - deployLogger); - builder.build(new ModelElement(clusterXml.getDocumentElement())); + deployLogger, + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()); + return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); } private void assertLimits(Double expCtrlDisk, Double expCtrlMemory, Double expNodeDisk, Double expNodeMemory, Fixture f) { @@ -173,15 +251,15 @@ public class ClusterResourceLimitsTest { } private void assertLimits(Double expDisk, Double expMemory, ResourceLimits limits) { - assertLimit(expDisk, limits.getDiskLimit()); - assertLimit(expMemory, limits.getMemoryLimit()); + assertLimit(expDisk, limits.getDiskLimit(), "disk"); + assertLimit(expMemory, limits.getMemoryLimit(), "memory"); } - private void assertLimit(Double expLimit, Optional<Double> actLimit) { + private void assertLimit(Double expLimit, Optional<Double> actLimit, String limitType) { if (expLimit == null) { assertFalse(actLimit.isPresent()); } else { - assertEquals(expLimit, actLimit.get(), 0.00001); + assertEquals(limitType + " limit not as expected", expLimit, actLimit.get(), 0.00001); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 10bb00168bb..dedf344c546 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -22,12 +23,15 @@ public class FleetControllerClusterTest { new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)).build(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); + ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, false, - new BaseDeployLogger()) + new BaseDeployLogger(), + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) .build(root.getDeployState(), root, clusterElement.getXml()); } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 4ad7865c93b..8477f8fcbea 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -58,18 +58,6 @@ <group name="foo" environment="aws_stage" /> </secret-store> - <rest-api path="jersey1"> - <components bundle="my-bundle" /> - <components bundle="other-bundle"> - <package>com.yahoo.foo</package> - <package>com.yahoo.bar</package> - </components> - - </rest-api> - <rest-api path="jersey/2"> - <components bundle="my-bundle" /> - </rest-api> - <servlet id="my-servlet" class="com.yahoo.MyServlet" bundle="my-bundle"> <path>p/a/t/h</path> </servlet> diff --git a/config-model/src/test/schema-test-files/standalone-container.xml b/config-model/src/test/schema-test-files/standalone-container.xml index b77654521ee..a7769a2f57b 100644 --- a/config-model/src/test/schema-test-files/standalone-container.xml +++ b/config-model/src/test/schema-test-files/standalone-container.xml @@ -1,21 +1,6 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <container id='qrsCluster_1' version='1.0'> - <rest-api path="jersey1"> - <components bundle="my-bundle" /> - <components bundle="other-bundle" /> -<!-- - <inject component="foo-component" for-name="com.yahoo.Foo" /> - <inject component="bar-component" for-name="com.yahoo.Bar" /> ---> - </rest-api> - <rest-api path="jersey/2"> - <components bundle="my-bundle" /> - <components bundle="other-bundle"> - <package>com.yahoo.foo</package> - <package>com.yahoo.bar</package> - </components> - </rest-api> <servlet id="my-servlet" class="com.yahoo.MyServlet" bundle="my-bundle"> <path>p/a/t/h</path> diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index 29bd38ea891..628e1c013e6 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -11,8 +11,8 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.RawConfig; -import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -32,24 +32,29 @@ public class ConfigProxyRpcServerTest { private static final String hostname = "localhost"; private static final int port = 12345; private static final String configSourceAddress = "tcp/" + hostname + ":" + port; - private TestServer server; - private TestClient client; + private static TestServer server; + private static TestClient client; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void setup() throws ListenFailedException { + @BeforeClass + public static void setup() throws ListenFailedException { server = new TestServer(); client = new TestClient(server.listenPort()); } - @After - public void teardown() { + @AfterClass + public static void teardown() { client.close(); server.close(); } + private static void reset() throws ListenFailedException { + teardown(); + setup(); + } + @Test public void basic() { ProxyServer proxy = createTestServer(new MockConfigSource()); @@ -75,7 +80,9 @@ public class ConfigProxyRpcServerTest { * Tests listCachedConfig RPC command */ @Test - public void testRpcMethodListCachedConfig() { + public void testRpcMethodListCachedConfig() throws ListenFailedException { + reset(); + Request req = new Request("listCachedConfig"); client.invoke(req); @@ -129,7 +136,9 @@ public class ConfigProxyRpcServerTest { * Tests printStatistics RPC command */ @Test - public void testRpcMethodListSourceConnections() { + public void testRpcMethodListSourceConnections() throws ListenFailedException { + reset(); + Request req = new Request("listSourceConnections"); client.invoke(req); @@ -218,7 +227,9 @@ public class ConfigProxyRpcServerTest { * Tests updateSources RPC command */ @Test - public void testRpcMethodUpdateSources() { + public void testRpcMethodUpdateSources() throws ListenFailedException { + reset(); + Request req = new Request("updateSources"); String spec1 = "tcp/a:19070"; String spec2 = "tcp/b:19070"; diff --git a/config/pom.xml b/config/pom.xml index 6e4e26ed0f1..8355587c10b 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -15,6 +15,12 @@ <dependencies> <!-- provided scope --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp index 7500d798b24..dceb9751f99 100644 --- a/config/src/tests/payload_converter/payload_converter.cpp +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("payload_converter"); + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/payload_converter.h> +#include <vespa/vespalib/data/slime/slime.h> + +#include <vespa/log/log.h> +LOG_SETUP("payload_converter"); using namespace config; using namespace vespalib; diff --git a/config/src/vespa/config/common/payload_converter.h b/config/src/vespa/config/common/payload_converter.h index 3e6cbbb44a6..27d1cf7c1c1 100644 --- a/config/src/vespa/config/common/payload_converter.h +++ b/config/src/vespa/config/common/payload_converter.h @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/object_traverser.h> +#include <vespa/vespalib/data/slime/array_traverser.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 7ac55b7c70e..6f2f42c98d5 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> #include "frtconfigrequest.h" #include "protocol.h" +#include <vespa/vespalib/data/slime/slime.h> class FRT_Values; class FRT_RPCRequest; diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def index 20ca6b10450..26b0e4a4a37 100644 --- a/configdefinitions/src/vespa/summary.def +++ b/configdefinitions/src/vespa/summary.def @@ -4,5 +4,6 @@ namespace=vespa.config.search defaultsummaryid int default=-1 classes[].id int classes[].name string +classes[].omitsummaryfeatures bool default=false classes[].fields[].name string classes[].fields[].type string diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index 006e266916c..b08fc8eeafe 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -6,8 +6,8 @@ zooKeeperConfigFile string default="conf/zookeeper/zookeeper.cfg" # For more info about the values below, see ZooKeeper documentation -# tick time in milliseconds -tickTime int default=2000 +# tick time in milliseconds, min and max session timeout are 2 and 20 times this value +tickTime int default=6000 initLimit int default=20 syncLimit int default=15 maxClientConnections int default=0 diff --git a/configserver-client/pom.xml b/configserver-client/pom.xml index 0a29ba003f4..39005c9ccab 100644 --- a/configserver-client/pom.xml +++ b/configserver-client/pom.xml @@ -30,6 +30,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver/pom.xml b/configserver/pom.xml index 3b7fef085b1..ac98cfa5d5d 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -142,6 +142,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <!-- To get all necessary test deps. --> <groupId>com.yahoo.vespa</groupId> <artifactId>container-test</artifactId> @@ -162,12 +168,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-jersey2</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2d5940ebba2..56fd6a64305 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -56,9 +56,9 @@ import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.TesterClient; -import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse; import com.yahoo.vespa.config.server.http.v2.PrepareResult; -import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse; import com.yahoo.vespa.config.server.metrics.DeploymentMetricsRetriever; import com.yahoo.vespa.config.server.metrics.ProtonMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -547,15 +547,27 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { + public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, String pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. - String relativePath = "clustercontroller-status/" + pathSuffix; + String pathPrefix; + switch (serviceName) { + case "container-clustercontroller": { + pathPrefix = "clustercontroller-status/v1/"; + break; + } + case "distributor": + case "storagenode": { + pathPrefix = ""; + break; + } + default: + throw new NotFoundException("No status page for service: " + serviceName); + } - return httpProxy.get(getApplication(applicationId), hostName, - CLUSTERCONTROLLER_CONTAINER.serviceName, relativePath); + return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix + pathSuffix); } public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java index eb57b9346c7..c6ab70adce7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; import com.yahoo.path.Path; @@ -17,7 +17,7 @@ public class SuperModelGenerationCounter implements GenerationCounter { private final CuratorCounter counter; public SuperModelGenerationCounter(Curator curator) { - this.counter = new CuratorCounter(curator, counterPath.getAbsolute()); + this.counter = new CuratorCounter(curator, counterPath); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java index 15a9b8f74d0..8097cd24514 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java @@ -1,16 +1,19 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; import com.google.common.base.Splitter; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import java.util.logging.Level; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.curator.Curator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -24,10 +27,12 @@ public class UserConfigDefinitionRepo implements ConfigDefinitionRepo { // For testing only public UserConfigDefinitionRepo() {} - public UserConfigDefinitionRepo(ConfigCurator configCurator, String appPath) { - if (configCurator.exists(appPath)) { - for (String nodeName : configCurator.getChildren(appPath)) { - String payload = configCurator.getData(appPath, nodeName); + public UserConfigDefinitionRepo(Curator curator, Path appPath) { + if (curator.exists(appPath)) { + for (String nodeName : curator.getChildren(appPath)) { + String payload = curator.getData(appPath.append(nodeName)) + .map(Utf8::toString) + .orElseThrow(() -> new IllegalArgumentException("No config definition data at " + nodeName)); ConfigDefinitionKey dKey = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName); defs.put(dKey, new ConfigDefinition(dKey.getName(), Splitter.on("\n").splitToList(payload).toArray(new String[0]))); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 5662fb15890..551f9060418 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -169,7 +169,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; private final boolean enableFeedBlockInDistributor; - private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; @@ -183,6 +182,8 @@ public class ModelContextImpl implements ModelContext { private final int largeRankExpressionLimit; private final boolean throwIfResourceLimitsSpecified; private final boolean dryRunOnnxOnSetup; + private final double resourceLimitDisk; + private final double resourceLimitMemory; public FeatureFlags(FlagSource source, ApplicationId appId) { this.dedicatedClusterControllerFlavor = parseDedicatedClusterControllerFlavor(flagValue(source, appId, Flags.DEDICATED_CLUSTER_CONTROLLER_FLAVOR)); @@ -197,7 +198,6 @@ public class ModelContextImpl implements ModelContext { this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); - this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value(); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); @@ -211,6 +211,8 @@ public class ModelContextImpl implements ModelContext { this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); this.throwIfResourceLimitsSpecified = flagValue(source, appId, Flags.THROW_EXCEPTION_IF_RESOURCE_LIMITS_SPECIFIED); this.dryRunOnnxOnSetup = flagValue(source, appId, Flags.DRY_RUN_ONNX_ON_SETUP); + this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); + this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); } @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); } @@ -225,7 +227,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } - @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { @@ -241,6 +242,8 @@ public class ModelContextImpl implements ModelContext { @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean throwIfResourceLimitsSpecified() { return throwIfResourceLimitsSpecified; } @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } + @Override public double resourceLimitDisk() { return resourceLimitDisk; } + @Override public double resourceLimitMemory() { return resourceLimitMemory; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index f0a63757477..99632ec323e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -12,11 +12,11 @@ import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; -import java.util.logging.Level; import com.yahoo.path.Path; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; +import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; @@ -27,12 +27,13 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.logging.Level; import static com.yahoo.config.application.api.ApplicationPackage.*; -import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.DEFCONFIGS_ZK_SUBPATH; -import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.META_ZK_PATH; -import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USERAPP_ZK_SUBPATH; -import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH; /** * Reads and writes application package to and from ZooKeeper. @@ -41,14 +42,14 @@ import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONF */ public class ZooKeeperClient { - private final ConfigCurator configCurator; + private final Curator curator; private final DeployLogger logger; private final Path sessionPath; // session id private static final ApplicationFile.PathFilter xmlFilter = path -> path.getName().endsWith(".xml"); - public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, Path sessionPath) { - this.configCurator = configCurator; + public ZooKeeperClient(Curator curator, DeployLogger logger, Path sessionPath) { + this.curator = curator; this.logger = logger; this.sessionPath = sessionPath; } @@ -58,16 +59,14 @@ public class ZooKeeperClient { * This is the first operation on ZK during deploy. */ void initialize() { - if ( ! configCurator.exists(sessionPath.getAbsolute())) - configCurator.createNode(sessionPath.getAbsolute()); + curator.create(sessionPath); for (String subPath : Arrays.asList(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH, ZKApplicationPackage.fileRegistryNode)) { // TODO: The replaceFirst below is hackish. - configCurator.createNode(getZooKeeperAppPath().getAbsolute(), - subPath.replaceFirst("/", "")); + curator.create(getZooKeeperAppPath().append(subPath.replaceFirst("/", ""))); } } @@ -95,12 +94,12 @@ public class ZooKeeperClient { if (sds.isEmpty()) return; Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); - configCurator.createNode(zkPath.getAbsolute()); + curator.create(zkPath); // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); for (NamedReader sd : sds) { - configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader())); + curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader()))); sd.getReader().close(); } } @@ -154,7 +153,7 @@ public class ZooKeeperClient { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... if (file.isDirectory()) { - configCurator.createNode(path.append(name).getAbsolute()); + curator.create(path.append(name)); if (recurse) { writeDir(file, path.append(name), filenameFilter, recurse); } @@ -192,7 +191,7 @@ public class ZooKeeperClient { try (InputStream inputStream = file.createInputStream()) { inputStream.transferTo(baos); baos.flush(); - configCurator.putData(zkPath.append(file.getPath().getName()).getAbsolute(), baos.toByteArray()); + curator.set(zkPath.append(file.getPath().getName()), baos.toByteArray()); } } @@ -201,7 +200,7 @@ public class ZooKeeperClient { ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); final List<ApplicationFile> files = dir.listFiles(); if (files == null || files.isEmpty()) { - configCurator.createNode(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude).getAbsolute()); + curator.create(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude)); } writeDir(dir, getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude), @@ -218,21 +217,20 @@ public class ZooKeeperClient { for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) { ConfigDefinitionKey key = entry.getKey(); String contents = entry.getValue().getUnparsedContent(); - writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); - writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH), contents); + writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH), contents); } logger.log(Level.FINE, configDefs.size() + " user config definitions"); } - private void writeConfigDefinition(String name, String namespace, String path, String data) { - configCurator.putDefData(namespace + "." + name, path, com.yahoo.text.Utf8.toBytes(data)); + private void writeConfigDefinition(String name, String namespace, Path path, String data) { + curator.set(path.append(namespace + "." + name), Utf8.toBytes(data)); } private void write(Version vespaVersion, FileRegistry fileRegistry) { String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); - configCurator.putData(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).getAbsolute(), - vespaVersion.toFullString(), - exportedRegistry); + curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()), + Utf8.toBytes(exportedRegistry)); } /** @@ -242,13 +240,13 @@ public class ZooKeeperClient { * @param metaData The application metadata. */ private void writeMetadata(ApplicationMetaData metaData) { - configCurator.putData(getZooKeeperAppPath(META_ZK_PATH).getAbsolute(), metaData.asJsonBytes()); + curator.set(getZooKeeperAppPath(META_ZK_PATH), metaData.asJsonBytes()); } void cleanupZooKeeper() { try { List.of(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH) - .forEach(path -> configCurator.deleteRecurse(getZooKeeperAppPath(path).getAbsolute())); + .forEach(path -> curator.delete(getZooKeeperAppPath(path))); } catch (Exception e) { logger.log(Level.WARNING, "Could not clean up in zookeeper: " + Exceptions.toMessageString(e)); //Might be called in an exception handler before re-throw, so do not throw here. @@ -277,8 +275,8 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData(sessionPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), - AllocatedHostsSerializer.toJson(hosts)); + curator.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map<Version, FileRegistry> fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index ff70ce1e1b2..cdbd7378151 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -18,8 +18,8 @@ import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.http.v2.HttpConfigRequests; -import com.yahoo.vespa.config.server.http.v2.TenantRequest; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.TenantRequest; import com.yahoo.vespa.config.util.ConfigUtils; /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index dfbce72d4ba..7e8940d28b3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -14,9 +14,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareAndActivateResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.model.content.Content; import org.apache.hc.core5.http.ContentType; import org.eclipse.jetty.http.MultiPartFormInputStream; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 3634a6825a3..ec7b6616bb6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -6,47 +6,42 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.jdisc.application.UriPattern; -import com.yahoo.slime.Cursor; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; +import com.yahoo.restapi.Path; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationReindexing; -import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; -import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.http.v2.request.ApplicationContentRequest; +import com.yahoo.vespa.config.server.http.v2.response.ApplicationSuspendedResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeleteApplicationResponse; +import com.yahoo.vespa.config.server.http.v2.response.GetApplicationResponse; +import com.yahoo.vespa.config.server.http.v2.response.QuotaUsageResponse; +import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; -import java.net.URLDecoder; import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Stream; import static com.yahoo.yolean.Exceptions.uncheck; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.stream.Collectors.toList; /** * Operations on applications (delete, wait for config convergence, restart, application content etc.) @@ -55,28 +50,6 @@ import static java.util.stream.Collectors.toList; */ public class ApplicationHandler extends HttpHandler { - private static final List<UriPattern> URI_PATTERNS = Stream.of( - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindex", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindexing", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/validate-secret-store", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/quota", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", - "http://*/application/v2/tenant/*/application/*") - .map(UriPattern::new) - .collect(toList()); - private final Zone zone; private final ApplicationRepository applicationRepository; @@ -90,152 +63,135 @@ public class ApplicationHandler extends HttpHandler { } @Override - public HttpResponse handleDELETE(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - - if (isReindexingRequest(request)) { - applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false)); - return createMessageResponse("Reindexing disabled"); - } + public HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri()); + + if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return getApplicationResponse(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value())); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return getApplicationResponse(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/content/{*}")) return content(applicationId(path), path.getRest(), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest()); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge")) return listServiceConverge(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/{command}")) return testerRequest(applicationId(path), path.get("command"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path)); + return ErrorResponse.notFoundError("Nothing at " + path); + } - if (applicationRepository.delete(applicationId)) - return new DeleteApplicationResponse(Response.Status.OK, applicationId); + @Override + public HttpResponse handlePOST(HttpRequest request) { + Path path = new Path(request.getUri()); - return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found"); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/run/{suite}")) return testerStartTests(applicationId(path), path.get("suite"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/validate-secret-store")) return validateSecretStore(applicationId(path), request); + return ErrorResponse.notFoundError("Nothing at " + path); } @Override - public HttpResponse handleGET(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); - - if (isServiceConvergeRequest(request)) { - // Expects both hostname and port in the request (hostname:port) - String hostAndPort = getHostNameFromRequest(request); - return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), - timeout, getVespaVersionFromRequest(request)); - } + public HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri()); - if (isClusterControllerStatusRequest(request)) { - String hostName = getHostNameFromRequest(request); - String pathSuffix = URLDecoder.decode(getPathSuffix(request), UTF_8); - return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix); - } + if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return deleteApplication(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value())); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return deleteApplication(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return disableReindexing(applicationId(path)); + return ErrorResponse.notFoundError("Nothing at " + path); + } - if (isReindexingRequest(request)) { - return getReindexingStatus(applicationId); - } + private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) { + return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), + getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + } - if (isContentRequest(request)) { - long sessionId = applicationRepository.getSessionIdForApplication(applicationId); - String contentPath = getBindingMatch(request).group(7); - ApplicationFile applicationFile = - applicationRepository.getApplicationFileFromSession(applicationId.tenant(), - sessionId, - contentPath, - ContentRequest.getApplicationFileMode(request.getMethod())); - ApplicationContentRequest contentRequest = new ApplicationContentRequest(request, - sessionId, - applicationId, - zone, - contentPath, - applicationFile); - return new ContentHandler().get(contentRequest); - } + private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) { + return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), + getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + } - if (isServiceConvergeListRequest(request)) { - return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), timeout, - getVespaVersionFromRequest(request)); - } + private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, String pathSuffix) { + return applicationRepository.serviceStatusPage(applicationId, hostname, service, pathSuffix); + } - if (isFiledistributionStatusRequest(request)) { - return applicationRepository.filedistributionStatus(applicationId, timeout); - } + private HttpResponse content(ApplicationId applicationId, String contentPath, HttpRequest request) { + long sessionId = applicationRepository.getSessionIdForApplication(applicationId); + ApplicationFile applicationFile = + applicationRepository.getApplicationFileFromSession(applicationId.tenant(), + sessionId, + contentPath, + ContentRequest.getApplicationFileMode(request.getMethod())); + ApplicationContentRequest contentRequest = new ApplicationContentRequest(request, + sessionId, + applicationId, + zone, + contentPath, + applicationFile); + return new ContentHandler().get(contentRequest); + } - if (isLogRequest(request)) { - Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); - String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse(""); - return applicationRepository.getLogs(applicationId, hostname, apiParams); - } + private HttpResponse filedistributionStatus(ApplicationId applicationId, HttpRequest request) { + return applicationRepository.filedistributionStatus(applicationId, getTimeoutFromRequest(request)); + } - if (isProtonMetricsRequest(request)) { - return applicationRepository.getProtonMetrics(applicationId); - } + private HttpResponse logs(ApplicationId applicationId, HttpRequest request) { + Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); + String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse(""); + return applicationRepository.getLogs(applicationId, hostname, apiParams); + } - if (isDeploymentMetricsRequest(request)) { - return applicationRepository.getDeploymentMetrics(applicationId); - } + private HttpResponse protonMetrics(ApplicationId applicationId) { + return applicationRepository.getProtonMetrics(applicationId); + } - if (isIsSuspendedRequest(request)) { - return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); - } + private HttpResponse deploymentMetrics(ApplicationId applicationId) { + return applicationRepository.getDeploymentMetrics(applicationId); + } - if (isTesterRequest(request)) { - String testerCommand = getTesterCommandFromRequest(request); - switch (testerCommand) { - case "status": - return applicationRepository.getTesterStatus(applicationId); - case "log": - Long after = Long.valueOf(request.getProperty("after")); - return applicationRepository.getTesterLog(applicationId, after); - case "ready": - return applicationRepository.isTesterReady(applicationId); - case "report": - return applicationRepository.getTestReport(applicationId); - default: - throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); - } - } + private HttpResponse isSuspended(ApplicationId applicationId) { + return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); + } - if (isQuotaUsageRequest(request)) { - var quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId); - return new QuotaUsageResponse(quotaUsageRate); + private HttpResponse testerRequest(ApplicationId applicationId, String command, HttpRequest request) { + switch (command) { + case "status": + return applicationRepository.getTesterStatus(applicationId); + case "log": + Long after = Long.valueOf(request.getProperty("after")); + return applicationRepository.getTesterLog(applicationId, after); + case "ready": + return applicationRepository.isTesterReady(applicationId); + case "report": + return applicationRepository.getTestReport(applicationId); + default: + throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); } + } - return getApplicationResponse(applicationId); + private HttpResponse quotaUsage(ApplicationId applicationId) { + double quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId); + return new QuotaUsageResponse(quotaUsageRate); } - GetApplicationResponse getApplicationResponse(ApplicationId applicationId) { + private HttpResponse getApplicationResponse(ApplicationId applicationId) { return new GetApplicationResponse(Response.Status.OK, - applicationRepository.getApplicationGeneration(applicationId), - applicationRepository.getAllVersions(applicationId), - applicationRepository.getApplicationPackageReference(applicationId)); + applicationRepository.getApplicationGeneration(applicationId), + applicationRepository.getAllVersions(applicationId), + applicationRepository.getApplicationPackageReference(applicationId)); } - @Override - public HttpResponse handlePOST(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - - if (isRestartRequest(request)) - return restart(request, applicationId); - - if (isTesterStartTestsRequest(request)) { - byte[] data; - try { - data = IOUtils.readBytes(request.getData(), 1024 * 1000); - } catch (IOException e) { - throw new IllegalArgumentException("Could not read data in request " + request); - } - return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data); - } - - if (isReindexRequest(request)) { - return triggerReindexing(request, applicationId); - } - - if (isReindexingRequest(request)) { - applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true)); - return createMessageResponse("Reindexing enabled"); - } - - if (isValidateSecretStoreRequest(request)) { - var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes())); - return applicationRepository.validateSecretStore(applicationId, zone.system(), slime); - } - - throw new NotFoundException("Illegal POST request '" + request.getUri() + "'"); + public HttpResponse deleteApplication(ApplicationId applicationId) { + if (applicationRepository.delete(applicationId)) + return new DeleteApplicationResponse(applicationId); + return ErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found"); } + private Model getActiveModelOrThrow(ApplicationId id) { return applicationRepository.getActiveApplicationSet(id) .orElseThrow(() -> new NotFoundException("Application '" + id + "' not found")) @@ -243,7 +199,7 @@ public class ApplicationHandler extends HttpHandler { .getModel(); } - private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) { + private HttpResponse triggerReindexing(ApplicationId applicationId, HttpRequest request) { Model model = getActiveModelOrThrow(applicationId); Map<String, Set<String>> documentTypes = model.documentTypesByCluster(); Map<String, Set<String>> indexedDocumentTypes = model.indexedDocumentTypesByCluster(); @@ -274,7 +230,7 @@ public class ApplicationHandler extends HttpHandler { return reindexing; }); - return createMessageResponse(reindexed.entrySet().stream() + return new MessageResponse(reindexed.entrySet().stream() .filter(cluster -> ! cluster.getValue().isEmpty()) .map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'") .reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId) @@ -284,6 +240,16 @@ public class ApplicationHandler extends HttpHandler { .toString()); } + public HttpResponse disableReindexing(ApplicationId applicationId) { + applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false)); + return new MessageResponse("Reindexing disabled"); + } + + private HttpResponse enableReindexing(ApplicationId applicationId) { + applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true)); + return new MessageResponse("Reindexing enabled"); + } + private HttpResponse getReindexingStatus(ApplicationId applicationId) { Tenant tenant = applicationRepository.getTenant(applicationId); if (tenant == null) @@ -294,239 +260,42 @@ public class ApplicationHandler extends HttpHandler { applicationRepository.getClusterReindexingStatus(applicationId)); } - private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { - if (getBindingMatch(request).groupCount() != 7) - throw new NotFoundException("Illegal POST restart request '" + request.getUri() + - "': Must have 6 arguments but had " + (getBindingMatch(request).groupCount() - 1)); - applicationRepository.restart(applicationId, hostFilterFrom(request)); - return new JSONResponse(Response.Status.OK); // return empty - } - - private HostFilter hostFilterFrom(HttpRequest request) { - return HostFilter.from(request.getProperty("hostname"), - request.getProperty("flavor"), - request.getProperty("clusterType"), - request.getProperty("clusterId")); - } - - private static BindingMatch<?> getBindingMatch(HttpRequest request) { - return URI_PATTERNS.stream() - .map(pattern -> { - UriPattern.Match match = pattern.match(request.getUri()); - if (match == null) return null; - return new BindingMatch<>(match, new Object(), pattern); - }) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Illegal url for config request: " + request.getUri())); - } - - private static boolean isRestartRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/restart"); - } - - private static boolean isReindexRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/reindex"); - } - - private static boolean isReindexingRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/reindexing"); - } - - private static boolean isIsSuspendedRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/suspended"); - } - - private static boolean isProtonMetricsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().endsWith("/metrics/proton"); - } - - private static boolean isDeploymentMetricsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().endsWith("/metrics/deployment"); - } - - private static boolean isLogRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/logs"); - } - - private static boolean isValidateSecretStoreRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/validate-secret-store"); - } - - private static boolean isServiceConvergeListRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/serviceconverge"); - } - - private static boolean isServiceConvergeRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().contains("/serviceconverge/"); - } - - private static boolean isClusterControllerStatusRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 9 && - request.getUri().getPath().contains("/clustercontroller/"); - } - - private static boolean isContentRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() > 7 && - request.getUri().getPath().contains("/content/"); - } - - private static boolean isFiledistributionStatusRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().contains("/filedistributionstatus"); - } - - private static boolean isTesterRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().contains("/tester"); - } - - private static boolean isTesterStartTestsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 9 && - request.getUri().getPath().contains("/tester/run/"); - } - - private static boolean isQuotaUsageRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/quota"); - } - - private static String getHostNameFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(7); - } - - private static String getTesterCommandFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(7); - } - - private static String getSuiteFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(8); - } - - private static String getPathSuffix(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(8); - } - - private static ApplicationId getApplicationIdFromRequest(HttpRequest req) { - // Two bindings for this: with full app id or only application name - BindingMatch<?> bm = getBindingMatch(req); - if (bm.groupCount() > 4) return createFromRequestFullAppId(bm); - return createFromRequestSimpleAppId(bm); - } - - // The URL pattern with only tenant and application given - private static ApplicationId createFromRequestSimpleAppId(BindingMatch<?> bm) { - TenantName tenant = TenantName.from(bm.group(2)); - ApplicationName application = ApplicationName.from(bm.group(3)); - return new ApplicationId.Builder().tenant(tenant).applicationName(application).build(); - } - - // The URL pattern with full app id given - private static ApplicationId createFromRequestFullAppId(BindingMatch<?> bm) { - String tenant = bm.group(2); - String application = bm.group(3); - String instance = bm.group(6); - return new ApplicationId.Builder() - .tenant(tenant) - .applicationName(application).instanceName(instance) - .build(); - } - - private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) { - String vespaVersion = request.getProperty("vespaVersion"); - return (vespaVersion == null || vespaVersion.isEmpty()) - ? Optional.empty() - : Optional.of(Version.fromString(vespaVersion)); + private HttpResponse restart(ApplicationId applicationId, HttpRequest request) { + HostFilter filter = HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId")); + applicationRepository.restart(applicationId, filter); + return new MessageResponse("Success"); } - private static class DeleteApplicationResponse extends JSONResponse { - DeleteApplicationResponse(int status, ApplicationId applicationId) { - super(status); - object.setString("message", "Application '" + applicationId + "' deleted"); + private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) { + byte[] data; + try { + data = IOUtils.readBytes(request.getData(), 1024 * 1000); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read data in request " + request); } + return applicationRepository.startTests(applicationId, suite, data); } - private static class GetApplicationResponse extends JSONResponse { - GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) { - super(status); - object.setLong("generation", generation); - object.setString("applicationPackageFileReference", applicationPackageReference.orElse("")); - Cursor modelVersionArray = object.setArray("modelVersions"); - modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); - } + private HttpResponse validateSecretStore(ApplicationId applicationId, HttpRequest request) { + var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes())); + return applicationRepository.validateSecretStore(applicationId, zone.system(), slime); } - private static class ApplicationSuspendedResponse extends JSONResponse { - ApplicationSuspendedResponse(boolean suspended) { - super(Response.Status.OK); - object.setBool("suspended", suspended); - } + private static ApplicationId applicationId(Path path) { + return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); } - private static class QuotaUsageResponse extends JSONResponse { - QuotaUsageResponse(double usageRate) { - super(Response.Status.OK); - object.setDouble("rate", usageRate); - } + private static Duration getTimeoutFromRequest(HttpRequest request) { + return HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); } - static class ReindexingResponse extends JSONResponse { - ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing, - Map<String, ClusterReindexing> clusters) { - super(Response.Status.OK); - object.setBool("enabled", reindexing.enabled()); - Cursor clustersObject = object.setObject("clusters"); - documentTypes.forEach((cluster, types) -> { - Cursor clusterObject = clustersObject.setObject(cluster); - Cursor pendingObject = clusterObject.setObject("pending"); - Cursor readyObject = clusterObject.setObject("ready"); - - for (String type : types) { - Cursor statusObject = readyObject.setObject(type); - if (reindexing.clusters().containsKey(cluster)) { - if (reindexing.clusters().get(cluster).pending().containsKey(type)) - pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type)); - - if (reindexing.clusters().get(cluster).ready().containsKey(type)) - setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type)); - } - if (clusters.containsKey(cluster)) - if (clusters.get(cluster).documentTypeStatus().containsKey(type)) - setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type)); - } - }); - } - - private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) { - object.setLong("readyMillis", readyStatus.ready().toEpochMilli()); - } - - private static void setStatus(Cursor object, ClusterReindexing.Status status) { - object.setLong("startedMillis", status.startedAt().toEpochMilli()); - status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli())); - status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state)); - status.message().ifPresent(message -> object.setString("message", message)); - status.progress().ifPresent(progress -> object.setDouble("progress", progress)); - } - - } - - private static JSONResponse createMessageResponse(String message) { - return new JSONResponse(Response.Status.OK) { { object.setString("message", message); } }; + private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) { + return Optional.ofNullable(request.getProperty("vespaVersion")) + .filter(s -> !s.isEmpty()) + .map(Version::fromString); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 1ea41b85983..0a69c7ed904 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; /** * Handler for getting tenant and application for a given hostname. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 1787431e841..76a39bbcadb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.http.HttpConfigRequest; import com.yahoo.vespa.config.server.http.HttpConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java index 71be471193a..3210f5eb8e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java @@ -15,6 +15,8 @@ import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.http.HttpConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java index bbc2efe9c36..425aa1c2bd2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java @@ -10,6 +10,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.http.HttpConfigRequest; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index b06d5c31ac6..20688f4f2f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -10,6 +10,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.http.v2.response.ListApplicationsResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 4e75234620f..f97e272b132 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -14,6 +14,7 @@ import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.TimeoutBudget; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index 9f526bbdce8..4664fdf3557 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.request.SessionContentRequestV2; /** * A handler that will return content or content status for files or directories diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 61f099fb8ea..ea7d11d9802 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.application.UriPattern; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; @@ -17,13 +16,12 @@ import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; -import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.config.server.http.v2.response.SessionCreateResponse; import org.apache.hc.core5.http.ContentType; import java.net.URI; import java.time.Duration; import java.util.List; -import java.util.stream.Collectors; /** * A handler that is able to create a session from an application package, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 6fa2075807f..0d6d5ed943a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index bb94f8d442a..a9961762f8b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -10,6 +10,10 @@ import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.ListTenantsResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantCreateResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantDeleteResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantGetResponse; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.yolean.Exceptions; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java index af3ab0b1b83..b7ec6272c3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; @@ -18,12 +18,12 @@ public class ApplicationContentRequest extends ContentRequest { private final ApplicationId applicationId; private final Zone zone; - ApplicationContentRequest(HttpRequest request, - long sessionId, - ApplicationId applicationId, - Zone zone, - String contentPath, - ApplicationFile applicationFile) { + public ApplicationContentRequest(HttpRequest request, + long sessionId, + ApplicationId applicationId, + Zone zone, + String contentPath, + ApplicationFile applicationFile) { super(request, sessionId, contentPath, applicationFile); this.applicationId = applicationId; this.zone = zone; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java index 8b482997044..d190ccaec12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import java.net.URI; @@ -45,7 +45,7 @@ public class HttpConfigRequests { } - static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) { + public static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) { Tenant tenant = tenantRepository.getTenant(request.getApplicationId().tenant()); if (tenant==null) throw new NotFoundException("No such tenant: "+request.getApplicationId().tenant()); return tenant.getRequestHandler(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java index 208b682df9a..cd9c44db85b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.collections.Tuple2; import com.yahoo.config.provision.ApplicationName; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java index c5a9d36f493..4f34b2fb332 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; @@ -20,7 +20,7 @@ public class SessionContentRequestV2 extends ContentRequest { private final TenantName tenantName; private final long sessionId; - SessionContentRequestV2(HttpRequest request, + public SessionContentRequestV2(HttpRequest request, long sessionId, TenantName tenantName, String path, @@ -35,7 +35,7 @@ public class SessionContentRequestV2 extends ContentRequest { return "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; } - static String getContentPath(HttpRequest request) { + public static String getContentPath(HttpRequest request) { BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern); return bm.group(4); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java index 0fae091aa78..84071ca2a98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java new file mode 100644 index 00000000000..c2449d9a31d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.http.JSONResponse; + +public class ApplicationSuspendedResponse extends JSONResponse { + public ApplicationSuspendedResponse(boolean suspended) { + super(Response.Status.OK); + object.setBool("suspended", suspended); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java new file mode 100644 index 00000000000..375a6d5e57f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java @@ -0,0 +1,11 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.restapi.MessageResponse; + +public class DeleteApplicationResponse extends MessageResponse { + public DeleteApplicationResponse(ApplicationId applicationId) { + super("Application '" + applicationId + "' deleted"); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java index cdfdce91500..253e37f2b0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java new file mode 100644 index 00000000000..dc124891540 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.component.Version; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.server.http.JSONResponse; + +import java.util.List; +import java.util.Optional; + +public class GetApplicationResponse extends JSONResponse { + public GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) { + super(status); + object.setLong("generation", generation); + object.setString("applicationPackageFileReference", applicationPackageReference.orElse("")); + Cursor modelVersionArray = object.setArray("modelVersions"); + modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java index a4527305abb..6b1f7031d81 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java index 3789939429c..ec1b3c83604 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; +import java.util.Set; + /** * Tenant list response * @@ -13,7 +14,7 @@ import com.yahoo.slime.Cursor; */ public class ListTenantsResponse extends SlimeJsonResponse { - ListTenantsResponse(ImmutableSet<TenantName> tenants) { + public ListTenantsResponse(Set<TenantName> tenants) { Cursor tenantArray = slime.setObject().setArray("tenants"); tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java index 99b95f9244c..25e770c7211 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java new file mode 100644 index 00000000000..e30f0f45098 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.http.JSONResponse; + +public class QuotaUsageResponse extends JSONResponse { + public QuotaUsageResponse(double usageRate) { + super(Response.Status.OK); + object.setDouble("rate", usageRate); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java new file mode 100644 index 00000000000..e73d4a9cb56 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ClusterReindexing; +import com.yahoo.vespa.config.server.http.JSONResponse; + +import java.util.Map; +import java.util.Set; + +public class ReindexingResponse extends JSONResponse { + public ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing, + Map<String, ClusterReindexing> clusters) { + super(Response.Status.OK); + object.setBool("enabled", reindexing.enabled()); + Cursor clustersObject = object.setObject("clusters"); + documentTypes.forEach((cluster, types) -> { + Cursor clusterObject = clustersObject.setObject(cluster); + Cursor pendingObject = clusterObject.setObject("pending"); + Cursor readyObject = clusterObject.setObject("ready"); + + for (String type : types) { + Cursor statusObject = readyObject.setObject(type); + if (reindexing.clusters().containsKey(cluster)) { + if (reindexing.clusters().get(cluster).pending().containsKey(type)) + pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type)); + + if (reindexing.clusters().get(cluster).ready().containsKey(type)) + setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type)); + } + if (clusters.containsKey(cluster)) + if (clusters.get(cluster).documentTypeStatus().containsKey(type)) + setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type)); + } + }); + } + + private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) { + object.setLong("readyMillis", readyStatus.ready().toEpochMilli()); + } + + private static void setStatus(Cursor object, ClusterReindexing.Status status) { + object.setLong("startedMillis", status.startedAt().toEpochMilli()); + status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli())); + status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state)); + status.message().ifPresent(message -> object.setString("message", message)); + status.progress().ifPresent(progress -> object.setDouble("progress", progress)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java index 9c0fbdf2613..f14f9cc6575 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java index faf02a1ea4f..047962ea16b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java index 7bace4749a8..d219ff64b4e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -8,15 +8,16 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareAndActivateResponse extends SlimeJsonResponse { +public class SessionPrepareAndActivateResponse extends SlimeJsonResponse { - SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { + public SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { super(result.deployLogger().slime()); TenantName tenantName = applicationId.tenant(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java index a97cd37d3b4..5f50a5236fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -9,19 +9,20 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.Type; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareResponse extends SlimeJsonResponse { +public class SessionPrepareResponse extends SlimeJsonResponse { - SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { + public SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions()); } - SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { + public SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java index 6ff2b30075d..395a52df9f2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java index d21584c8cdc..fbfb4d581c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java index b918cab7828..3a50a68cb21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java index 7fc2c47c06c..2681952cae9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse; import java.net.URI; import java.util.Collection; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java index 5078fba8b38..4c111687c41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse; import java.net.URI; import java.util.Collection; import java.util.function.Predicate; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 95b29235e2f..27ee040fe4f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -37,8 +37,8 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.config.server.zookeeper.ZKApplication; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.BooleanFlag; @@ -114,7 +114,6 @@ public class SessionRepository { private final SessionPreparer sessionPreparer; private final Path sessionsPath; private final TenantName tenantName; - private final ConfigCurator configCurator; private final SessionCounter sessionCounter; private final SecretStore secretStore; private final HostProvisionerProvider hostProvisionerProvider; @@ -123,11 +122,12 @@ public class SessionRepository { private final Zone zone; private final ModelFactoryRegistry modelFactoryRegistry; private final ConfigDefinitionRepo configDefinitionRepo; + private final int maxNodeSize; public SessionRepository(TenantName tenantName, TenantApplications applicationRepo, SessionPreparer sessionPreparer, - ConfigCurator configCurator, + Curator curator, Metrics metrics, StripedExecutor<TenantName> zkWatcherExecutor, PermanentApplicationPackage permanentApplicationPackage, @@ -140,13 +140,13 @@ public class SessionRepository { Zone zone, Clock clock, ModelFactoryRegistry modelFactoryRegistry, - ConfigDefinitionRepo configDefinitionRepo) { + ConfigDefinitionRepo configDefinitionRepo, + int maxNodeSize) { this.tenantName = tenantName; - this.configCurator = configCurator; - sessionCounter = new SessionCounter(configCurator, tenantName); + sessionCounter = new SessionCounter(curator, tenantName); this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.clock = clock; - this.curator = configCurator.curator(); + this.curator = curator; this.sessionLifetime = Duration.ofSeconds(configserverConfig.sessionLifetime()); this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); this.permanentApplicationPackage = permanentApplicationPackage; @@ -163,6 +163,7 @@ public class SessionRepository { this.zone = zone; this.modelFactoryRegistry = modelFactoryRegistry; this.configDefinitionRepo = configDefinitionRepo; + this.maxNodeSize = maxNodeSize; loadSessions(Flags.LOAD_LOCAL_SESSIONS_WHEN_BOOTSTRAPPING.bindTo(flagSource)); // Needs to be done before creating cache below this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); @@ -591,7 +592,7 @@ public class SessionRepository { private void ensureSessionPathDoesNotExist(long sessionId) { Path sessionPath = getSessionPath(sessionId); - if (configCurator.exists(sessionPath.getAbsolute())) { + if (curator.exists(sessionPath)) { throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); } } @@ -661,8 +662,7 @@ public class SessionRepository { private Optional<ApplicationSet> getApplicationSet(long sessionId) { Optional<ApplicationSet> applicationSet = Optional.empty(); try { - RemoteSession session = getRemoteSession(sessionId); - applicationSet = Optional.ofNullable(ensureApplicationLoaded(session)); + applicationSet = Optional.ofNullable(getRemoteSession(sessionId)).map(this::ensureApplicationLoaded); } catch (IllegalArgumentException e) { // Do nothing if we have no currently active session } @@ -776,12 +776,12 @@ public class SessionRepository { } Path getSessionStatePath(long sessionId) { - return getSessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + return getSessionPath(sessionId).append(ZKApplication.SESSIONSTATE_ZK_SUBPATH); } private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { String serverId = configserverConfig.serverId(); - return new SessionZooKeeperClient(curator, configCurator, tenantName, sessionId, serverId); + return new SessionZooKeeperClient(curator, tenantName, sessionId, serverId, maxNodeSize); } private File getAndValidateExistingSessionAppDir(long sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index dfe4f12195d..c7b19968f0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.ZKApplication; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; @@ -36,6 +36,7 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.yolean.Exceptions.uncheck; @@ -62,23 +63,23 @@ public class SessionZooKeeperClient { private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates"; private final Curator curator; - private final ConfigCurator configCurator; private final TenantName tenantName; private final Path sessionPath; private final Path sessionStatusPath; private final String serverId; // hostname + private final int maxNodeSize; - public SessionZooKeeperClient(Curator curator, - ConfigCurator configCurator, - TenantName tenantName, - long sessionId, - String serverId) { + public SessionZooKeeperClient(Curator curator, TenantName tenantName, long sessionId, String serverId, int maxNodeSize) { this.curator = curator; - this.configCurator = configCurator; this.tenantName = tenantName; this.sessionPath = getSessionPath(tenantName, sessionId); this.serverId = serverId; - this.sessionStatusPath = sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + this.sessionStatusPath = sessionPath.append(ZKApplication.SESSIONSTATE_ZK_SUBPATH); + this.maxNodeSize = maxNodeSize; + } + + public SessionZooKeeperClient(Curator curator, TenantName tenantName, long sessionId, String serverId) { + this(curator, tenantName, sessionId, serverId, 10 * 1024 * 1024); } public void writeStatus(Session.Status sessionStatus) { @@ -91,10 +92,11 @@ public class SessionZooKeeperClient { public Session.Status readStatus() { try { - String data = configCurator.getData(sessionStatusPath.getAbsolute()); - return Session.Status.parse(data); + Optional<byte[]> data = curator.getData(sessionStatusPath); + return data.map(d -> Session.Status.parse(Utf8.toString(d))).orElse(Session.Status.NONE); } catch (Exception e) { - log.log(Level.INFO, "Unable to read session status, assuming it was deleted"); + log.log(Level.INFO, "Failed to read session status at " + sessionStatusPath.getAbsolute() + + ", will assume session has been removed: ", e); return Session.Status.NONE; } } @@ -136,93 +138,93 @@ public class SessionZooKeeperClient { } public ApplicationPackage loadApplicationPackage() { - return new ZKApplicationPackage(configCurator, sessionPath); + return new ZKApplicationPackage(curator, sessionPath, maxNodeSize); } public ConfigDefinitionRepo getUserConfigDefinitions() { - return new UserConfigDefinitionRepo(configCurator, sessionPath.append(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute()); + return new UserConfigDefinitionRepo(curator, sessionPath.append(USER_DEFCONFIGS_ZK_SUBPATH)); } - private String applicationIdPath() { - return sessionPath.append(APPLICATION_ID_PATH).getAbsolute(); + private Path applicationIdPath() { + return sessionPath.append(APPLICATION_ID_PATH); } public void writeApplicationId(ApplicationId id) { if ( ! id.tenant().equals(tenantName)) throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenantName + "'"); - configCurator.putData(applicationIdPath(), id.serializedForm()); + curator.set(applicationIdPath(), Utf8.toBytes(id.serializedForm())); } public Optional<ApplicationId> readApplicationId() { - if ( ! configCurator.exists(applicationIdPath())) return Optional.empty(); - return Optional.of(ApplicationId.fromSerializedForm(configCurator.getData(applicationIdPath()))); + return curator.getData(applicationIdPath()).map(d -> ApplicationId.fromSerializedForm(Utf8.toString(d))); } void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) { applicationPackageReference.ifPresent( - reference -> configCurator.putData(applicationPackageReferencePath(), reference.value())); + reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value()))); } FileReference readApplicationPackageReference() { - if ( ! configCurator.exists(applicationPackageReferencePath())) return null; // This should not happen. - return new FileReference(configCurator.getData(applicationPackageReferencePath())); + Optional<byte[]> data = curator.getData(applicationPackageReferencePath()); + if (data.isEmpty()) return null; // This should not happen. + + return new FileReference(Utf8.toString(data.get())); } - private String applicationPackageReferencePath() { - return sessionPath.append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute(); + private Path applicationPackageReferencePath() { + return sessionPath.append(APPLICATION_PACKAGE_REFERENCE_PATH); } - private String versionPath() { - return sessionPath.append(VERSION_PATH).getAbsolute(); + private Path versionPath() { + return sessionPath.append(VERSION_PATH); } - private String dockerImageRepositoryPath() { - return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute(); + private Path dockerImageRepositoryPath() { + return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH); } - private String athenzDomainPath() { - return sessionPath.append(ATHENZ_DOMAIN).getAbsolute(); + private Path athenzDomainPath() { + return sessionPath.append(ATHENZ_DOMAIN); } - private String quotaPath() { - return sessionPath.append(QUOTA_PATH).getAbsolute(); + private Path quotaPath() { + return sessionPath.append(QUOTA_PATH); } - private String tenantSecretStorePath() { - return sessionPath.append(TENANT_SECRET_STORES_PATH).getAbsolute(); + private Path tenantSecretStorePath() { + return sessionPath.append(TENANT_SECRET_STORES_PATH); } - private String operatorCertificatesPath() { - return sessionPath.append(OPERATOR_CERTIFICATES_PATH).getAbsolute(); + private Path operatorCertificatesPath() { + return sessionPath.append(OPERATOR_CERTIFICATES_PATH); } public void writeVespaVersion(Version version) { - configCurator.putData(versionPath(), version.toString()); + curator.set(versionPath(), Utf8.toBytes(version.toString())); } public Version readVespaVersion() { - if ( ! configCurator.exists(versionPath())) return Vtag.currentVersion; // TODO: This should not be possible any more - verify and remove - return new Version(configCurator.getData(versionPath())); + Optional<byte[]> data = curator.getData(versionPath()); + // TODO: Empty version should not be possible any more - verify and remove + return data.map(d -> new Version(Utf8.toString(d))).orElse(Vtag.currentVersion); } public Optional<DockerImage> readDockerImageRepository() { - if ( ! configCurator.exists(dockerImageRepositoryPath())) return Optional.empty(); - String dockerImageRepository = configCurator.getData(dockerImageRepositoryPath()); - return dockerImageRepository.isEmpty() ? Optional.empty() : Optional.of(DockerImage.fromString(dockerImageRepository)); + Optional<byte[]> dockerImageRepository = curator.getData(dockerImageRepositoryPath()); + return dockerImageRepository.map(d -> DockerImage.fromString(Utf8.toString(d))); } public void writeDockerImageRepository(Optional<DockerImage> dockerImageRepository) { - dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo.untagged())); + dockerImageRepository.ifPresent(repo -> curator.set(dockerImageRepositoryPath(), Utf8.toBytes(repo.untagged()))); } public Instant readCreateTime() { - String path = getCreateTimePath(); - if ( ! configCurator.exists(path)) return Instant.EPOCH; - return Instant.ofEpochSecond(Long.parseLong(configCurator.getData(path))); + Optional<byte[]> data = curator.getData(getCreateTimePath()); + return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))).orElse(Instant.EPOCH); } - private String getCreateTimePath() { - return sessionPath.append(CREATE_TIME_PATH).getAbsolute(); + private Path getCreateTimePath() { + return sessionPath.append(CREATE_TIME_PATH); } AllocatedHosts getAllocatedHosts() { @@ -231,14 +233,13 @@ public class SessionZooKeeperClient { } public ZooKeeperDeployer createDeployer(DeployLogger logger) { - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, sessionPath); + ZooKeeperClient zkClient = new ZooKeeperClient(curator, logger, sessionPath); return new ZooKeeperDeployer(zkClient); } public Transaction createWriteStatusTransaction(Session.Status status) { - String path = sessionStatusPath.getAbsolute(); CuratorTransaction transaction = new CuratorTransaction(curator); - if (configCurator.exists(path)) { + if (curator.exists(sessionStatusPath)) { transaction.add(CuratorOperations.setData(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name()))); } else { transaction.add(CuratorOperations.create(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name()))); @@ -247,59 +248,56 @@ public class SessionZooKeeperClient { } public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) { - athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.value())); + athenzDomain.ifPresent(domain -> curator.set(athenzDomainPath(), Utf8.toBytes(domain.value()))); } public Optional<AthenzDomain> readAthenzDomain() { - if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty(); - return Optional.ofNullable(configCurator.getData(athenzDomainPath())) - .filter(domain -> ! domain.isBlank()) - .map(AthenzDomain::from); + return curator.getData(athenzDomainPath()) + .map(Utf8::toString) + .filter(domain -> !domain.isBlank()) + .map(AthenzDomain::from); } public void writeQuota(Optional<Quota> maybeQuota) { maybeQuota.ifPresent(quota -> { var bytes = uncheck(() -> SlimeUtils.toJsonBytes(quota.toSlime())); - configCurator.putData(quotaPath(), bytes); + curator.set(quotaPath(), bytes); }); } public Optional<Quota> readQuota() { - if ( ! configCurator.exists(quotaPath())) return Optional.empty(); - return Optional.ofNullable(configCurator.getData(quotaPath())) - .map(SlimeUtils::jsonToSlime) - .map(slime -> Quota.fromSlime(slime.get())); + return curator.getData(quotaPath()) + .map(SlimeUtils::jsonToSlime) + .map(slime -> Quota.fromSlime(slime.get())); } public void writeTenantSecretStores(List<TenantSecretStore> tenantSecretStores) { if (!tenantSecretStores.isEmpty()) { var bytes = uncheck(() -> SlimeUtils.toJsonBytes(TenantSecretStoreSerializer.toSlime(tenantSecretStores))); - configCurator.putData(tenantSecretStorePath(), bytes); + curator.set(tenantSecretStorePath(), bytes); } } public List<TenantSecretStore> readTenantSecretStores() { - if ( ! configCurator.exists(tenantSecretStorePath())) return List.of(); - return Optional.ofNullable(configCurator.getData(tenantSecretStorePath())) - .map(SlimeUtils::jsonToSlime) - .map(slime -> TenantSecretStoreSerializer.listFromSlime(slime.get())) - .orElse(List.of()); + return curator.getData(tenantSecretStorePath()) + .map(SlimeUtils::jsonToSlime) + .map(slime -> TenantSecretStoreSerializer.listFromSlime(slime.get())) + .orElse(List.of()); } public void writeOperatorCertificates(List<X509Certificate> certificates) { if( ! certificates.isEmpty()) { var bytes = uncheck(() -> SlimeUtils.toJsonBytes(OperatorCertificateSerializer.toSlime(certificates))); - configCurator.putData(operatorCertificatesPath(), bytes); + curator.set(operatorCertificatesPath(), bytes); } } public List<X509Certificate> readOperatorCertificates() { - if ( ! configCurator.exists(operatorCertificatesPath())) return List.of(); - return Optional.ofNullable(configCurator.getData(operatorCertificatesPath())) - .map(SlimeUtils::jsonToSlime) - .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get())) - .orElse(List.of()); + return curator.getData(operatorCertificatesPath()) + .map(SlimeUtils::jsonToSlime) + .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get())) + .orElse(List.of()); } /** @@ -312,7 +310,7 @@ public class SessionZooKeeperClient { transaction.add(CuratorOperations.create(sessionPath.getAbsolute())); transaction.add(CuratorOperations.create(sessionPath.append(UPLOAD_BARRIER).getAbsolute())); transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations()); - transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond())))); + transaction.add(CuratorOperations.create(getCreateTimePath().getAbsolute(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond())))); transaction.commit(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 2f7b397cbd9..b10a90de714 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.Lock; import com.yahoo.concurrent.Locks; @@ -30,7 +31,6 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.session.SessionRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; @@ -98,7 +98,6 @@ public class TenantRepository { private final Locks<TenantName> tenantLocks = new Locks<>(1, TimeUnit.MINUTES); private final HostRegistry hostRegistry; private final TenantListener tenantListener; - private final ConfigCurator configCurator; private final Curator curator; private final Metrics metrics; private final MetricUpdater metricUpdater; @@ -120,13 +119,14 @@ public class TenantRepository { private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); private final Optional<Curator.DirectoryCache> directoryCache; + private final ZookeeperServerConfig zookeeperServerConfig; /** * Creates a new tenant repository */ @Inject public TenantRepository(HostRegistry hostRegistry, - ConfigCurator configCurator, + Curator curator, Metrics metrics, FlagSource flagSource, SecretStore secretStore, @@ -137,9 +137,10 @@ public class TenantRepository { ModelFactoryRegistry modelFactoryRegistry, ConfigDefinitionRepo configDefinitionRepo, ReloadListener reloadListener, - TenantListener tenantListener) { + TenantListener tenantListener, + ZookeeperServerConfig zookeeperServerConfig) { this(hostRegistry, - configCurator, + curator, metrics, new StripedExecutor<>(), new StripedExecutor<>(), @@ -155,11 +156,12 @@ public class TenantRepository { modelFactoryRegistry, configDefinitionRepo, reloadListener, - tenantListener); + tenantListener, + zookeeperServerConfig); } public TenantRepository(HostRegistry hostRegistry, - ConfigCurator configCurator, + Curator curator, Metrics metrics, StripedExecutor<TenantName> zkApplicationWatcherExecutor , StripedExecutor<TenantName> zkSessionWatcherExecutor, @@ -175,12 +177,13 @@ public class TenantRepository { ModelFactoryRegistry modelFactoryRegistry, ConfigDefinitionRepo configDefinitionRepo, ReloadListener reloadListener, - TenantListener tenantListener) { + TenantListener tenantListener, + ZookeeperServerConfig zookeeperServerConfig) { this.hostRegistry = hostRegistry; this.configserverConfig = configserverConfig; this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), new DaemonThreadFactory("bootstrap-tenant-")); - this.curator = configCurator.curator(); + this.curator = curator; this.metrics = metrics; metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.zkCacheExecutor = zkCacheExecutor; @@ -197,7 +200,7 @@ public class TenantRepository { this.configDefinitionRepo = configDefinitionRepo; this.reloadListener = reloadListener; this.tenantListener = tenantListener; - this.configCurator = configCurator; + this.zookeeperServerConfig = zookeeperServerConfig; curator.framework().getConnectionStateListenable().addListener(this::stateChanged); @@ -344,7 +347,7 @@ public class TenantRepository { SessionRepository sessionRepository = new SessionRepository(tenantName, applicationRepo, sessionPreparer, - configCurator, + curator, metrics, zkSessionWatcherExecutor, permanentApplicationPackage, @@ -357,7 +360,8 @@ public class TenantRepository { zone, clock, modelFactoryRegistry, - configDefinitionRepo); + configDefinitionRepo, + zookeeperServerConfig.juteMaxBuffer()); log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created + ". Bootstrapping in " + Duration.between(start, Instant.now())); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created); @@ -528,7 +532,7 @@ public class TenantRepository { } public void close() { - directoryCache.ifPresent(Curator.DirectoryCache::close); + directoryCache.ifPresent(com.yahoo.vespa.curator.Curator.DirectoryCache::close); try { zkCacheExecutor.shutdown(); checkForRemovedApplicationsService.shutdown(); @@ -601,6 +605,6 @@ public class TenantRepository { return barriersPath; } - public Curator getCurator() { return curator; } + public com.yahoo.vespa.curator.Curator getCurator() { return curator; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java deleted file mode 100644 index 5bfa06a29dd..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.curator.Curator; - -import java.util.List; -import java.util.logging.Level; - -/** - * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server - * and knows about how config content is mapped to node names and stored. - * <p> - * Usage details: - * Config ids are stored as foo#bar#c0 instead of foo/bar/c0, for simplicity. - * Keep the amount of domain-specific logic here to a minimum. - * Data for one application x is stored on this form: - * /config/v2/tenants/x/sessions/y/defconfigs - * /config/v2/tenants/x/sessions/y/userapp - * <p> - * The user application structure is exactly the same as in the user's app dir during deploy. - * The current live app id (for example y) is stored in the node //config/v2/tenants/x/applications/<application-id> - * It is updated outside this class, typically in config server when activating config - * - * @author Vegard Havdal - * @author bratseth - */ -public class ConfigCurator { - - /** Path for def files, under one app */ - public static final String DEFCONFIGS_ZK_SUBPATH = "/defconfigs"; - - /** Path for def files, under one app */ - public static final String USER_DEFCONFIGS_ZK_SUBPATH = "/userdefconfigs"; - - /** Path for metadata about an application */ - public static final String META_ZK_PATH = "/meta"; - - /** Path for the app package's dir structure, under one app */ - public static final String USERAPP_ZK_SUBPATH = "/userapp"; - - /** Path for session state */ - public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; - - private final Curator curator; - - public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName()); - - /** The maximum size of a ZooKeeper node */ - private final int maxNodeSize; - - public static ConfigCurator create(Curator curator) { - return new ConfigCurator(curator, 1024*1024*10); - } - - @Inject - public ConfigCurator(Curator curator, ZookeeperServerConfig config) { - this(curator, config.juteMaxBuffer()); - } - - private ConfigCurator(Curator curator, int maxNodeSize) { - this.curator = curator; - this.maxNodeSize = maxNodeSize; - log.log(Level.CONFIG, "Using jute max buffer size " + this.maxNodeSize); - testZkConnection(); - } - - /** Returns the curator instance this wraps */ - public Curator curator() { return curator; } - - /** Cleans and creates a zookeeper completely */ - void initAndClear(String path) { - try { - if (exists(path)) - deleteRecurse(path); - createRecurse(path); - } - catch (Exception e) { - throw new RuntimeException("Exception clearing path " + path + " in ZooKeeper", e); - } - } - - /** Creates a path. If the path already exists this does nothing. */ - private void createRecurse(String path) { - try { - if (exists(path)) return; - curator.framework().create().creatingParentsIfNeeded().forPath(path); - } - catch (Exception e) { - throw new RuntimeException("Exception creating path " + path + " in ZooKeeper", e); - } - } - - /** Returns the data at a path and node. Replaces / by # in node names. */ - public String getData(String path, String node) { - return getData(createFullPath(path, node)); - } - - /** Returns the data at a path */ - public String getData(String path) { - byte[] data = getBytes(path); - return (data == null) ? null : Utf8.toString(data); - } - - /** - * Returns the data at a path, or null if the path does not exist. - * - * @param path a String with a pathname. - * @return a byte array with data. - */ - public byte[] getBytes(String path) { - if ( ! exists(path)) throw new IllegalArgumentException("Cannot read data from path " + path + ", it does not exist"); - - try { - return curator.framework().getData().forPath(path); - } - catch (Exception e) { - throw new RuntimeException("Exception reading from path " + path + " in ZooKeeper", e); - } - } - - /** Returns whether a path exists in zookeeper */ - public boolean exists(String path, String node) { - return exists(createFullPath(path, node)); - } - - /** Returns whether a path exists in zookeeper */ - public boolean exists(String path) { - try { - return curator.framework().checkExists().forPath(path) != null; - } - catch (Exception e) { - throw new RuntimeException("Exception checking existence of path " + path + " in ZooKeeper", e); - } - } - - /** Creates a Zookeeper node. If the node already exists this does nothing. */ - public void createNode(String path) { - if ( ! exists(path)) - createRecurse(path); - } - - /** Creates a Zookeeper node synchronously. Replaces / by # in node names. */ - public void createNode(String path, String node) { - createNode(createFullPath(path, node)); - } - - private String createFullPath(String path, String node) { - return path + "/" + toConfigserverName(node); - } - - /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */ - public void putData(String path, String node, String data) { - putData(path, node, Utf8.toBytes(data)); - } - - /** Sets data at a given path. Creates the node if it doesn't exist */ - public void putData(String path, String data) { - putData(path, Utf8.toBytes(data)); - } - - private void ensureDataIsNotTooLarge(byte[] toPut, String path) { - if (toPut.length >= maxNodeSize) { - throw new IllegalArgumentException("Error: too much zookeeper data in node: " - + "[" + toPut.length + " bytes] (path " + path + ")"); - } - } - - /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */ - private void putData(String path, String node, byte[] data) { - putData(createFullPath(path, node), data); - } - - /** Sets data at a given path. Creates the path if it doesn't exist */ - public void putData(String path, byte[] data) { - try { - ensureDataIsNotTooLarge(data, path); - if (exists(path)) - curator.framework().setData().forPath(path, data); - else - curator.framework().create().creatingParentsIfNeeded().forPath(path, data); - } - catch (Exception e) { - throw new RuntimeException("Exception writing to path " + path + " in ZooKeeper", e); - } - } - - /** - * Replaces / with # in the given node. - * - * @param node a zookeeper node name - * @return a config server node name - */ - private String toConfigserverName(String node) { - if (node.startsWith("/")) node = node.substring(1); - return node.replaceAll("/", "#"); - } - - /** - * Lists thh children at the given path. - * - * @return the local names of the children at this path, or an empty list (never null) if none. - */ - public List<String> getChildren(String path) { - try { - return curator.framework().getChildren().forPath(path); - } - catch (Exception e) { - throw new RuntimeException("Exception getting children of path " + path + " in ZooKeeper", e); - } - } - - /** - * Puts config definition data and metadata into ZK. - * - * @param name The config definition name (including namespace) - * @param path /zoopath - * @param data The contents to write to ZK (as a byte array) - */ - public void putDefData(String name, String path, byte[] data) { - putData(path, name, data); - } - - /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */ - public void deleteRecurse(String path) { - try { - if ( ! exists(path)) return; - curator.framework().delete().deletingChildrenIfNeeded().forPath(path); - } - catch (Exception e) { - throw new RuntimeException("Exception deleting path " + path, e); - } - } - - private void testZkConnection() { // This is not necessary, but allows us to give a useful error message - if (curator.connectionSpec().isEmpty()) return; - try { - curator.framework().checkExists().forPath("/dummy"); - } - catch (Exception e) { - log.log(Level.SEVERE, "Unable to connect to ZooKeeper on " + curator.connectionSpec() + - ". Please verify that VESPA_CONFIGSERVERS points to the correct configserver(s) " + - "on all config server nodes and are the same config server(s) as in services.xml, " + - "and that they are started. " + - "Check the log(s) for config server errors. Aborting.", e); - } - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 216c9edd0d0..f36ec164a14 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.config.server.zookeeper; import java.util.logging.Level; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.recipes.CuratorCounter; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Logger; /** * A counter that sets its initial value to the number of apps in zookeeper if no counter value is set. Subclass @@ -16,14 +20,14 @@ import java.util.List; */ public class InitializedCounter { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(InitializedCounter.class.getName()); + private static final Logger log = java.util.logging.Logger.getLogger(InitializedCounter.class.getName()); final CuratorCounter counter; - private final String sessionsDirPath; + private final Path sessionsDirPath; - InitializedCounter(ConfigCurator configCurator, String counterPath, String sessionsDirPath) { + InitializedCounter(Curator curator, Path counterPath, Path sessionsDirPath) { this.sessionsDirPath = sessionsDirPath; - this.counter = new CuratorCounter(configCurator.curator(), counterPath); - initializeCounterValue(getLatestSessionId(configCurator, sessionsDirPath)); + this.counter = new CuratorCounter(curator, counterPath); + initializeCounterValue(getLatestSessionId(curator, sessionsDirPath)); } private void initializeCounterValue(Long latestSessionId) { @@ -40,8 +44,8 @@ public class InitializedCounter { * * @return true, if an application exists, false otherwise */ - private static boolean applicationExists(ConfigCurator configCurator, String appsPath) { - return configCurator.exists(appsPath); + private static boolean applicationExists(Curator curator, Path appsPath) { + return curator.exists(appsPath); } /** @@ -50,12 +54,12 @@ public class InitializedCounter { * * @return generation of the latest deployed application */ - private static Long getLatestSessionId(ConfigCurator configCurator, String appsPath) { - if (!applicationExists(configCurator, appsPath)) return null; + private static Long getLatestSessionId(Curator curator, Path appsPath) { + if (!applicationExists(curator, appsPath)) return null; Long newestGeneration = null; try { - if (!getDeployedApplicationGenerations(configCurator, appsPath).isEmpty()) { - newestGeneration = Collections.max(getDeployedApplicationGenerations(configCurator, appsPath)); + if (!getDeployedApplicationGenerations(curator, appsPath).isEmpty()) { + newestGeneration = Collections.max(getDeployedApplicationGenerations(curator, appsPath)); } } catch (Exception e) { log.log(Level.WARNING, "Could not get newest application generation from Zookeeper"); @@ -63,10 +67,10 @@ public class InitializedCounter { return newestGeneration; } - private static List<Long> getDeployedApplicationGenerations(ConfigCurator configCurator, String appsPath) { + private static List<Long> getDeployedApplicationGenerations(Curator curator, Path appsPath) { ArrayList<Long> generations = new ArrayList<>(); try { - List<String> stringGenerations = configCurator.getChildren(appsPath); + List<String> stringGenerations = curator.getChildren(appsPath); if (stringGenerations != null && !(stringGenerations.isEmpty())) { for (String s : stringGenerations) { generations.add(Long.parseLong(s)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java index 17d0f7e426e..c30743bbaea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Curator; /** * A counter keeping track of session ids in an atomic fashion across multiple config servers. @@ -11,10 +12,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; */ public class SessionCounter extends InitializedCounter { - public SessionCounter(ConfigCurator configCurator, TenantName tenantName) { - super(configCurator, - TenantRepository.getTenantPath(tenantName).append("sessionCounter").getAbsolute(), - TenantRepository.getSessionsPath(tenantName).getAbsolute()); + public SessionCounter(Curator curator, TenantName tenantName) { + super(curator, + TenantRepository.getTenantPath(tenantName).append("sessionCounter"), + TenantRepository.getSessionsPath(tenantName)); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 084f26bd368..04ce5b4f63a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -3,11 +3,12 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -18,12 +19,29 @@ import java.util.List; */ public class ZKApplication { - private final ConfigCurator zk; + /** Path for def files, under one app */ + public static final String DEFCONFIGS_ZK_SUBPATH = "/defconfigs"; + /** Path for def files, under one app */ + public static final String USER_DEFCONFIGS_ZK_SUBPATH = "/userdefconfigs"; + /** Path for metadata about an application */ + public static final String META_ZK_PATH = "/meta"; + /** Path for the app package's dir structure, under one app */ + public static final String USERAPP_ZK_SUBPATH = "/userapp"; + /** Path for session state */ + public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; + private final Curator curator; private final Path appPath; + /** The maximum size of a ZooKeeper node */ + private final int maxNodeSize; - ZKApplication(ConfigCurator zk, Path appPath) { - this.zk = zk; + ZKApplication(Curator curator, Path appPath, int maxNodeSize) { + this.curator = curator; this.appPath = appPath; + this.maxNodeSize = maxNodeSize; + } + + ZKApplication(Curator curator, Path appPath) { + this(curator, appPath, 10 * 1024 * 1024); } /** @@ -37,7 +55,7 @@ public class ZKApplication { * @return the files in the given path, or an empty list if the directory does not exist or is empty. * The list gets owned by the caller and can be modified freely. */ - List<NamedReader> getAllDataFromDirectory(String path, String fileNameSuffix, boolean recursive) { + List<NamedReader> getAllDataFromDirectory(Path path, String fileNameSuffix, boolean recursive) { return getAllDataFromDirectory(path, "", fileNameSuffix, recursive); } @@ -46,23 +64,22 @@ public class ZKApplication { * * @param namePrefix the prefix to prepend to the returned reader names */ - private List<NamedReader> getAllDataFromDirectory(String path, String namePrefix, String fileNameSuffix, boolean recursive) { - String fullPath = getFullPath(path); + private List<NamedReader> getAllDataFromDirectory(Path path, String namePrefix, String fileNameSuffix, boolean recursive) { List<NamedReader> result = new ArrayList<>(); List<String> children = getChildren(path); try { for (String child : children) { if (fileNameSuffix == null || child.endsWith(fileNameSuffix)) { - result.add(new NamedReader(namePrefix + child, reader(zk.getData(fullPath, child)))); + result.add(new NamedReader(namePrefix + child, reader(getData(path.append(child))))); } if (recursive) - result.addAll(getAllDataFromDirectory(path + "/" + child, + result.addAll(getAllDataFromDirectory(path.append(child), namePrefix + child + "/", fileNameSuffix, recursive)); } return result; } catch (Exception e) { - throw new RuntimeException("Could not retrieve all data from '" + fullPath + "' in zookeeper", e); + throw new RuntimeException("Could not retrieve all data from '" + path + "' in zookeeper", e); } } @@ -70,59 +87,45 @@ public class ZKApplication { * Retrieves a node relative to the node of the live application * * @param path a path relative to the currently active application - * @param node a path relative to the path above * @return a Reader that can be used to get the data */ - Reader getDataReader(String path, String node) { - return reader(getData(path, node)); + Reader getDataReader(Path path) { + return reader(getData(path)); } - public String getData(String path, String node) { - if ( ! exists(path, node)) throw new IllegalArgumentException("No node for " + getFullPath(path) + "/" + node + " exists"); - - try { - return zk.getData(getFullPath(path), node); - } catch (Exception e) { - throw new IllegalArgumentException("Could not retrieve node '" + - getFullPath(path) + "/" + node + "' in zookeeper", e); - } + NamedReader getNamedReader(String name, Path path) { + return new NamedReader(name, reader(getData(path))); } - public String getData(String path) { - if ( ! exists(path)) throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); + public String getData(Path path) { + return Utf8.toString(getBytesInternal(getFullPath(path))); + } - try { - return zk.getData(getFullPath(path)); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e); - } + private byte[] getBytesInternal(Path path) { + return curator.getData(path) + .orElseThrow(() -> new IllegalArgumentException("Could not get data from '" + + path + "' in zookeeper")); } - public byte[] getBytes(String path) { - try { - return zk.getBytes(getFullPath(path)); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e); - } + public byte[] getBytes(Path path) { + return getBytesInternal(getFullPath(path)); } - void putData(String path, String data) { + void putData(Path path, String data) { + byte[] bytes = Utf8.toBytes(data); + ensureDataIsNotTooLarge(bytes, path); try { - zk.putData(getFullPath(path), data); + curator.set(getFullPath(path), bytes); } catch (RuntimeException e) { throw new IllegalArgumentException("Could not put data to node '" + getFullPath(path) + "' in zookeeper", e); } } - /** - * Checks if the given node exists under path under this live app - * - * @param path a zookeeper path - * @param node a zookeeper node - * @return true if the node exists in the path, false otherwise - */ - public boolean exists(String path, String node) { - return zk.exists(getFullPath(path), node); + private void ensureDataIsNotTooLarge(byte[] toPut, Path path) { + if (toPut.length >= maxNodeSize) { + throw new IllegalArgumentException("Error: too much zookeeper data in node: " + + "[" + toPut.length + " bytes] (path " + path + ")"); + } } /** @@ -131,16 +134,16 @@ public class ZKApplication { * @param path a zookeeper path * @return true if the node exists in the path, false otherwise */ - public boolean exists(String path) { - return zk.exists(getFullPath(path)); + public boolean exists(Path path) { + return curator.exists(getFullPath(path)); } - private String getFullPath(String path) { + private Path getFullPath(Path path) { Path fullPath = appPath; if (path != null) { fullPath = appPath.append(path); } - return fullPath.getAbsolute(); + return fullPath; } /** @@ -148,8 +151,8 @@ public class ZKApplication { * * @param path path to delete */ - void deleteRecurse(String path) { - zk.deleteRecurse(getFullPath(path)); + void deleteRecurse(Path path) { + curator.delete(getFullPath(path)); } /** @@ -158,31 +161,21 @@ public class ZKApplication { * @param path a path relative to the currently active application * @return a list of file names, which is empty (never null) if the path does not exist */ - public List<String> getChildren(String path) { - String fullPath = getFullPath(path); - if (! zk.exists(fullPath)) return Collections.emptyList(); - return zk.getChildren(fullPath); + public List<String> getChildren(Path path) { + return curator.getChildren(getFullPath(path)); } private static Reader reader(String string) { return new StringReader(string); } - public void create(String path) { - if (path != null && !path.startsWith("/")) path = "/" + path; + public void create(Path path) { try { - zk.createNode(getFullPath(path)); + curator.create(getFullPath(path)); } catch (RuntimeException e) { throw new IllegalArgumentException(e); } } - Reader getDataReader(String path) { - String data = getData(path); - if (data == null) - throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); - return reader(data); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 674c0f72c40..3b828e32ed7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -5,14 +5,23 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.path.Path; import com.yahoo.io.IOUtils; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; import java.util.logging.Level; import com.yahoo.vespa.config.util.ConfigUtils; -import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; + /** * @author Ulf Lilleengen * @author Vegard Havdal @@ -30,7 +39,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public boolean isDirectory() { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); if (zkApp.exists(zkPath)) { String data = zkApp.getData(zkPath); return data == null || data.isEmpty() || ! zkApp.getChildren(zkPath).isEmpty(); @@ -41,7 +50,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public boolean exists() { try { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); return zkApp.exists(zkPath); } catch (RuntimeException e) { return false; @@ -60,7 +69,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public Reader createReader() throws FileNotFoundException { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); if ( ! zkApp.exists(zkPath)) throw new FileNotFoundException("No such path: " + path); return new StringReader(zkApp.getData(zkPath)); @@ -68,7 +77,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public InputStream createInputStream() throws FileNotFoundException { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); if ( ! zkApp.exists(zkPath)) throw new FileNotFoundException("No such path: " + path); return new ByteArrayInputStream(zkApp.getBytes(zkPath)); @@ -76,7 +85,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public ApplicationFile createDirectory() { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); if (isDirectory()) return this; if (exists()) { throw new IllegalArgumentException("Unable to create directory, file exists: " + path); @@ -88,7 +97,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public ApplicationFile writeFile(Reader input) { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); try { String data = IOUtils.readAll(input); String status = ContentStatusNew; @@ -105,7 +114,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public ApplicationFile appendFile(String value) { - String zkPath = getZKPath(path); + Path zkPath = getZKPath(path); String status = ContentStatusNew; if (zkApp.exists(zkPath)) { status = ContentStatusChanged; @@ -120,7 +129,7 @@ class ZKApplicationFile extends ApplicationFile { @Override public List<ApplicationFile> listFiles(PathFilter filter) { - String userPath = getZKPath(path); + Path userPath = getZKPath(path); List<ApplicationFile> ret = new ArrayList<>(); for (String zkChild : zkApp.getChildren(userPath)) { Path childPath = path.append(zkChild); @@ -132,15 +141,15 @@ class ZKApplicationFile extends ApplicationFile { return ret; } - private static String getZKPath(Path path) { + private static Path getZKPath(Path path) { if (path.isRoot()) { - return ConfigCurator.USERAPP_ZK_SUBPATH; + return Path.fromString(USERAPP_ZK_SUBPATH); } - return ConfigCurator.USERAPP_ZK_SUBPATH + "/" + path.getRelative(); + return Path.fromString(USERAPP_ZK_SUBPATH).append(path); } private void writeMetaFile(String input, String status) { - String metaPath = getZKPath(getMetaPath()); + Path metaPath = getZKPath(getMetaPath()); StringWriter writer = new StringWriter(); try { mapper.writeValue(writer, new MetaData(status, input == null ? "" : ConfigUtils.getMd5(input))); @@ -152,7 +161,7 @@ class ZKApplicationFile extends ApplicationFile { } public MetaData getMetaData() { - String metaPath = getZKPath(getMetaPath()); + Path metaPath = getZKPath(getMetaPath()); log.log(Level.FINE, () -> "Getting metadata for " + metaPath); if (!zkApp.exists(getZKPath(path))) { if (zkApp.exists(metaPath)) { @@ -167,7 +176,7 @@ class ZKApplicationFile extends ApplicationFile { return new MetaData(ContentStatusNew, isDirectory() ? "" : ConfigUtils.getMd5(zkApp.getData(getZKPath(path)))); } - private MetaData getMetaDataFromZk(String metaPath) { + private MetaData getMetaDataFromZk(Path metaPath) { try { return mapper.readValue(zkApp.getBytes(metaPath), MetaData.class); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index bcb19a8f25a..cc9b799f2d9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -21,10 +21,10 @@ import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.Reader; -import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -33,6 +33,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; + /** * Represents an application residing in zookeeper. * @@ -49,16 +52,21 @@ public class ZKApplicationPackage implements ApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; - public ZKApplicationPackage(ConfigCurator zk, Path sessionPath) { - verifyAppPath(zk, sessionPath); - zkApplication = new ZKApplication(zk, sessionPath); + public ZKApplicationPackage(Curator curator, Path sessionPath, int maxNodeSize) { + verifyAppPath(curator, sessionPath); + zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize); metaData = readMetaDataFromLiveApp(zkApplication); importFileRegistries(); allocatedHosts = importAllocatedHosts(); } + // For testing + ZKApplicationPackage(Curator curator, Path sessionPath) { + this(curator, sessionPath, 10 * 1024 * 1024); + } + private Optional<AllocatedHosts> importAllocatedHosts() { - if ( ! zkApplication.exists(allocatedHostsNode)) return Optional.empty(); + if ( ! zkApplication.exists(Path.fromString(allocatedHostsNode))) return Optional.empty(); return Optional.of(readAllocatedHosts()); } @@ -69,14 +77,14 @@ public class ZKApplicationPackage implements ApplicationPackage { */ private AllocatedHosts readAllocatedHosts() { try { - return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(allocatedHostsNode)); + return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(Path.fromString(allocatedHostsNode))); } catch (Exception e) { throw new RuntimeException("Unable to read allocated hosts", e); } } private void importFileRegistries() { - List<String> perVersionFileRegistryNodes = zkApplication.getChildren(fileRegistryNode); + List<String> perVersionFileRegistryNodes = zkApplication.getChildren(Path.fromString(fileRegistryNode)); perVersionFileRegistryNodes .forEach(version -> fileRegistryMap.put(Version.fromString(version), @@ -85,18 +93,19 @@ public class ZKApplicationPackage implements ApplicationPackage { private PreGeneratedFileRegistry importFileRegistry(String fileRegistryNode) { try { - return PreGeneratedFileRegistry.importRegistry(zkApplication.getDataReader(fileRegistryNode)); + return PreGeneratedFileRegistry.importRegistry(zkApplication.getDataReader(Path.fromString(fileRegistryNode))); } catch (Exception e) { throw new RuntimeException("Could not determine which files to distribute", e); } } private ApplicationMetaData readMetaDataFromLiveApp(ZKApplication liveApp) { - String metaDataString = liveApp.getData(ConfigCurator.META_ZK_PATH); + Path metaPath = Path.fromString(ZKApplication.META_ZK_PATH); + String metaDataString = liveApp.getData(metaPath); if (metaDataString == null || metaDataString.isEmpty()) { return null; } - return ApplicationMetaData.fromJsonString(liveApp.getData(ConfigCurator.META_ZK_PATH)); + return ApplicationMetaData.fromJsonString(liveApp.getData(metaPath)); } @Override @@ -104,8 +113,8 @@ public class ZKApplicationPackage implements ApplicationPackage { return metaData; } - private static void verifyAppPath(ConfigCurator zk, Path appPath) { - if (!zk.exists(appPath.getAbsolute())) + private static void verifyAppPath(Curator zk, Path appPath) { + if (!zk.exists(appPath)) throw new RuntimeException("App with path " + appPath + " does not exist"); } @@ -125,7 +134,7 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getHosts() { - if (zkApplication.exists(ConfigCurator.USERAPP_ZK_SUBPATH, HOSTS)) + if (zkApplication.exists(Path.fromString(USERAPP_ZK_SUBPATH).append(HOSTS))) return getUserAppData(HOSTS); return null; } @@ -133,9 +142,9 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getSchemas() { List<NamedReader> schemas = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { + for (String sd : zkApplication.getChildren(Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR))) { if (sd.endsWith(SD_NAME_SUFFIX)) - schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); + schemas.add(zkApplication.getNamedReader(sd, Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR).append(sd))); } return schemas; } @@ -161,7 +170,7 @@ public class ZKApplicationPackage implements ApplicationPackage { private Reader retrieveConfigDefReader(String def) { try { - return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); + return zkApplication.getNamedReader("configdefinition", Path.fromString(DEFCONFIGS_ZK_SUBPATH).append(def)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not retrieve config definition " + def, e); } @@ -171,7 +180,7 @@ public class ZKApplicationPackage implements ApplicationPackage { public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { Map<ConfigDefinitionKey, UnparsedConfigDefinition> ret = new LinkedHashMap<>(); - List<String> allDefs = zkApplication.getChildren(ConfigCurator.DEFCONFIGS_ZK_SUBPATH); + List<String> allDefs = zkApplication.getChildren(Path.fromString(DEFCONFIGS_ZK_SUBPATH)); for (String nodeName : allDefs) { ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName); @@ -201,7 +210,7 @@ public class ZKApplicationPackage implements ApplicationPackage { */ @Override public List<NamedReader> getFiles(Path relativePath, String suffix, boolean recurse) { - return zkApplication.getAllDataFromDirectory(ConfigCurator.USERAPP_ZK_SUBPATH + '/' + relativePath.getRelative(), suffix, recurse); + return zkApplication.getAllDataFromDirectory(Path.fromString(USERAPP_ZK_SUBPATH).append(relativePath), suffix, recurse); } @Override @@ -226,7 +235,7 @@ public class ZKApplicationPackage implements ApplicationPackage { public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES.getName()); } private Optional<Reader> optionalFile(String file) { - if (zkApplication.exists(ConfigCurator.USERAPP_ZK_SUBPATH, file)) + if (zkApplication.exists(Path.fromString(USERAPP_ZK_SUBPATH).append(file))) return Optional.of(getUserAppData(file)); else return Optional.empty(); @@ -246,17 +255,17 @@ public class ZKApplicationPackage implements ApplicationPackage { } private Reader getUserAppData(String node) { - return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH, node); + return zkApplication.getDataReader(Path.fromString(USERAPP_ZK_SUBPATH).append(node)); } @Override public Reader getRankingExpression(String name) { - return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); + return zkApplication.getDataReader(Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR).append(name)); } @Override public File getFileReference(Path pathRelativeToAppDir) { - String path = ConfigCurator.USERAPP_ZK_SUBPATH + "/" + pathRelativeToAppDir.getRelative(); + Path path = Path.fromString(USERAPP_ZK_SUBPATH).append(pathRelativeToAppDir); // File does not exist: Manufacture a non-existing file if ( ! zkApplication.exists(path)) return new File(pathRelativeToAppDir.getRelative()); @@ -266,8 +275,8 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public void validateIncludeDir(String dirName) { - String fullPath = ConfigCurator.USERAPP_ZK_SUBPATH + "/" + dirName; - if ( ! zkApplication.exists(fullPath)) { + Path path = Path.fromString(USERAPP_ZK_SUBPATH).append(dirName); + if ( ! zkApplication.exists(path)) { throw new IllegalArgumentException("Cannot include directory '" + dirName + "', as it does not exist in ZooKeeper!"); } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index d0e366b11a1..13a660221fd 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -30,7 +30,6 @@ <component id="com.yahoo.vespa.config.server.host.HostRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.ApplicationRepository" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> - <component id="com.yahoo.vespa.config.server.zookeeper.ConfigCurator" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker" bundle="configserver" /> <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" /> diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 24acf79705d..f9eb5e3a0a5 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -175,6 +175,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.io.tmpdir=${VESPA_HOME}/tmp \ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 934440f03d1..1b31d02d222 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -21,6 +21,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; +import com.yahoo.path.Path; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigKey; @@ -43,7 +44,6 @@ import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -104,7 +104,6 @@ public class ApplicationRepositoryTest { private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; private Curator curator; - private ConfigCurator configCurator; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -115,7 +114,6 @@ public class ApplicationRepositoryTest { @Before public void setup() throws IOException { curator = new MockCurator(); - configCurator = ConfigCurator.create(curator); ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) @@ -331,8 +329,8 @@ public class ApplicationRepositoryTest { assertNotNull(applicationData.getApplicationId()); assertNotNull(sessionRepository.getLocalSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); - String sessionNode = sessionRepository.getSessionPath(sessionId).getAbsolute(); - assertTrue(configCurator.exists(sessionNode)); + Path sessionNode = sessionRepository.getSessionPath(sessionId); + assertTrue(curator.exists(sessionNode)); TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs(); File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); assertTrue(sessionFile.exists()); @@ -344,8 +342,8 @@ public class ApplicationRepositoryTest { assertTrue(provisioner.removed()); assertEquals(tenant.getName(), provisioner.lastApplicationId().tenant()); assertEquals(applicationId(), provisioner.lastApplicationId()); - assertTrue(configCurator.exists(sessionNode)); - assertEquals(Session.Status.DELETE.name(), configCurator.getData(sessionNode + "/sessionState")); + assertTrue(curator.exists(sessionNode)); + assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertTrue(sessionFile.exists()); assertFalse(applicationRepository.delete(applicationId())); @@ -395,8 +393,8 @@ public class ApplicationRepositoryTest { assertTrue(applicationRepository.delete(applicationId())); // Session should be in state DELETE - String sessionNode = sessionRepository.getSessionPath(sessionId).getAbsolute(); - assertEquals(Session.Status.DELETE.name(), configCurator.getData(sessionNode + "/sessionState")); + Path sessionNode = sessionRepository.getSessionPath(sessionId); + assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertNotNull(sessionRepository.getRemoteSession(sessionId)); // session still exists assertNull(applicationRepository.getActiveSession(applicationId())); // but it is not active try { @@ -472,7 +470,6 @@ public class ApplicationRepositoryTest { sessionId, FilesApplicationPackage.fromFile(testApp), new SessionZooKeeperClient(curator, - configCurator, tenant1, sessionId, ConfigUtils.getCanonicalHostName())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index e20363af4e9..2b1fa9acd93 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -14,8 +14,9 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostSpec; import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; import org.junit.Rule; @@ -31,6 +32,10 @@ import java.util.Map; import java.util.Optional; import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -46,13 +51,13 @@ public class ZooKeeperClientTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private ConfigCurator zk; - private final String appPath = "/1"; + private Curator zk; + private final Path appPath = Path.fromString("/1"); @Before public void setupZK() throws IOException { - zk = ConfigCurator.create(new MockCurator()); - ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), Path.fromString(appPath)); + zk = new MockCurator(); + ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), appPath); ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("foo", "/bar/baz", @@ -81,30 +86,30 @@ public class ZooKeeperClientTest { @Test public void testInitZooKeeper() { - ConfigCurator zk = ConfigCurator.create(new MockCurator()); + Curator zk = new MockCurator(); BaseDeployLogger logger = new BaseDeployLogger(); long generation = 1L; ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, Path.fromString("/1")); zooKeeperClient.initialize(); - String appPath = "/"; + Path appPath = Path.fromString("/"); assertThat(zk.getChildren(appPath).size(), is(1)); - assertTrue(zk.exists("/" + generation)); - String currentAppPath = appPath + generation; - assertTrue(zk.exists(currentAppPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); + Path currentAppPath = appPath.append(String.valueOf(generation)); + assertTrue(zk.exists(currentAppPath)); + assertTrue(zk.exists(currentAppPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")))); assertThat(zk.getChildren(currentAppPath).size(), is(4)); } @Test public void testFeedDefFilesToZooKeeper() { - String defsPath = appPath + ConfigCurator.DEFCONFIGS_ZK_SUBPATH; - assertTrue(zk.exists(appPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); + Path defsPath = appPath.append(DEFCONFIGS_ZK_SUBPATH); + assertTrue(zk.exists(appPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")))); List<String> children = zk.getChildren(defsPath); assertEquals(defsPath + " children", 1, children.size()); Collections.sort(children); assertThat(children.get(0), is("a.b.test2")); - assertTrue(zk.exists(appPath, ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); - String userDefsPath = appPath + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; + assertTrue(zk.exists(appPath.append(USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")))); + Path userDefsPath = appPath.append(USER_DEFCONFIGS_ZK_SUBPATH); children = zk.getChildren(userDefsPath); assertThat(children.size(), is(1)); Collections.sort(children); @@ -113,8 +118,9 @@ public class ZooKeeperClientTest { @Test public void testFeedAppMetaDataToZooKeeper() { - assertTrue(zk.exists(appPath, ConfigCurator.META_ZK_PATH)); - ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(zk.getData(appPath, ConfigCurator.META_ZK_PATH)); + assertTrue(zk.exists(appPath.append(META_ZK_PATH))); + ApplicationMetaData metaData = ApplicationMetaData.fromJsonString( + Utf8.toString(zk.getData(appPath.append(META_ZK_PATH)).get())); assertTrue(metaData.getChecksum().length() > 0); assertTrue(metaData.isInternalRedeploy()); assertThat(metaData.getDeployedByUser(), is("foo")); @@ -126,34 +132,34 @@ public class ZooKeeperClientTest { @Test public void testVersionedFileRegistry() { - String fileRegPath = appPath + "/" + ZKApplicationPackage.fileRegistryNode; + Path fileRegPath = appPath.append(ZKApplicationPackage.fileRegistryNode); assertTrue(zk.exists(fileRegPath)); - assertTrue(zk.exists(fileRegPath + "/1.2.3")); - assertTrue(zk.exists(fileRegPath + "/3.2.1")); + assertTrue(zk.exists(fileRegPath.append("/1.2.3"))); + assertTrue(zk.exists(fileRegPath.append("/3.2.1"))); // assertNull("Data at " + fileRegPath, zk.getData(fileRegPath)); Not null any more .. hm } @Test public void include_dirs_are_written_to_ZK() { - assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/" + "dir1", "default.xml")); - assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain2.xml")); - assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain3.xml")); + assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("dir1").append("default.xml"))); + assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain2.xml"))); + assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain3.xml"))); } @Test public void search_chain_dir_written_to_ZK() { - assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml").getAbsolute())); - assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml").getAbsolute())); - assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml").getAbsolute())); + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml"))); + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml"))); + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml"))); } private Path appPath() { - return Path.fromString(appPath).append(ConfigCurator.USERAPP_ZK_SUBPATH); + return appPath.append(USERAPP_ZK_SUBPATH); } @Test public void testWritingHostNamesToZooKeeper() throws IOException { - ConfigCurator zk = ConfigCurator.create(new MockCurator()); + Curator zk = new MockCurator(); BaseDeployLogger logger = new BaseDeployLogger(); Path app = Path.fromString("/1"); ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app); @@ -163,9 +169,9 @@ public class ZooKeeperClientTest { ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2); zooKeeperClient.write(AllocatedHosts.withHosts(hosts)); Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); - assertTrue(zk.exists(hostsPath.getAbsolute())); + assertTrue(zk.exists(hostsPath)); - AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute())); + AllocatedHosts deserialized = fromJson(zk.getData(hostsPath).get()); assertEquals(hosts, deserialized.getHosts()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java index 641fbe5bf41..ebbb10c2d2a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java @@ -1,15 +1,16 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -33,7 +34,7 @@ public class ZooKeeperDeployerTest { @Test public void require_that_deployer_is_initialized() throws IOException { - ConfigCurator zkfacade = ConfigCurator.create(new MockCurator()); + Curator curator = new MockCurator(); File serverdbDir = folder.newFolder("serverdb"); File defsDir = new File(serverdbDir, "serverdefs"); try { @@ -42,21 +43,22 @@ public class ZooKeeperDeployerTest { e.printStackTrace(); fail(); } - deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/1")); - deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/2")); + deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/1")); + deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/2")); } - public void deploy(ApplicationPackage applicationPackage, ConfigCurator configCurator, Path appPath) throws IOException { + public void deploy(ApplicationPackage applicationPackage, Curator curator, Path appPath) throws IOException { MockDeployLogger logger = new MockDeployLogger(); - ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, appPath); + ZooKeeperClient client = new ZooKeeperClient(curator, logger, appPath); ZooKeeperDeployer deployer = new ZooKeeperDeployer(client); deployer.deploy(applicationPackage, Collections.singletonMap(new Version(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); - assertTrue(configCurator.exists(appPath.getAbsolute())); + assertTrue(curator.exists(appPath)); } private static class MockDeployLogger implements DeployLogger { @Override public void log(Level level, String message) { } } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 22a5b3795d9..46d94ec476b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -32,7 +32,7 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SecretStoreValidator; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.StaticResponse; -import com.yahoo.vespa.config.server.http.v2.ApplicationHandler.ReindexingResponse; +import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -60,7 +60,6 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Stream; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.container.jdisc.HttpRequest.createTestRequest; import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -72,9 +71,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author hmusum @@ -343,10 +341,9 @@ public class ApplicationHandlerTest { } @Test - public void testClusterControllerStatus() throws Exception { + public void testServiceStatus() throws Exception { applicationRepository.deploy(testApp, prepareParams(applicationId)); String host = "foo.yahoo.com"; - String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1"; HttpProxy mockHttpProxy = mock(HttpProxy.class); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) @@ -356,11 +353,19 @@ public class ApplicationHandlerTest { .withHttpProxy(mockHttpProxy) .build(); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); - when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName), eq("clustercontroller-status/v1/clusterName1"))) - .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); + doAnswer(invoc -> new StaticResponse(200, "text/html", "<html>" + + "host=" + invoc.getArgument(1, String.class) + "," + + "service=" + invoc.getArgument(2, String.class) + "," + + "path=" + invoc.getArgument(3, String.class) + "</html>")).when(mockHttpProxy).get(any(), any(), any(), any()); - HttpResponse response = mockHandler.handle(createTestRequest(url, GET)); - assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>"); + HttpResponse response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/container-clustercontroller/" + host + "/status/some/path/clusterName1", GET)); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=clustercontroller-status/v1/some/path/clusterName1</html>"); + + response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/status/something", GET)); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=something</html>"); + + response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/fake-service/" + host + "/status/something", GET)); + assertHttpStatusCodeAndMessage(response, 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No status page for service: fake-service\"}"); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java deleted file mode 100644 index 0451ef84e09..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.config.util.ConfigUtils; -import com.yahoo.vespa.curator.Curator; - -import java.util.Optional; - -/** - * Overrides application package fetching, because this part is hard to do without feeding a full app. - * - * @author Ulf Lilleengen - */ -public class MockSessionZKClient extends SessionZooKeeperClient { - - private final ApplicationPackage app; - private Optional<AllocatedHosts> info = Optional.empty(); - - public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId) { - this(curator, tenantName, sessionId, (ApplicationPackage) null); - } - - public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, Optional<AllocatedHosts> allocatedHosts) { - this(curator, tenantName, sessionId); - this.info = allocatedHosts; - } - - MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, ApplicationPackage application) { - super(curator, - ConfigCurator.create(curator), - tenantName, - sessionId, - ConfigUtils.getCanonicalHostName()); - this.app = application; - curator.create(TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); - } - - @Override - public ApplicationPackage loadApplicationPackage() { - if (app != null) return app; - return new MockApplicationPackage.Builder().withEmptyServices().build(); - } - - @Override - AllocatedHosts getAllocatedHosts() { - return info.orElseThrow(() -> new IllegalStateException("Could not find allocated hosts")); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 4995dc0decc..a9e7ffd5b5a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -39,7 +39,7 @@ import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.ZKApplication; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -87,7 +87,6 @@ public class SessionPreparerTest { Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; - private ConfigCurator configCurator; private SessionPreparer preparer; private final MockSecretStore secretStore = new MockSecretStore(); private ConfigserverConfig configserverConfig; @@ -101,7 +100,6 @@ public class SessionPreparerTest { @Before public void setUp() throws IOException { curator = new MockCurator(); - configCurator = ConfigCurator.create(curator); configserverConfig = new ConfigserverConfig.Builder() .fileReferencesDir(folder.newFolder().getAbsolutePath()) .configServerDBDir(folder.newFolder().getAbsolutePath()) @@ -162,7 +160,7 @@ public class SessionPreparerTest { .build(), sessionId); Path sessionPath = sessionPath(sessionId); - assertFalse(configCurator.exists(sessionPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + assertFalse(curator.exists(sessionPath.append(ZKApplication.USERAPP_ZK_SUBPATH).append("services.xml"))); } @Test @@ -181,7 +179,7 @@ public class SessionPreparerTest { @Test public void require_that_application_is_prepared() throws Exception { prepare(testApp); - assertTrue(configCurator.exists(sessionPath(1).append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + assertTrue(curator.exists(sessionPath(1).append(ZKApplication.USERAPP_ZK_SUBPATH).append("services.xml"))); } @Test(expected = InvalidApplicationException.class) @@ -234,7 +232,7 @@ public class SessionPreparerTest { @Test public void require_that_file_reference_of_application_package_is_written_to_zk() throws Exception { prepare(testApp); - assertTrue(configCurator.exists(sessionPath(1).append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute())); + assertTrue(curator.exists(sessionPath(1).append(APPLICATION_PACKAGE_REFERENCE_PATH))); } @Test @@ -376,7 +374,7 @@ public class SessionPreparerTest { } private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { - return new SessionZooKeeperClient(curator, configCurator, applicationId().tenant(), sessionId, ConfigUtils.getCanonicalHostName()); + return new SessionZooKeeperClient(curator, applicationId().tenant(), sessionId, ConfigUtils.getCanonicalHostName()); } private Path sessionPath(long sessionId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index 3fa00e3ecc3..2c131e56d67 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -285,7 +284,6 @@ public class SessionRepositoryTest { private void createSession(long sessionId, boolean wait) { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, - ConfigCurator.create(curator), tenantName, sessionId, ConfigUtils.getCanonicalHostName()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index f977011efbc..57ef55cc890 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; import com.yahoo.config.FileReference; @@ -9,7 +9,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -22,6 +21,7 @@ import java.time.Instant; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.SESSIONSTATE_ZK_SUBPATH; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -35,7 +35,6 @@ public class SessionZooKeeperClientTest { private static final TenantName tenantName = TenantName.defaultName(); private Curator curator; - private ConfigCurator configCurator; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -43,7 +42,6 @@ public class SessionZooKeeperClientTest { @Before public void setup() { curator = new MockCurator(); - configCurator = ConfigCurator.create(curator); curator.create(sessionsPath()); } @@ -68,16 +66,16 @@ public class SessionZooKeeperClientTest { int sessionId = 2; SessionZooKeeperClient zkc = createSessionZKClient(sessionId); zkc.writeStatus(Session.Status.NEW); - String path = sessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(); - assertTrue(configCurator.exists(path)); - assertThat(configCurator.getData(path), is("NEW")); + Path path = sessionPath(sessionId).append(SESSIONSTATE_ZK_SUBPATH); + assertTrue(curator.exists(path)); + assertThat(Utf8.toString(curator.getData(path).get()), is("NEW")); } @Test public void require_that_status_is_read_from_zk() { int sessionId = 3; SessionZooKeeperClient zkc = createSessionZKClient(sessionId); - curator.set(sessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH), Utf8.toBytes("PREPARE")); + curator.set(sessionPath(sessionId).append(SESSIONSTATE_ZK_SUBPATH), Utf8.toBytes("PREPARE")); assertThat(zkc.readStatus(), is(Session.Status.PREPARE)); } @@ -91,9 +89,9 @@ public class SessionZooKeeperClientTest { int sessionId = 3; SessionZooKeeperClient zkc = createSessionZKClient(sessionId); zkc.writeApplicationId(id); - String path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute(); - assertTrue(configCurator.exists(path)); - assertThat(configCurator.getData(path), is(id.serializedForm())); + Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH); + assertTrue(curator.exists(path)); + assertThat(Utf8.toString(curator.getData(path).get()), is(id.serializedForm())); } @Test @@ -163,15 +161,14 @@ public class SessionZooKeeperClientTest { private void assertApplicationIdParse(long sessionId, String idString, String expectedIdString) { SessionZooKeeperClient zkc = createSessionZKClient(sessionId); - String path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute(); - configCurator.putData(path, idString); + Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH); + curator.set(path, Utf8.toBytes(idString)); ApplicationId applicationId = zkc.readApplicationId().get(); assertThat(applicationId.serializedForm(), is(expectedIdString)); } private SessionZooKeeperClient createSessionZKClient(long sessionId) { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, - ConfigCurator.create(curator), tenantName, sessionId, ConfigUtils.getCanonicalHostName()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 464b3d1ab64..4cf188a0b33 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.Version; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; @@ -27,7 +28,6 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -208,7 +208,7 @@ public class TenantRepositoryTest { public FailingDuringBootstrapTenantRepository(ConfigserverConfig configserverConfig) { super(new HostRegistry(), - ConfigCurator.create(new MockCurator()), + new MockCurator(), Metrics.createTestMetrics(), new StripedExecutor<>(new InThreadExecutorService()), new StripedExecutor<>(new InThreadExecutorService()), @@ -224,7 +224,8 @@ public class TenantRepositoryTest { new ModelFactoryRegistry(List.of(new VespaModelFactory(new NullConfigModelRegistry()))), new TestConfigDefinitionRepo(), new TenantApplicationsTest.MockReloadListener(), - new MockTenantListener()); + new MockTenantListener(), + new ZookeeperServerConfig.Builder().myid(0).build()); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java index 687d58fd23b..47e71f6d0fa 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.NullConfigModelRegistry; @@ -17,7 +18,6 @@ import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.FlagSource; @@ -47,7 +47,7 @@ public class TestTenantRepository extends TenantRepository { ReloadListener reloadListener, TenantListener tenantListener) { super(hostRegistry, - ConfigCurator.create(curator), + curator, metrics, new StripedExecutor<>(new InThreadExecutorService()), new StripedExecutor<>(new InThreadExecutorService()), @@ -63,7 +63,8 @@ public class TestTenantRepository extends TenantRepository { modelFactoryRegistry, configDefinitionRepo, reloadListener, - tenantListener); + tenantListener, + new ZookeeperServerConfig.Builder().myid(0).build()); } public static class Builder { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java deleted file mode 100644 index 9a8aec72564..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.zookeeper; - -import com.yahoo.text.Utf8; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.*; - -/** - * Tests the ZKFacade using a curator mock. - * - * @author hmusum - */ -public class ConfigCuratorTest { - - private final String defKey1 = "attributes"; - - private final String payload1 = "attribute[5]\n" + - "attribute[0].name Popularity\n" + - "attribute[0].datatype string\n" + - "attribute[0].collectiontype single\n" + - "attribute[0].removeifzero false\n" + - "attribute[0].createifnonexistent false\n" + - "attribute[0].loadtype \"always\"\n" + - "attribute[0].uniqueonly false\n" + - "attribute[0].sparse false\n" + - "attribute[0].noupdate false\n" + - "attribute[0].fastsearch false\n" + - "attribute[0].fastaggregate false\n" + - "attribute[0].fastersearch false\n" + - "attribute[1].name atA\n" + - "attribute[1].datatype string\n" + - "attribute[1].collectiontype weightedset\n" + - "attribute[1].removeifzero false\n" + - "attribute[1].createifnonexistent false\n" + - "attribute[1].loadtype \"always\"\n" + - "attribute[1].uniqueonly false\n" + - "attribute[1].sparse false\n" + - "attribute[1].noupdate false\n" + - "attribute[1].fastsearch true\n" + - "attribute[1].fastaggregate false\n" + - "attribute[1].fastersearch false\n" + - "attribute[2].name default_fieldlength\n" + - "attribute[2].datatype uint32\n" + - "attribute[2].collectiontype single\n" + - "attribute[2].removeifzero false\n" + - "attribute[2].createifnonexistent false\n" + - "attribute[2].loadtype \"always\"\n" + - "attribute[2].uniqueonly false\n" + - "attribute[2].sparse false\n" + - "attribute[2].noupdate true\n" + - "attribute[2].fastsearch false\n" + - "attribute[2].fastaggregate false\n" + - "attribute[2].fastersearch false\n" + - "attribute[3].name default_literal_fieldlength\n" + - "attribute[3].datatype uint32\n" + - "attribute[3].collectiontype single\n" + - "attribute[3].removeifzero false\n" + - "attribute[3].createifnonexistent false\n" + - "attribute[3].loadtype \"always\"\n" + - "attribute[3].uniqueonly false\n" + - "attribute[3].sparse false\n" + - "attribute[3].noupdate true\n" + - "attribute[3].fastsearch false\n" + - "attribute[3].fastaggregate false\n" + - "attribute[3].fastersearch false\n" + - "attribute[4].name artist_fieldlength\n" + - "attribute[4].datatype uint32\n" + - "attribute[4].collectiontype single\n" + - "attribute[4].removeifzero false\n" + - "attribute[4].createifnonexistent false\n" + - "attribute[4].loadtype \"always\"\n" + - "attribute[4].uniqueonly false\n" + - "attribute[4].sparse false\n" + - "attribute[4].noupdate true\n" + - "attribute[4].fastsearch false\n" + - "attribute[4].fastaggregate false\n" + - "attribute[4].fastersearch false\n"; - - private final String payload3 = "attribute[5]\n" + - "attribute[0].name Popularity\n" + - "attribute[0].datatype String\n" + - "attribute[0].collectiontype single\n" + - "attribute[0].removeifzero false\n" + - "attribute[0].createifnonexistent false\n" + - "attribute[0].loadtype \"always\"\n" + - "attribute[0].uniqueonly false\n" + - "attribute[0].sparse false\n" + - "attribute[0].noupdate false\n" + - "attribute[0].fastsearch false\n" + - "attribute[0].fastaggregate false\n" + - "attribute[0].fastersearch false\n" + - "attribute[1].name atA\n" + - "attribute[1].datatype string\n" + - "attribute[1].collectiontype weightedset\n" + - "attribute[1].removeifzero false\n" + - "attribute[1].createifnonexistent false\n" + - "attribute[1].loadtype \"always\"\n" + - "attribute[1].uniqueonly false\n" + - "attribute[1].sparse false\n" + - "attribute[1].noupdate false\n" + - "attribute[1].fastsearch true\n" + - "attribute[1].fastaggregate false\n" + - "attribute[1].fastersearch false\n" + - "attribute[2].name default_fieldlength\n" + - "attribute[2].datatype uint32\n" + - "attribute[2].collectiontype single\n" + - "attribute[2].removeifzero false\n" + - "attribute[2].createifnonexistent false\n" + - "attribute[2].loadtype \"always\"\n" + - "attribute[2].uniqueonly false\n" + - "attribute[2].sparse false\n" + - "attribute[2].noupdate true\n" + - "attribute[2].fastsearch false\n" + - "attribute[2].fastaggregate false\n" + - "attribute[2].fastersearch false\n" + - "attribute[3].name default_literal_fieldlength\n" + - "attribute[3].datatype uint32\n" + - "attribute[3].collectiontype single\n" + - "attribute[3].removeifzero false\n" + - "attribute[3].createifnonexistent false\n" + - "attribute[3].loadtype \"always\"\n" + - "attribute[3].uniqueonly false\n" + - "attribute[3].sparse false\n" + - "attribute[3].noupdate true\n" + - "attribute[3].fastsearch false\n" + - "attribute[3].fastaggregate false\n" + - "attribute[3].fastersearch false\n" + - "attribute[4].name artist_fieldlength\n" + - "attribute[4].datatype uint32\n" + - "attribute[4].collectiontype single\n" + - "attribute[4].removeifzero false\n" + - "attribute[4].createifnonexistent false\n" + - "attribute[4].loadtype \"always\"\n" + - "attribute[4].uniqueonly false\n" + - "attribute[4].sparse false\n" + - "attribute[4].noupdate true\n" + - "attribute[4].fastsearch false\n" + - "attribute[4].fastaggregate false\n" + - "attribute[4].fastersearch false\n"; - - private void initAndClearZK(ConfigCurator zkIf) { - zkIf.initAndClear(ConfigCurator.DEFCONFIGS_ZK_SUBPATH); - zkIf.initAndClear(ConfigCurator.USERAPP_ZK_SUBPATH); - } - - private ConfigCurator deployApp() { - ConfigCurator zkIf = create(); - initAndClearZK(zkIf); - zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1, payload1); - // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, cfgKey1, payload3); - String partitionsDef = "version=7\\n" + - "dataset[].id int\\n" + - "dataset[].partbits int default=6"; - zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, "partitions", partitionsDef); - String partitionsUser = "dataset[0].partbits 8\\n"; - // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, "partitions", partitionsUser); - return zkIf; - } - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void testZKInterface() { - ConfigCurator zkIf = create(); - zkIf.putData("", "test", "foo"); - zkIf.putData("/test", "me", "bar"); - zkIf.putData("", "test;me;now,then", "baz"); - assertEquals(zkIf.getData("", "test"), "foo"); - assertEquals(zkIf.getData("/test", "me"), "bar"); - assertEquals(zkIf.getData("", "test;me;now,then"), "baz"); - } - - @Test - public void testNonExistingPath() { - ConfigCurator configCurator = create(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Cannot read data from path /non-existing, it does not exist"); - configCurator.getData("/non-existing"); - } - - @Test - public void testWatcher() { - ConfigCurator zkIf = create(); - - zkIf.putData("", "test", "foo"); - String data = zkIf.getData("/test"); - assertEquals(data, "foo"); - zkIf.putData("", "/test", "bar"); - data = zkIf.getData("/test"); - assertEquals(data, "bar"); - - zkIf.getChildren("/"); - zkIf.putData("", "test2", "foo2"); - } - - private ConfigCurator create() { - return ConfigCurator.create(new MockCurator()); - } - - @Test - public void testGetDeployedData() { - ConfigCurator zkIf = deployApp(); - assertEquals(zkIf.getData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1), payload1); - } - - @Test - public void testEmptyData() { - ConfigCurator zkIf = create(); - zkIf.createNode("/empty", "data"); - assertEquals("", zkIf.getData("/empty", "data")); - } - - @Test - public void testRecursiveDelete() { - ConfigCurator configCurator = create(); - configCurator.putData("/foo", Utf8.toBytes("sadsdfsdfsdfsdf")); - configCurator.putData("/foo/bar", Utf8.toBytes("dsfsdffds")); - configCurator.putData("/foo/baz", - Utf8.toBytes("sdf\u00F8l ksdfl skdflsk dflsdkfd welkr3k lkr e4kt4 54l4l353k l534klk3lk4l33k5l 353l4k l43k l4k")); - configCurator.putData("/foo/bar/dill", Utf8.toBytes("sdfsfe 23 42 3 3 2342")); - configCurator.putData("/foo", Utf8.toBytes("sdcfsdfsdf")); - configCurator.putData("/foo", Utf8.toBytes("sdcfsd sdfdffsdf")); - configCurator.deleteRecurse("/foo"); - assertFalse(configCurator.exists("/foo")); - assertFalse(configCurator.exists("/foo/bar")); - assertFalse(configCurator.exists("/foo/bar/dill")); - assertFalse(configCurator.exists("/foo/bar/baz")); - try { - configCurator.getChildren("/foo"); - fail("Got children from nonexisting ZK path"); - } catch (RuntimeException e) { - assertTrue(e.getCause().getMessage().matches(".*NoNode.*")); - } - configCurator.deleteRecurse("/nonexisting"); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java index f745e023126..656e81a4c2f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Test; @@ -14,12 +16,12 @@ public class InitializedCounterTest { @Test public void requireThatCounterIsInitializedFromNumberOfSessions() { - ConfigCurator configCurator = ConfigCurator.create(new MockCurator()); - configCurator.createNode("/sessions"); - configCurator.createNode("/sessions/1"); - configCurator.createNode("/sessions/2"); + Curator curator = new MockCurator(); + curator.create(Path.fromString("/sessions")); + curator.create(Path.fromString("/sessions/1")); + curator.create(Path.fromString("/sessions/2")); - InitializedCounter counter = new InitializedCounter(configCurator, "/counter", "/sessions"); + InitializedCounter counter = new InitializedCounter(curator, Path.fromString("/counter"), Path.fromString("/sessions")); assertThat(counter.counter.get(), is(2L)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java index a34c17dc909..b149aa94441 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java @@ -1,9 +1,11 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationFileTest; import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -11,6 +13,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; import static org.junit.Assert.assertTrue; /** @@ -21,20 +24,20 @@ public class ZKApplicationFileTest extends ApplicationFileTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private void feed(ConfigCurator zk, File dirToFeed) { + private void feed(Curator curator, File dirToFeed) { assertTrue(dirToFeed.isDirectory()); - String appPath = "/0"; - ZKApplicationPackageTest.feedZooKeeper(zk, dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); - zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles"); + Path appPath = Path.fromString("/0"); + ZKApplicationPackageTest.feedZooKeeper(curator, dirToFeed, appPath.append(USERAPP_ZK_SUBPATH), null, true); + curator.set(appPath.append(ZKApplicationPackage.fileRegistryNode), Utf8.toBytes("dummyfiles")); } @Override public ApplicationFile getApplicationFile(Path path) throws IOException{ - ConfigCurator configCurator = ConfigCurator.create(new MockCurator()); + Curator curator = new MockCurator(); File tmp = temporaryFolder.newFolder(); writeAppTo(tmp); - feed(configCurator, tmp); - return new ZKApplicationFile(path, new ZKApplication(configCurator, Path.fromString("/0"))); + feed(curator, tmp); + return new ZKApplicationFile(path, new ZKApplication(curator, Path.fromString("/0"))); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 458cdb82066..4b4b4b92627 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; import org.junit.Rule; @@ -31,6 +32,8 @@ import java.util.Set; import java.util.regex.Pattern; import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH; +import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -59,20 +62,20 @@ public class ZKApplicationPackageTest { Optional.of(DockerImage.fromString(dockerImage))))); } - private ConfigCurator configCurator; + private Curator curator; @Rule public TemporaryFolder tmpDir = new TemporaryFolder(); @Before public void setup() { - configCurator = ConfigCurator.create(new MockCurator()); + curator = new MockCurator(); } @Test public void testBasicZKFeed() throws IOException { - feed(configCurator, new File(APP)); - ZKApplicationPackage zkApp = new ZKApplicationPackage(configCurator, Path.fromString("/0")); + feed(curator, new File(APP)); + ZKApplicationPackage zkApp = new ZKApplicationPackage(curator, Path.fromString("/0")); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches()); assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); @@ -103,13 +106,14 @@ public class ZKApplicationPackageTest { assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); } - private void feed(ConfigCurator zk, File dirToFeed) throws IOException { + private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException { assertTrue(dirToFeed.isDirectory()); - feedZooKeeper(zk, dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + Path sessionPath = Path.fromString("/0"); + feedZooKeeper(zk, dirToFeed, sessionPath.append(USERAPP_ZK_SUBPATH), null, true); String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; - zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); - zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS)); + zk.set(sessionPath.append(META_ZK_PATH), Utf8.toBytes(metaData)); + zk.set(sessionPath.append(ZKApplicationPackage.fileRegistryNode).append("/3.0.0"), Utf8.toBytes("dummyfiles")); + zk.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode), toJson(ALLOCATED_HOSTS)); } private static class MockNodeFlavors extends NodeFlavors{ @@ -131,7 +135,7 @@ public class ZKApplicationPackageTest { * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper * @param recurse recurse subdirectories */ - static void feedZooKeeper(ConfigCurator zk, File dir, String path, FilenameFilter filenameFilter, boolean recurse) { + static void feedZooKeeper(com.yahoo.vespa.curator.Curator zk, File dir, Path path, FilenameFilter filenameFilter, boolean recurse) { try { if (filenameFilter == null) { filenameFilter = acceptsAllFileNameFilter; @@ -141,12 +145,13 @@ public class ZKApplicationPackageTest { } for (File file : listFiles(dir, filenameFilter)) { if (file.getName().startsWith(".")) continue; //.svn , .git ... + Path filePath = path.append(file.getName()); if (file.isFile()) { - String contents = IOUtils.readFile(file); - zk.putData(path, file.getName(), contents); + byte[] contents = IOUtils.readFileBytes(file); + zk.set(filePath, contents); } else if (recurse && file.isDirectory()) { - zk.createNode(path, file.getName()); - feedZooKeeper(zk, file, path + '/' + file.getName(), filenameFilter, recurse); + zk.create(filePath); + feedZooKeeper(zk, file, filePath, filenameFilter, recurse); } } } diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java new file mode 100644 index 00000000000..28199fd727b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.entity; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-core/pom.xml b/container-core/pom.xml index 2b87d79daa4..c7fe2998530 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -140,10 +140,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> @@ -211,6 +207,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -262,11 +264,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpmime</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> <scope>test</scope> @@ -302,17 +299,6 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents.client5</groupId> - <artifactId>httpclient5</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 9f6f5cf7ea1..08a468a3031 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -117,8 +117,11 @@ public class HandlersConfigurerDi { @Override public void installPlatformBundles(Collection<String> bundlePaths) { - log.fine("Installing platform bundles."); - platformBundleLoader.useBundles(new ArrayList<>(bundlePaths)); + // Don't install physical bundles for test frameworks, where all platform bundles are on the classpath. + if (osgiFramework.isFelixFramework()) { + log.fine("Installing platform bundles."); + platformBundleLoader.useBundles(new ArrayList<>(bundlePaths)); + } } @Override diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index d98a865e1fb..087be0f17c5 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -13,7 +13,9 @@ import com.yahoo.container.core.config.HandlersConfigurerDi; import com.yahoo.container.di.CloudSubscriberFactory; import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; @@ -139,6 +141,7 @@ public class HandlersConfigurerTestWrapper { // Needed by e.g. SearchHandler bind(Linguistics.class).to(SimpleLinguistics.class).in(Scopes.SINGLETON); bind(ContainerThreadPool.class).to(SimpleContainerThreadpool.class); + bind(Metric.class).to(MockMetric.class); } }); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java new file mode 100644 index 00000000000..9610648ad41 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author bjorncs + */ +class EchoRequestHandler extends AbstractRequestHandler { + @Override + public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { + int port = request.getUri().getPort(); + Response response = new Response(OK); + response.headers().put("Jdisc-Local-Port", Integer.toString(port)); + return handler.handleResponse(response); + } +} + diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 0f625b5c3df..bd3000a0188 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -5,14 +5,12 @@ import com.google.inject.AbstractModule; import com.google.inject.Module; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogEntry; -import com.yahoo.container.logging.ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry; import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; import com.yahoo.jdisc.References; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingSetSelector; -import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; @@ -28,12 +26,7 @@ import com.yahoo.jdisc.http.HttpResponse; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.server.jetty.JettyTestDriver.TlsClientAuth; import com.yahoo.jdisc.service.BindingSetNotFoundException; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrBuilder; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; @@ -42,39 +35,22 @@ import org.apache.hc.client5.http.entity.mime.FormBodyPartBuilder; import org.apache.hc.client5.http.entity.mime.StringBody; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.H2AsyncClientBuilder; -import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; -import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; -import org.apache.hc.core5.http2.HttpVersionPolicy; import org.assertj.core.api.Assertions; -import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1; -import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; -import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.server.handler.AbstractHandlerContainer; -import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLHandshakeException; -import javax.security.auth.x500.X500Principal; import java.io.IOException; -import java.math.BigInteger; import java.net.BindException; import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -85,10 +61,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Pattern; import static com.yahoo.jdisc.Response.Status.GATEWAY_TIMEOUT; @@ -105,8 +78,8 @@ import static com.yahoo.jdisc.http.HttpHeaders.Names.X_DISABLE_CHUNKING; import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED; import static com.yahoo.jdisc.http.HttpHeaders.Values.CLOSE; import static com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.ResponseValidator; -import static com.yahoo.security.KeyAlgorithm.EC; -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; import static org.cthul.matchers.CthulMatchers.containsPattern; import static org.cthul.matchers.CthulMatchers.matchesPattern; import static org.hamcrest.CoreMatchers.containsString; @@ -114,13 +87,10 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.anyOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -133,8 +103,6 @@ import static org.mockito.Mockito.when; */ public class HttpServerTest { - private static final Logger log = Logger.getLogger(HttpServerTest.class.getName()); - @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -666,205 +634,6 @@ public class HttpServerTest { } @Test - public void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .build(); - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1"); - assumeTrue("TLSv1.1 must be enabled in installed JDK", tlsv11Enabled); - - assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake() throws IOException { - Path serverPrivateKeyFile = tmpFolder.newFile().toPath(); - Path serverCertificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(serverPrivateKeyFile, serverCertificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(serverCertificateFile, serverPrivateKeyFile, metricConsumer, connectionLog); - - Path clientPrivateKeyFile = tmpFolder.newFile().toPath(); - Path clientCertificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile); - - SSLContext clientCtx = new SslContextBuilder() - .withKeyStore(clientPrivateKeyFile, clientCertificateFile) - .withTrustStore(serverCertificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake() throws IOException { - Path rootPrivateKeyFile = tmpFolder.newFile().toPath(); - Path rootCertificateFile = tmpFolder.newFile().toPath(); - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS); - generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile, privateKeyFile, certificateFile, notAfter); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(rootCertificateFile, rootPrivateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(rootCertificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - - } - - @Test - public void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false); - - String proxiedRemoteAddress = "192.168.0.100"; - int proxiedRemotePort = 12345; - sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, proxiedRemotePort)); - sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, proxiedRemotePort)); - assertTrue(driver.close()); - - assertEquals(2, requestLogMock.entries().size()); - assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort); - assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort); - Assertions.assertThat(connectionLog.logEntries()).hasSize(2); - assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort); - assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get()); - assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort); - assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get()); - } - - @Test - public void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true); - - String proxiedRemoteAddress = "192.168.0.100"; - sendJettyClientRequest(driver, certificateFile, null); - sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, 12345)); - sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, 12345)); - assertTrue(driver.close()); - - assertEquals(3, requestLogMock.entries().size()); - assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0); - assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0); - assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0); - Assertions.assertThat(connectionLog.logEntries()).hasSize(3); - assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0); - assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345); - assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345); - } - - @Test - public void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, connectionLog, /*mixedMode*/false); - - String proxiedRemoteAddress = "192.168.0.100"; - int proxiedRemotePort = 12345; - String proxyLocalAddress = "10.0.0.10"; - int proxyLocalPort = 23456; - V2.Tag v2Tag = new V2.Tag(V2.Tag.Command.PROXY, null, V2.Tag.Protocol.STREAM, - proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null); - ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag); - assertTrue(driver.close()); - - int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port")); - assertNotEquals(proxyLocalPort, clientPort); - assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue()); - } - - @Test public void requireThatConnectionIsTrackedInConnectionLog() throws Exception { Path privateKeyFile = tmpFolder.newFile().toPath(); Path certificateFile = tmpFolder.newFile().toPath(); @@ -932,41 +701,6 @@ public class HttpServerTest { .set(MetricDefinitions.REQUESTS_PER_CONNECTION, 1L, MetricConsumerMock.STATIC_CONTEXT); } - private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag) - throws Exception { - HttpClient client = createJettyHttpClient(certificateFile); - try { - int maxAttempts = 3; - for (int attempt = 0; attempt < maxAttempts; attempt++) { - try { - ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) - .tag(tag) - .send(); - assertEquals(200, response.getStatus()); - return response; - } catch (ExecutionException e) { - // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. - // We have been unable to reproduce this locally. The cause is therefor currently unknown. - log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); - Thread.sleep(10); - } - } - throw new AssertionError("Failed to send request, see log for details"); - } finally { - client.stop(); - } - } - - // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. - private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception { - SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client(); - clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); - clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); - - HttpClient client = new HttpClient(clientSslCtxFactory); - client.start(); - return client; - } private static CloseableHttpAsyncClient createHttp2Client(JettyTestDriver driver) { TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() @@ -980,58 +714,6 @@ public class HttpServerTest { return client; } - private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) { - assertEquals(expectedAddress, entry.peerAddress().get()); - if (expectedPort > 0) { - assertEquals(expectedPort, entry.peerPort().getAsInt()); - } - } - - private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) { - if (expectedAddress != null) { - Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress); - } else { - Assertions.assertThat(entry.remoteAddress()).isEmpty(); - } - if (expectedPort > 0) { - Assertions.assertThat(entry.remotePort()).hasValue(expectedPort); - } else { - Assertions.assertThat(entry.remotePort()).isEmpty(); - } - } - - private static void assertSslHandshakeFailurePresent( - ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) { - Assertions.assertThat(entry.sslHandshakeFailure()).isPresent(); - ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get(); - assertEquals(expectedType, failure.type()); - ExceptionEntry exceptionEntry = failure.exceptionChain().get(0); - assertEquals(expectedException.getName(), exceptionEntry.name()); - } - - private static JettyTestDriver createSslWithProxyProtocolTestDriver( - Path certificateFile, Path privateKeyFile, RequestLog requestLog, - ConnectionLog connectionLog, boolean mixedMode) { - ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() - .http2Enabled(true) - .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder() - .enabled(true) - .mixedMode(mixedMode)) - .ssl(new ConnectorConfig.Ssl.Builder() - .enabled(true) - .privateKeyFile(privateKeyFile.toString()) - .certificateFile(certificateFile.toString()) - .caCertificateFile(certificateFile.toString())); - return JettyTestDriver.newConfiguredInstance( - new EchoRequestHandler(), - new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)), - connectorConfig, - binder -> { - binder.bind(RequestLog.class).toInstance(requestLog); - binder.bind(ConnectionLog.class).toInstance(connectionLog); - }); - } - private static JettyTestDriver createSslWithTlsClientAuthenticationEnforcer(Path certificateFile, Path privateKeyFile) { ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() .tlsClientAuthEnforcer( @@ -1051,63 +733,6 @@ public class HttpServerTest { binder -> {}); } - private static JettyTestDriver createSslTestDriver( - Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) throws IOException { - Module extraModule = binder -> { - binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock()); - binder.bind(ConnectionLog.class).toInstance(connectionLog); - }; - return JettyTestDriver.newInstanceWithSsl( - new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, TlsClientAuth.NEED, extraModule); - } - - private static void assertHttpsRequestTriggersSslHandshakeException( - JettyTestDriver testDriver, - SSLContext sslContext, - String protocolOverride, - String cipherOverride, - String expectedExceptionSubstring) throws IOException { - List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null; - List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null; - try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) { - client.get("/status.html"); - fail("SSLHandshakeException expected"); - } catch (SSLHandshakeException e) { - assertThat(e.getMessage(), containsString(expectedExceptionSubstring)); - } catch (SSLException e) { - // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread. - log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e); - // Only ignore a subset of exceptions - assertThat(e.getMessage(), anyOf(containsString("readHandshakeRecord"), containsString("Broken pipe"))); - } - } - - private static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException { - KeyPair keyPair = KeyUtils.generateKeypair(EC); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); - - X509Certificate certificate = X509CertificateBuilder - .fromKeypair( - keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) - .build(); - Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); - } - - private static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile, - Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException { - generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile); - X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile)); - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile)); - - KeyPair keyPair = KeyUtils.generateKeypair(EC); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); - Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build(); - X509Certificate certificate = X509CertificateBuilder - .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE) - .build(); - Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); - } - private static RequestHandler mockRequestHandler() { final RequestHandler mockRequestHandler = mock(RequestHandler.class); when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE); @@ -1237,17 +862,6 @@ public class HttpServerTest { } } - private static class EchoRequestHandler extends AbstractRequestHandler { - - @Override - public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { - int port = request.getUri().getPort(); - Response response = new Response(OK); - response.headers().put("Jdisc-Local-Port", Integer.toString(port)); - return handler.handleResponse(response); - } - } - private static class OkRequestHandler extends AbstractRequestHandler { @Override public ContentChannel handleRequest(Request request, ResponseHandler handler) { diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java new file mode 100644 index 00000000000..d29abea024e --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java @@ -0,0 +1,198 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.container.logging.ConnectionLogEntry; +import com.yahoo.container.logging.RequestLog; +import com.yahoo.container.logging.RequestLogEntry; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.security.SslContextBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.assertj.core.api.Assertions; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class ProxyProtocolTest { + + private static final Logger log = Logger.getLogger(ProxyProtocolTest.class.getName()); + + private static Path privateKeyFile; + private static Path certificateFile; + private InMemoryConnectionLog connectionLog; + private InMemoryRequestLog requestLogMock; + private JettyTestDriver driver; + + @BeforeAll + static void generateCrypto(@TempDir Path tmpFolder) throws IOException { + privateKeyFile = tmpFolder.resolve("key.pem"); + certificateFile = tmpFolder.resolve("cert.pem"); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + } + + @BeforeEach + void initializeServer() { + requestLogMock = new InMemoryRequestLog(); + connectionLog = new InMemoryConnectionLog(); + driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false); + } + + @Test + void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception { + String proxiedRemoteAddress = "192.168.0.100"; + int proxiedRemotePort = 12345; + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, proxiedRemotePort)); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, proxiedRemotePort)); + assertTrue(driver.close()); + + assertEquals(2, requestLogMock.entries().size()); + assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort); + assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort); + Assertions.assertThat(connectionLog.logEntries()).hasSize(2); + assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort); + assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get()); + assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort); + assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get()); + } + + @Test + void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception { + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true); + + String proxiedRemoteAddress = "192.168.0.100"; + sendJettyClientRequest(driver, certificateFile, null); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, 12345)); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, 12345)); + assertTrue(driver.close()); + + assertEquals(3, requestLogMock.entries().size()); + assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0); + assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0); + assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0); + Assertions.assertThat(connectionLog.logEntries()).hasSize(3); + assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0); + assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345); + assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345); + } + + @Test + void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception { + String proxiedRemoteAddress = "192.168.0.100"; + int proxiedRemotePort = 12345; + String proxyLocalAddress = "10.0.0.10"; + int proxyLocalPort = 23456; + ProxyProtocolClientConnectionFactory.V2.Tag v2Tag = new ProxyProtocolClientConnectionFactory.V2.Tag(ProxyProtocolClientConnectionFactory.V2.Tag.Command.PROXY, null, ProxyProtocolClientConnectionFactory.V2.Tag.Protocol.STREAM, + proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null); + ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag); + assertTrue(driver.close()); + + int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port")); + assertNotEquals(proxyLocalPort, clientPort); + assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue()); + } + + private static JettyTestDriver createSslWithProxyProtocolTestDriver( + Path certificateFile, Path privateKeyFile, RequestLog requestLog, + ConnectionLog connectionLog, boolean mixedMode) { + ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() + .http2Enabled(true) + .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder() + .enabled(true) + .mixedMode(mixedMode)) + .ssl(new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyFile.toString()) + .certificateFile(certificateFile.toString()) + .caCertificateFile(certificateFile.toString())); + return JettyTestDriver.newConfiguredInstance( + new EchoRequestHandler(), + new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)), + connectorConfig, + binder -> { + binder.bind(RequestLog.class).toInstance(requestLog); + binder.bind(ConnectionLog.class).toInstance(connectionLog); + }); + } + + private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag) + throws Exception { + HttpClient client = createJettyHttpClient(certificateFile); + try { + int maxAttempts = 3; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) + .tag(tag) + .send(); + assertEquals(200, response.getStatus()); + return response; + } catch (ExecutionException e) { + // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. + // We have been unable to reproduce this locally. The cause is therefor currently unknown. + log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); + Thread.sleep(10); + } + } + throw new AssertionError("Failed to send request, see log for details"); + } finally { + client.stop(); + } + } + + // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. + private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception { + SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client(); + clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); + + HttpClient client = new HttpClient(clientSslCtxFactory); + client.start(); + return client; + } + + private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) { + assertEquals(expectedAddress, entry.peerAddress().get()); + if (expectedPort > 0) { + assertEquals(expectedPort, entry.peerPort().getAsInt()); + } + } + + private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) { + if (expectedAddress != null) { + Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress); + } else { + Assertions.assertThat(entry.remoteAddress()).isEmpty(); + } + if (expectedPort > 0) { + Assertions.assertThat(entry.remotePort()).hasValue(expectedPort); + } else { + Assertions.assertThat(entry.remotePort()).isEmpty(); + } + } + + +} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java new file mode 100644 index 00000000000..0f71dd87d00 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java @@ -0,0 +1,189 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.container.logging.ConnectionLogEntry; +import com.yahoo.security.SslContextBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +class SslHandshakeMetricsTest { + + private static final Logger log = Logger.getLogger(SslHandshakeMetricsTest.class.getName()); + private static Path privateKeyFile; + private static Path certificateFile; + + @BeforeAll + static void generateCrypto(@TempDir Path tmpFolder) throws IOException { + privateKeyFile = tmpFolder.resolve("private-key.pem"); + certificateFile = tmpFolder.resolve("certificate.pem"); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + } + + @Test + void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .build(); + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(privateKeyFile, certificateFile) + .build(); + + boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1"); + assumeTrue(tlsv11Enabled, "TLSv1.1 must be enabled in installed JDK"); + + assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(privateKeyFile, certificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake(@TempDir Path tmpFolder) throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + Path clientPrivateKeyFile = tmpFolder.resolve("client-key.pem"); + Path clientCertificateFile = tmpFolder.resolve("client-cert.pem"); + generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile); + + SSLContext clientCtx = new SslContextBuilder() + .withKeyStore(clientPrivateKeyFile, clientCertificateFile) + .withTrustStore(certificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake(@TempDir Path tmpFolder) throws IOException { + Path endEntityKeyFile = tmpFolder.resolve("client-key.pem"); + Path endEntitycertificateFile = tmpFolder.resolve("client-cert.pem"); + Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile, endEntityKeyFile, endEntitycertificateFile, notAfter); + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(endEntityKeyFile, endEntitycertificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + } + + + private static void assertHttpsRequestTriggersSslHandshakeException( + JettyTestDriver testDriver, + SSLContext sslContext, + String protocolOverride, + String cipherOverride, + String expectedExceptionSubstring) throws IOException { + List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null; + List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null; + try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) { + client.get("/status.html"); + fail("SSLHandshakeException expected"); + } catch (SSLHandshakeException e) { + assertThat(e.getMessage()).contains(expectedExceptionSubstring); + } catch (SSLException e) { + // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread. + log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e); + // Only ignore a subset of exceptions + assertTrue(e.getMessage().contains("readHandshakeRecord") || e.getMessage().contains("Broken pipe"), e.getMessage()); + } + } + + private static void assertSslHandshakeFailurePresent( + ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) { + assertThat(entry.sslHandshakeFailure()).isPresent(); + ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get(); + assertEquals(expectedType, failure.type()); + ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry exceptionEntry = failure.exceptionChain().get(0); + assertEquals(expectedException.getName(), exceptionEntry.name()); + } + +} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java new file mode 100644 index 00000000000..626ab521773 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.google.inject.Module; +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.Pkcs10Csr; +import com.yahoo.security.Pkcs10CsrBuilder; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; + +import javax.security.auth.x500.X500Principal; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; + +/** + * @author bjorncs + */ +class Utils { + + private Utils() {} + + static JettyTestDriver createSslTestDriver( + Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) { + Module extraModule = binder -> { + binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock()); + binder.bind(ConnectionLog.class).toInstance(connectionLog); + }; + return JettyTestDriver.newInstanceWithSsl( + new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, JettyTestDriver.TlsClientAuth.NEED, extraModule); + } + + static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException { + KeyPair keyPair = KeyUtils.generateKeypair(EC); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + } + + static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile, + Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException { + X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile)); + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile)); + + KeyPair keyPair = KeyUtils.generateKeypair(EC); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build(); + X509Certificate certificate = X509CertificateBuilder + .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + } +} diff --git a/container-dev/pom.xml b/container-dev/pom.xml index cfe9c0a6d8e..6366022c342 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -96,11 +96,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container-jersey2</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespa_jersey2</artifactId> <version>${project.version}</version> <type>pom</type> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 5446c9e1698..b255b6af02a 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -170,14 +170,11 @@ configgen.jar, config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, - container-jersey2-jar-with-dependencies.jar, container-search-and-docproc-jar-with-dependencies.jar, container-search-gui-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, - model-evaluation-jar-with-dependencies.jar, - model-integration-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, security-utils-jar-with-dependencies.jar, diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 4518868cb81..223124c29d6 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -215,6 +215,7 @@ exec $numactlcmd $envcmd java \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.io.tmpdir="${VESPA_HOME}/tmp" \ diff --git a/container-jersey2/.gitignore b/container-jersey2/.gitignore deleted file mode 100644 index ea8c4bf7f35..00000000000 --- a/container-jersey2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/container-jersey2/CMakeLists.txt b/container-jersey2/CMakeLists.txt deleted file mode 100644 index d2490563372..00000000000 --- a/container-jersey2/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(container-jersey2) diff --git a/container-jersey2/OWNERS b/container-jersey2/OWNERS deleted file mode 100644 index 3b2ba1ede81..00000000000 --- a/container-jersey2/OWNERS +++ /dev/null @@ -1 +0,0 @@ -gjoranv diff --git a/container-jersey2/README.md b/container-jersey2/README.md deleted file mode 100644 index 43f83ddccbe..00000000000 --- a/container-jersey2/README.md +++ /dev/null @@ -1,4 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# JDisc Jersey integration - -The integration layer between JDisc and Jersey2. diff --git a/container-jersey2/abi-spec.json b/container-jersey2/abi-spec.json deleted file mode 100644 index 2f5ef3efb70..00000000000 --- a/container-jersey2/abi-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "com.yahoo.container.jaxrs.annotation.Component": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.annotation.Annotation" - ], - "attributes": [ - "public", - "interface", - "abstract", - "annotation" - ], - "methods": [], - "fields": [] - } -}
\ No newline at end of file diff --git a/container-jersey2/pom.xml b/container-jersey2/pom.xml deleted file mode 100644 index 8f24b5edcb7..00000000000 --- a/container-jersey2/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>container-jersey2</artifactId> - <version>7-SNAPSHOT</version> - <packaging>container-plugin</packaging> - <dependencies> - - <!-- COMPILE scope --> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - </dependency> - - - <!-- PROVIDED scope --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa_jersey2</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - <type>pom</type> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - <scope>provided</scope> - </dependency> - - <!-- TEST scope --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> - </plugin> - <plugin> - <!-- Explicit for IntelliJ to detect correct language level from parent --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/Component.java b/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/Component.java deleted file mode 100644 index da8f35eaa00..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/Component.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jaxrs.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation for injecting jdisc container components into jaxrs resources and providers - * @author Tony Vaagenes - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.PARAMETER) -public @interface Component {} - - diff --git a/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/package-info.java b/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/package-info.java deleted file mode 100644 index 8628d52bbb8..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/jaxrs/annotation/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author Tony Vaagenes - */ -@PublicApi -@ExportPackage -package com.yahoo.container.jaxrs.annotation; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java deleted file mode 100644 index 7ff9646cb27..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey; - -import com.yahoo.container.di.config.ResolveDependencyException; -import com.yahoo.container.di.config.RestApiContext; -import com.yahoo.container.jaxrs.annotation.Component; -import org.glassfish.hk2.api.Injectee; -import org.glassfish.hk2.api.InjectionResolver; -import org.glassfish.hk2.api.ServiceHandle; - -import javax.inject.Singleton; - -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Resolves jdisc container components for jersey 2 components. - * - * @author Tony Vaagenes - * @author ollivir - */ -@Singleton // jersey2 requirement: InjectionResolvers must be in the Singleton scope -public class ComponentGraphProvider implements InjectionResolver<Component> { - private Collection<RestApiContext.Injectable> injectables; - - public ComponentGraphProvider(Collection<RestApiContext.Injectable> injectables) { - this.injectables = injectables; - } - - @Override - public Object resolve(Injectee injectee, ServiceHandle<?> root) { - Class<?> wantedClass; - Type type = injectee.getRequiredType(); - if (type instanceof Class) { - wantedClass = (Class<?>) type; - } else { - throw new UnsupportedOperationException("Only classes are supported, got " + type); - } - - List<RestApiContext.Injectable> componentsWithMatchingType = new ArrayList<>(); - for (RestApiContext.Injectable injectable : injectables) { - if (wantedClass.isInstance(injectable.instance)) { - componentsWithMatchingType.add(injectable); - } - } - - if (componentsWithMatchingType.size() == 1) { - return componentsWithMatchingType.get(0).instance; - } else { - String injectionDescription = "class '" + wantedClass + "' to inject into Jersey resource/provider '" - + injectee.getInjecteeClass() + "')"; - if (componentsWithMatchingType.size() > 1) { - String ids = componentsWithMatchingType.stream().map(c -> c.id.toString()).collect(Collectors.joining(",")); - throw new ResolveDependencyException("Multiple components found of " + injectionDescription + ": " + ids); - } else { - throw new ResolveDependencyException("Could not find a component of " + injectionDescription + "."); - } - } - } - - @Override - public boolean isMethodParameterIndicator() { - return true; - } - - @Override - public boolean isConstructorParameterIndicator() { - return true; - } -} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java deleted file mode 100644 index 4c4e43bc8d5..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey; - -import javax.ws.rs.core.Application; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -public class JerseyApplication extends Application { - private Set<Class<?>> classes; - - public JerseyApplication(Collection<Class<?>> resourcesAndProviderClasses) { - this.classes = new HashSet<>(resourcesAndProviderClasses); - } - - @Override - public Set<Class<?>> getClasses() { - return classes; - } -} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java deleted file mode 100644 index b8c714fd3e3..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; -import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.container.di.config.RestApiContext; -import com.yahoo.container.di.config.RestApiContext.BundleInfo; -import com.yahoo.container.jaxrs.annotation.Component; -import org.eclipse.jetty.servlet.ServletHolder; -import org.glassfish.hk2.api.InjectionResolver; -import org.glassfish.hk2.api.TypeLiteral; -import org.glassfish.hk2.utilities.Binder; -import org.glassfish.hk2.utilities.binding.AbstractBinder; -import org.glassfish.jersey.media.multipart.MultiPartFeature; -import org.glassfish.jersey.server.ResourceConfig; -import org.glassfish.jersey.servlet.ServletContainer; -import org.objectweb.asm.ClassReader; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; - -import static com.yahoo.container.servlet.jersey.util.ResourceConfigUtil.registerComponent; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@SuppressWarnings("unused") // Injected -public class JerseyServletProvider implements Provider<ServletHolder> { - - private final ServletHolder jerseyServletHolder; - - public JerseyServletProvider(RestApiContext restApiContext) { - this.jerseyServletHolder = new ServletHolder(new ServletContainer(resourceConfig(restApiContext))); - } - - private ResourceConfig resourceConfig(RestApiContext restApiContext) { - ResourceConfig resourceConfig = ResourceConfig - .forApplication(new JerseyApplication(resourcesAndProviders(restApiContext.getBundles()))); - - registerComponent(resourceConfig, componentInjectorBinder(restApiContext)); - registerComponent(resourceConfig, jacksonDatatypeJdk8Provider()); - resourceConfig.register(MultiPartFeature.class); - - return resourceConfig; - } - - private static Collection<Class<?>> resourcesAndProviders(Collection<BundleInfo> bundles) { - List<Class<?>> ret = new ArrayList<>(); - - for (BundleInfo bundle : bundles) { - for (String classEntry : bundle.getClassEntries()) { - Optional<String> className = detectResourceOrProvider(bundle.classLoader, classEntry); - className.ifPresent(cname -> ret.add(loadClass(bundle.symbolicName, bundle.classLoader, cname))); - } - } - return ret; - } - - private static Optional<String> detectResourceOrProvider(ClassLoader bundleClassLoader, String classEntry) { - try (InputStream inputStream = getResourceAsStream(bundleClassLoader, classEntry)) { - ResourceOrProviderClassVisitor visitor = ResourceOrProviderClassVisitor.visit(new ClassReader(inputStream)); - return visitor.getJerseyClassName(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static InputStream getResourceAsStream(ClassLoader bundleClassLoader, String classEntry) { - InputStream is = bundleClassLoader.getResourceAsStream(classEntry); - if (is == null) { - throw new RuntimeException("No entry " + classEntry + " in bundle " + bundleClassLoader); - } else { - return is; - } - } - - private static Class<?> loadClass(String bundleSymbolicName, ClassLoader classLoader, String className) { - try { - return classLoader.loadClass(className); - } catch (Exception e) { - throw new RuntimeException("Failed loading class " + className + " from bundle " + bundleSymbolicName, e); - } - } - - private static Binder componentInjectorBinder(RestApiContext restApiContext) { - final ComponentGraphProvider componentGraphProvider = new ComponentGraphProvider(restApiContext.getInjectableComponents()); - final TypeLiteral<InjectionResolver<Component>> componentAnnotationType = new TypeLiteral<InjectionResolver<Component>>() { - }; - - return new AbstractBinder() { - @Override - public void configure() { - bind(componentGraphProvider).to(componentAnnotationType); - } - }; - } - - private static JacksonJaxbJsonProvider jacksonDatatypeJdk8Provider() { - JacksonJaxbJsonProvider provider = new JacksonJaxbJsonProvider(); - provider.setMapper(new ObjectMapper().registerModule(new Jdk8Module()).registerModule(new JavaTimeModule())); - return provider; - } - - @Override - public ServletHolder get() { - return jerseyServletHolder; - } - - @Override - public void deconstruct() { - } - -} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java deleted file mode 100644 index bf53cec670e..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey; - -import org.objectweb.asm.AnnotationVisitor; -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.Opcodes; -import org.objectweb.asm.Type; - -import javax.ws.rs.Path; -import javax.ws.rs.ext.Provider; - -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -public class ResourceOrProviderClassVisitor extends ClassVisitor { - private String className = null; - private boolean isPublic = false; - private boolean isAbstract = false; - - private boolean isInnerClass = false; - private boolean isStatic = false; - - private boolean isAnnotated = false; - - public ResourceOrProviderClassVisitor() { - super(Opcodes.ASM7); - } - - public Optional<String> getJerseyClassName() { - if (isJerseyClass()) { - return Optional.of(getClassName()); - } else { - return Optional.empty(); - } - } - - public boolean isJerseyClass() { - return isAnnotated && isPublic && !isAbstract && (!isInnerClass || isStatic); - } - - public String getClassName() { - assert (className != null); - return org.objectweb.asm.Type.getObjectType(className).getClassName(); - } - - @Override - public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { - isPublic = isPublic(access); - className = name; - isAbstract = isAbstract(access); - } - - @Override - public void visitInnerClass(String name, String outerName, String innerName, int access) { - assert (className != null); - - if (name.equals(className)) { - isInnerClass = true; - isStatic = isStatic(access); - } - } - - @Override - public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - isAnnotated |= annotationClassDescriptors.contains(desc); - return null; - } - - private static Set<String> annotationClassDescriptors = new HashSet<>(); - - static { - annotationClassDescriptors.add(Type.getDescriptor(Path.class)); - annotationClassDescriptors.add(Type.getDescriptor(Provider.class)); - } - - private static boolean isPublic(int access) { - return isSet(Opcodes.ACC_PUBLIC, access); - } - - private static boolean isStatic(int access) { - return isSet(Opcodes.ACC_STATIC, access); - } - - private static boolean isAbstract(int access) { - return isSet(Opcodes.ACC_ABSTRACT, access); - } - - private static boolean isSet(int bits, int access) { - return (access & bits) == bits; - } - - public static ResourceOrProviderClassVisitor visit(ClassReader classReader) { - ResourceOrProviderClassVisitor visitor = new ResourceOrProviderClassVisitor(); - classReader.accept(visitor, ClassReader.SKIP_DEBUG | ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES); - return visitor; - } -} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/util/ResourceConfigUtil.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/util/ResourceConfigUtil.java deleted file mode 100644 index fdfcf6565cd..00000000000 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/util/ResourceConfigUtil.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.util; - -import org.glassfish.jersey.server.ResourceConfig; - -/** - * @author Tony Vaagenes - */ -public class ResourceConfigUtil { - /** - * Solves ambiguous reference to overloaded definition, see - * http://stackoverflow.com/questions/3313929/how-do-i-disambiguate-in-scala-between-methods-with-vararg-and-without - */ - public static void registerComponent(ResourceConfig config, Object component) { - config.register(component); - } -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/AbstractResource.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/AbstractResource.java deleted file mode 100644 index dfa3354032a..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/AbstractResource.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("ignored") -public abstract class AbstractResource { - -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/DummyAnnotation.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/DummyAnnotation.java deleted file mode 100644 index ca6e20aba4d..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/DummyAnnotation.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * @author Tony Vaagenes - */ -@Retention(RetentionPolicy.RUNTIME) -public @interface DummyAnnotation {} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InnerClass.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InnerClass.java deleted file mode 100644 index 048ac4cdd9b..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InnerClass.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -public class InnerClass { - @Path("ignored") - public class Inner {} -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InterfaceResource.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InterfaceResource.java deleted file mode 100644 index d013028db19..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/InterfaceResource.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("ignored") -public interface InterfaceResource { - -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NestedClass.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NestedClass.java deleted file mode 100644 index 11ff6364074..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NestedClass.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -public class NestedClass { - @Path("ignored") - public static class Nested {} -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NonPublicNestedClass.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NonPublicNestedClass.java deleted file mode 100644 index 8e4499fe6dc..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/NonPublicNestedClass.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -public class NonPublicNestedClass { - @Path("ignored") - static class Nested {} -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Provider.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Provider.java deleted file mode 100644 index c2c605db22c..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Provider.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -/** - * @author Tony Vaagenes - */ -@javax.ws.rs.ext.Provider -public class Provider { -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Resource.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Resource.java deleted file mode 100644 index edfa2449fc3..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/Resource.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("ignored") -public class Resource { -} diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java deleted file mode 100644 index 1f5ccf4bb10..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceOrProviderClassVisitorTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import com.yahoo.container.servlet.jersey.ResourceOrProviderClassVisitor; -import org.junit.Test; -import org.objectweb.asm.ClassReader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class ResourceOrProviderClassVisitorTest { - @Test - public void resource_is_detected() throws Exception { - assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.Resource.class); - } - - @Test - public void provider_is_detected() throws Exception { - assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.Provider.class); - } - - @Test - public void inner_class_is_ignored() throws Exception { - assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.InnerClass.Inner.class); - } - - @Test - public void nested_public_class_is_detected() throws Exception { - assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.NestedClass.Nested.class); - } - - @Test - public void nested_non_public_class_is_ignored() throws Exception { - assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.NonPublicNestedClass.Nested.class); - } - - @Test - public void resource_with_multiple_annotations_is_detected() throws Exception { - assert_is_accepted(com.yahoo.container.servlet.jersey.classvisitor.ResourceWithMultipleAnnotations.class); - } - - @Test - public void interface_is_ignored() throws Exception { - assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.InterfaceResource.class); - } - - @Test - public void abstract_class_is_ignored() throws Exception { - assert_is_ignored(com.yahoo.container.servlet.jersey.classvisitor.AbstractResource.class); - } - - @Test - public void className_is_equal_to_getName() throws Exception { - assertEquals(com.yahoo.container.servlet.jersey.classvisitor.Resource.class.getName(), analyzeClass(com.yahoo.container.servlet.jersey.classvisitor.Resource.class).getClassName()); - } - - public void assert_is_accepted(Class<?> clazz) throws Exception { - assertTrue(className(clazz) + " was not accepted", - analyzeClass(clazz).isJerseyClass()); - } - - public void assert_is_ignored(Class<?> clazz) throws Exception { - assertFalse(className(clazz) + " was not ignored", - analyzeClass(clazz).isJerseyClass()); - } - - public ResourceOrProviderClassVisitor analyzeClass(Class<?> clazz) throws Exception { - return ResourceOrProviderClassVisitor.visit(new ClassReader(className(clazz))); - } - - public String className(Class<?> clazz) { - return clazz.getName(); - } -} - - diff --git a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceWithMultipleAnnotations.java b/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceWithMultipleAnnotations.java deleted file mode 100644 index 7047422736e..00000000000 --- a/container-jersey2/src/test/java/com/yahoo/container/servlet/jersey/classvisitor/ResourceWithMultipleAnnotations.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey.classvisitor; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("ignored") -@DummyAnnotation -public class ResourceWithMultipleAnnotations { - -} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 9b92a78a7c9..af731e3ade0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -186,8 +186,8 @@ public class Dispatcher extends AbstractComponent { if (nodes.isEmpty()) return Optional.empty(); query.trace(false, 2, "Dispatching with search path ", searchPath); - return invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.empty(), + return invokerFactory.createSearchInvoker(searcher, + query, nodes, true, maxHitsPerNode); @@ -203,7 +203,6 @@ public class Dispatcher extends AbstractComponent { query.trace(false, 2, "Dispatching to ", node); return invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.empty(), List.of(node), true, maxHitsPerNode) @@ -222,7 +221,6 @@ public class Dispatcher extends AbstractComponent { boolean acceptIncompleteCoverage = (i == max - 1); Optional<SearchInvoker> invoker = invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.of(group.id()), group.nodes(), acceptIncompleteCoverage, maxHitsPerNode); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index adf7368faa2..fb04c8299e9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -3,6 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; @@ -41,9 +42,9 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private final Set<SearchInvoker> invokers; private final SearchCluster searchCluster; + private final Group group; private final LinkedBlockingQueue<SearchInvoker> availableForProcessing; private final Set<Integer> alreadyFailedNodes; - private final boolean isContentWellBalanced; private Query query; private boolean adaptiveTimeoutCalculated = false; @@ -60,14 +61,17 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private boolean timedOut = false; private boolean degradedByMatchPhase = false; - public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, boolean isContentWellBalanced, SearchCluster searchCluster, Set<Integer> alreadyFailedNodes) { + public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, + SearchCluster searchCluster, + Group group, + Set<Integer> alreadyFailedNodes) { super(Optional.empty()); this.invokers = Collections.newSetFromMap(new IdentityHashMap<>()); this.invokers.addAll(invokers); this.searchCluster = searchCluster; + this.group = group; this.availableForProcessing = newQueue(); this.alreadyFailedNodes = alreadyFailedNodes; - this.isContentWellBalanced = isContentWellBalanced; } /** @@ -85,7 +89,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM int originalOffset = query.getOffset(); int neededHits = originalHits + originalOffset; int q = neededHits; - if (isContentWellBalanced) { + if (group.isBalanced() && !group.isSparse()) { Double topkProbabilityOverrride = query.properties().getDouble(Dispatcher.topKProbability); q = (topkProbabilityOverrride != null) ? searchCluster.estimateHitsToFetch(neededHits, invokers.size(), topkProbabilityOverrride) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 1de274ce6cf..e602afadcfb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; @@ -13,7 +14,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; /** @@ -39,8 +39,7 @@ public abstract class InvokerFactory { * * @param searcher the searcher processing the query * @param query the search query being processed - * @param groupId the id of the node group to which the nodes belong - * @param nodes pre-selected list of content nodes + * @param nodes pre-selected list of content nodes, all in a group or a subset of a group * @param acceptIncompleteCoverage if some of the nodes are unavailable and this parameter is * false, verify that the remaining set of nodes has sufficient coverage * @return the invoker or empty if some node in the @@ -48,10 +47,10 @@ public abstract class InvokerFactory { */ Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, Query query, - OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage, int maxHits) { + Group group = searchCluster.group(nodes.get(0).group()).get(); // Nodes must be of the same group List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { @@ -90,7 +89,7 @@ public abstract class InvokerFactory { if (invokers.size() == 1 && failed == null) { return Optional.of(invokers.get(0)); } else { - return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster.isGroupWellBalanced(groupId), searchCluster, failed)); + return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster, group, failed)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index f6480f80c01..b29c3297aea 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -41,7 +41,7 @@ public class SearchPath { if (sp.isPresent()) { return sp.get().mapToNodes(cluster); } else { - return Collections.emptyList(); + return List.of(); } } @@ -75,7 +75,7 @@ public class SearchPath { private List<Node> mapToNodes(SearchCluster cluster) { if (cluster.groups().isEmpty()) { - return Collections.emptyList(); + return List.of(); } Group selectedGroup = selectGroup(cluster); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 7faad9d51cc..727fb64faef 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -16,16 +16,17 @@ import java.util.logging.Logger; */ public class Group { + private static final Logger log = Logger.getLogger(Group.class.getName()); + private final static double maxContentSkew = 0.10; // If documents on a node is more than 10% off from the average the group is unbalanced + private final static int minDocsPerNodeToRequireLowSkew = 100; + private final int id; private final ImmutableList<Node> nodes; - private final AtomicBoolean hasSufficientCoverage = new AtomicBoolean(true); private final AtomicBoolean hasFullCoverage = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); - private final AtomicBoolean isContentWellBalanced = new AtomicBoolean(true); - private final static double MAX_UNBALANCE = 0.10; // If documents on a node is more than 10% off from the average the group is unbalanced - private static final Logger log = Logger.getLogger(Group.class.getName()); + private final AtomicBoolean isBalanced = new AtomicBoolean(true); public Group(int id, List<Node> nodes) { this.id = id; @@ -60,37 +61,43 @@ public class Group { return (int) nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).count(); } - void aggregateNodeValues() { + public void aggregateNodeValues() { long activeDocs = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getActiveDocuments).sum(); activeDocuments.set(activeDocs); isBlockingWrites.set(nodes.stream().anyMatch(Node::isBlockingWrites)); int numWorkingNodes = workingNodes(); if (numWorkingNodes > 0) { long average = activeDocs / numWorkingNodes; - long deviation = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(node -> Math.abs(node.getActiveDocuments() - average)).sum(); - boolean isDeviationSmall = deviation <= maxUnbalance(activeDocs); - if ((!isContentWellBalanced.get() || isDeviationSmall != isContentWellBalanced.get()) && (activeDocs > 0)) { - log.info("Content is " + (isDeviationSmall ? "" : "not ") + "well balanced. Current deviation = " + deviation*100/activeDocs + " %" + - ". activeDocs = " + activeDocs + ", deviation = " + deviation + ", average = " + average); - isContentWellBalanced.set(isDeviationSmall); + long skew = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(node -> Math.abs(node.getActiveDocuments() - average)).sum(); + boolean balanced = skew <= activeDocs * maxContentSkew; + if (!isBalanced.get() || balanced != isBalanced.get()) { + if (!isSparse()) + log.info("Content is " + (balanced ? "" : "not ") + "well balanced. Current deviation = " + + skew * 100 / activeDocs + " %. activeDocs = " + activeDocs + ", skew = " + skew + + ", average = " + average); + isBalanced.set(balanced); } } else { - isContentWellBalanced.set(true); + isBalanced.set(true); } } - double maxUnbalance(long activeDocs) { - return Math.max(1, activeDocs * MAX_UNBALANCE); - } - /** Returns the active documents on this group. If unknown, 0 is returned. */ - long getActiveDocuments() { return activeDocuments.get(); } + long activeDocuments() { return activeDocuments.get(); } /** Returns whether any node in this group is currently blocking write operations */ public boolean isBlockingWrites() { return isBlockingWrites.get(); } - public boolean isContentWellBalanced() { return isContentWellBalanced.get(); } - public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) { + /** Returns whether the nodes in the group have about the same number of documents */ + public boolean isBalanced() { return isBalanced.get(); } + + /** Returns whether this group has too few documents per node to expect it to be balanced */ + public boolean isSparse() { + if (nodes.isEmpty()) return false; + return activeDocuments.get() / nodes.size() < minDocsPerNodeToRequireLowSkew; + } + + public boolean fullCoverageStatusChanged(boolean hasFullCoverageNow) { boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow); return previousState != hasFullCoverageNow; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 9ae25518969..54d5dfc91af 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -14,13 +14,10 @@ import com.yahoo.search.cluster.NodeManager; import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; -import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -311,9 +308,9 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean sufficientCoverage = isGroupCoverageSufficient(group.getActiveDocuments(), - group.getActiveDocuments()); - trackGroupCoverageChanges(group, sufficientCoverage, group.getActiveDocuments()); + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), + group.activeDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); } private void pingIterationCompletedMultipleGroups() { @@ -321,7 +318,7 @@ public class SearchCluster implements NodeManager<Node> { long medianDocuments = medianDocumentsPerGroup(); boolean anyGroupsSufficientCoverage = false; for (Group group : orderedGroups()) { - boolean sufficientCoverage = isGroupCoverageSufficient(group.getActiveDocuments(), + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), medianDocuments); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); @@ -331,7 +328,7 @@ public class SearchCluster implements NodeManager<Node> { private long medianDocumentsPerGroup() { if (orderedGroups().isEmpty()) return 0; - var activeDocuments = orderedGroups().stream().map(Group::getActiveDocuments).collect(Collectors.toList()); + var activeDocuments = orderedGroups().stream().map(Group::activeDocuments).collect(Collectors.toList()); return (long)Quantiles.median().compute(activeDocuments); } @@ -357,12 +354,6 @@ public class SearchCluster implements NodeManager<Node> { return true; } - public boolean isGroupWellBalanced(OptionalInt groupId) { - if (groupId.isEmpty()) return false; - Group group = groups().get(groupId.getAsInt()); - return (group != null) && group.isContentWellBalanced(); - } - /** * Calculate whether a subset of nodes in a group has enough coverage */ @@ -375,12 +366,12 @@ public class SearchCluster implements NodeManager<Node> { private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. - boolean changed = group.isFullCoverageStatusChanged(fullCoverage); + boolean changed = group.fullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; if (fullCoverage) { log.info("Cluster " + clusterId + ": " + group + " has full coverage. " + - "Active documents: " + group.getActiveDocuments() + "/" + medianDocuments + ", " + + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size()); } else { StringBuilder unresponsive = new StringBuilder(); @@ -389,7 +380,7 @@ public class SearchCluster implements NodeManager<Node> { unresponsive.append('\n').append(node); } log.warning("Cluster " + clusterId + ": " + group + " has reduced coverage: " + - "Active documents: " + group.getActiveDocuments() + "/" + medianDocuments + ", " + + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive)); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index d636d3bc925..ba034271a4c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -1,20 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; - import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.collections.ListMap; -import com.yahoo.container.jdisc.ExtendedResponse; import com.yahoo.container.handler.Coverage; import com.yahoo.container.handler.Timing; +import com.yahoo.container.jdisc.ExtendedResponse; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.container.logging.HitCounts; import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.processing.execution.Execution.Trace.LogValue; @@ -25,6 +20,12 @@ import com.yahoo.search.Result; import com.yahoo.search.query.context.QueryContext; import com.yahoo.yolean.trace.TraceNode; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; + /** * Wrap the result of a query as an HTTP response. * @@ -35,20 +36,21 @@ public class HttpSearchResponse extends ExtendedResponse { private final Result result; private final Query query; private final Renderer<Result> rendererCopy; + private final Metric metric; private final Timing timing; private final HitCounts hitCounts; private final TraceNode trace; - public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) { - this(status, result, query, renderer, null); + public HttpSearchResponse(int status, Result result, Query query, Renderer<Result> renderer) { + this(status, result, query, renderer, null, null); } - HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) { + HttpSearchResponse(int status, Result result, Query query, Renderer<Result> renderer, TraceNode trace, Metric metric) { super(status); this.query = query; this.result = result; this.rendererCopy = renderer; - + this.metric = metric; this.timing = SearchResponse.createTiming(query, result); this.hitCounts = SearchResponse.createHitCounts(query, result); this.trace = trace; @@ -98,7 +100,11 @@ public class HttpSearchResponse extends ExtendedResponse { } try { try { - waitableRender(output); + long nanoStart = System.nanoTime(); + ListenableFuture<Boolean> promise = waitableRender(output); + if (metric != null) { + promise.addListener(new RendererLatencyReporter(nanoStart), Runnable::run); + } } finally { if (!(rendererCopy instanceof AsynchronousSectionedRenderer)) { output.flush(); @@ -173,9 +179,23 @@ public class HttpSearchResponse extends ExtendedResponse { @Override public Iterable<LogValue> getLogValues() { QueryContext context = query.getContext(false); - return context == null - ? Collections::emptyIterator - : context::logValueIterator; + return context == null ? Collections::emptyIterator : context::logValueIterator; + } + + private class RendererLatencyReporter implements Runnable { + + final long nanoStart; + + RendererLatencyReporter(long nanoStart) { this.nanoStart = nanoStart; } + + @Override + public void run() { + long latencyNanos = System.nanoTime() - nanoStart; + Metric.Context ctx = metric.createContext(Map.of( + SearchHandler.RENDERER_DIMENSION, rendererCopy.getClassName(), + SearchHandler.MIME_DIMENSION, rendererCopy.getMimeType())); + metric.set(SearchHandler.RENDER_LATENCY_METRIC, latencyNanos, ctx); + } } } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index e2c15b6e35b..9f67603f62b 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -11,6 +11,7 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpMethodAclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -21,8 +22,6 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; -import com.yahoo.container.jdisc.AclMapping; -import com.yahoo.container.jdisc.RequestView; import com.yahoo.language.Linguistics; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; @@ -88,6 +87,9 @@ public class SearchHandler extends LoggingRequestHandler { /** Event name for number of connections to the search subsystem */ private static final String SEARCH_CONNECTIONS = "search_connections"; + static final String RENDER_LATENCY_METRIC = "jdisc.render.latency"; + static final String MIME_DIMENSION = "mime"; + static final String RENDERER_DIMENSION = "renderer"; private static final String JSON_CONTENT_TYPE = "application/json"; @@ -230,6 +232,8 @@ public class SearchHandler extends LoggingRequestHandler { new ExecutionFactory(chainsConfig, indexInfo, clusters, searchers, specialtokens, linguistics, renderers)); } + Metric metric() { return metric; } + private static int examineExecutor(Executor executor) { if (executor instanceof ThreadPoolExecutor) { return ((ThreadPoolExecutor) executor).getMaximumPoolSize(); @@ -273,7 +277,7 @@ public class SearchHandler extends LoggingRequestHandler { private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) { Query query = new Query(); Result result = new Result(query, errorMessage); - Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); + Renderer<Result> renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } @@ -328,10 +332,11 @@ public class SearchHandler extends LoggingRequestHandler { } // Transform result to response - Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); + Renderer<Result> renderer = toRendererCopy(query.getPresentation().getRenderer()); HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer, - extractTraceNode(query)); + extractTraceNode(query), + metric); response.setRequestType(Request.RequestType.READ); hostResponseHeaderKey.ifPresent(key -> response.headers().add(key, selfHostname)); @@ -489,8 +494,8 @@ public class SearchHandler extends LoggingRequestHandler { if (maxThreads > 3) { // cast to long to avoid overflows if maxThreads is at no // log value (maxint) - final long maxThreadsAsLong = maxThreads; - final long connectionsAsLong = connections; + long maxThreadsAsLong = maxThreads; + long connectionsAsLong = connections; // only log when exactly crossing the limit to avoid // spamming the log if (connectionsAsLong < maxThreadsAsLong * 9L / 10L) { @@ -514,7 +519,7 @@ public class SearchHandler extends LoggingRequestHandler { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + - " [" + request + "], received exception with no context", e); + " [" + request + "], received exception with no context", e); } else { log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } @@ -649,6 +654,7 @@ public class SearchHandler extends LoggingRequestHandler { .override(com.yahoo.jdisc.http.HttpRequest.Method.POST, AclMapping.Action.READ) .build(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index 1ba30275dc1..4fb9e1e9afa 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -38,10 +38,10 @@ public class QueryContext implements Cloneable { owner.getModel().getExecution().trace().trace(message,traceLevel); } /** - * Adds a key-value which will be logged to the access log for this query (by doing toString() on the value + * Adds a key-value which will be logged to the access log for this query (by doing toString() on the value). * Multiple values may be set to the same key. A value cannot be removed once set. */ - public void logValue(String key,Object value) { + public void logValue(String key, Object value) { owner.getModel().getExecution().trace().logValue(key, value.toString()); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 943390cb10c..be761acf2c2 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -157,7 +157,6 @@ public class DispatcherTest { @Override public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, Query query, - OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage, int maxHitsPerNode) { @@ -167,7 +166,7 @@ public class DispatcherTest { boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage); step++; if (nonEmpty) { - return Optional.of(new MockInvoker(nodes.get(0).key(), groupId)); + return Optional.of(new MockInvoker(nodes.get(0).key())); } else { return Optional.empty(); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 730aa0800e7..21a15165ab3 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; +import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.DefaultErrorHit; @@ -53,7 +55,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatAdaptiveTimeoutsAreNotUsedWithFullCoverageRequirement() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3); - SearchInvoker invoker = createInterleavedInvoker(cluster, 3); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3); expectedEvents.add(new Event(5000, 100, 0)); expectedEvents.add(new Event(4900, 100, 1)); @@ -67,7 +69,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatTimeoutsAreNotMarkedAsAdaptive() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3); - SearchInvoker invoker = createInterleavedInvoker(cluster, 3); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3); expectedEvents.add(new Event(5000, 300, 0)); expectedEvents.add(new Event(4700, 300, 1)); @@ -85,7 +87,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatAdaptiveTimeoutDecreasesTimeoutWhenCoverageIsReached() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(50.0), 1, 4); - SearchInvoker invoker = createInterleavedInvoker(cluster, 4); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 4); expectedEvents.add(new Event(5000, 100, 0)); expectedEvents.add(new Event(4900, 100, 1)); @@ -106,7 +108,7 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); - SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); @@ -127,7 +129,7 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(10101, 50155, 50155, 1, 1, DEGRADED_BY_MATCH_PHASE))); invokers.add(new MockInvoker(1, createCoverage(13319, 49845, 49845, 1, 1, DEGRADED_BY_MATCH_PHASE))); - SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); @@ -149,7 +151,7 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(5000, 50155, 50155, 1, 1, DEGRADED_BY_TIMEOUT))); invokers.add(new MockInvoker(1, createCoverage(4900, 49845, 49845, 1, 1, DEGRADED_BY_TIMEOUT))); - SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()),0); expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); @@ -171,7 +173,7 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); - SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(null); @@ -205,8 +207,8 @@ public class InterleavedSearchInvokerTest { private static final List<Double> A5Aux = Arrays.asList(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0); private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); - private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK, boolean isContentWellBalanced) throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, isContentWellBalanced); + private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK, Group group) throws IOException { + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, group); query.setHits(8); query.properties().set(Dispatcher.topKProbability, topKProbability); SearchInvoker [] invokers = invoker.invokers().toArray(new SearchInvoker[0]); @@ -228,17 +230,37 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatTopKProbabilityOverrideTakesEffect() throws IOException { - validateThatTopKProbabilityOverrideTakesEffect(null, 8, true); - validateThatTopKProbabilityOverrideTakesEffect(0.8, 7, true); + validateThatTopKProbabilityOverrideTakesEffect(null, 8, new Group(0, List.of())); + validateThatTopKProbabilityOverrideTakesEffect(0.8, 7, new Group(0, List.of())); } + @Test public void requireThatTopKProbabilityOverrideIsDisabledOnContentSkew() throws IOException { - validateThatTopKProbabilityOverrideTakesEffect(0.8, 8, false); + Node node0 = new Node(0, "host0", 0); + Node node1 = new Node(1, "host1", 0); + Group group = new Group(0, List.of(node0, node1)); + + node0.setActiveDocuments(1000000); + node1.setActiveDocuments(1100000); + group.aggregateNodeValues(); + validateThatTopKProbabilityOverrideTakesEffect(0.8, 8, group); + } + + @Test + public void requireThatTopKProbabilityOverrideIsDisabledOnLittleContent() throws IOException { + Node node0 = new Node(0, "host0", 0); + Node node1 = new Node(1, "host1", 0); + Group group = new Group(0, List.of(node0, node1)); + + node0.setActiveDocuments(10); + node1.setActiveDocuments(10); + group.aggregateNodeValues(); + validateThatTopKProbabilityOverrideTakesEffect(0.8, 8, group); } @Test public void requireThatMergeOfConcreteHitsObeySorting() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, true); + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of())); query.setHits(12); Result result = invoker.search(query, null); assertEquals(10, result.hits().size()); @@ -247,7 +269,7 @@ public class InterleavedSearchInvokerTest { assertEquals(0, result.getQuery().getOffset()); assertEquals(12, result.getQuery().getHits()); - invoker = createInterLeavedTestInvoker(B5, A5, true); + invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of())); result = invoker.search(query, null); assertEquals(10, result.hits().size()); assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); @@ -258,7 +280,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatMergeOfConcreteHitsObeyOffset() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, true); + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of())); query.setHits(3); query.setOffset(5); Result result = invoker.search(query, null); @@ -268,7 +290,7 @@ public class InterleavedSearchInvokerTest { assertEquals(0, result.getQuery().getOffset()); assertEquals(3, result.getQuery().getHits()); - invoker = createInterLeavedTestInvoker(B5, A5, true); + invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of())); query.setOffset(5); result = invoker.search(query, null); assertEquals(3, result.hits().size()); @@ -280,7 +302,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatMergeOfConcreteHitsObeyOffsetWithAuxilliaryStuff() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux, true); + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux, new Group(0, List.of())); query.setHits(3); query.setOffset(5); Result result = invoker.search(query, null); @@ -291,7 +313,7 @@ public class InterleavedSearchInvokerTest { assertEquals(0, result.getQuery().getOffset()); assertEquals(3, result.getQuery().getHits()); - invoker = createInterLeavedTestInvoker(B5Aux, A5Aux, true); + invoker = createInterLeavedTestInvoker(B5Aux, A5Aux, new Group(0, List.of())); query.setOffset(5); result = invoker.search(query, null); assertEquals(7, result.hits().size()); @@ -302,13 +324,12 @@ public class InterleavedSearchInvokerTest { assertEquals(3, result.getQuery().getHits()); } - private static InterleavedSearchInvoker createInterLeavedTestInvoker(List<Double> a, List<Double> b, - boolean isContentWellBalanced) { + private static InterleavedSearchInvoker createInterLeavedTestInvoker(List<Double> a, List<Double> b, Group group) { SearchCluster cluster = new MockSearchCluster("!", 1, 2); List<SearchInvoker> invokers = new ArrayList<>(); invokers.add(createInvoker(a, 0)); invokers.add(createInvoker(b, 1)); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, isContentWellBalanced, cluster, Collections.emptySet()); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, group, Collections.emptySet()); invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); return invoker; @@ -336,7 +357,7 @@ public class InterleavedSearchInvokerTest { Coverage errorCoverage = new Coverage(0, 0, 0); errorCoverage.setNodesTried(1); invokers.add(new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError("node is down"), errorCoverage)); - SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); expectedEvents.add(new Event(null, 1, 1)); expectedEvents.add(new Event(null, 100, 0)); @@ -354,12 +375,13 @@ public class InterleavedSearchInvokerTest { assertThat(cov.isDegradedByTimeout(), is(true)); } - private InterleavedSearchInvoker createInterleavedInvoker(SearchCluster searchCluster, int numInvokers) { + private InterleavedSearchInvoker createInterleavedInvoker(SearchCluster searchCluster, Group group, int numInvokers) { for (int i = 0; i < numInvokers; i++) { invokers.add(new MockInvoker(i)); } - return new InterleavedSearchInvoker(invokers, false, searchCluster, null) { + return new InterleavedSearchInvoker(invokers, searchCluster, group,null) { + @Override protected long currentTime() { return clock.millis(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index 53d1a2457d0..d86fcdfc25d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.Hit; @@ -17,27 +18,17 @@ import java.util.OptionalInt; class MockInvoker extends SearchInvoker { private final Coverage coverage; - private final OptionalInt groupId; private Query query; private List<Hit> hits; int hitsRequested; - protected MockInvoker(int key, Coverage coverage, OptionalInt groupId) { + protected MockInvoker(int key, Coverage coverage) { super(Optional.of(new Node(key, "?", 0))); this.coverage = coverage; - this.groupId = groupId; - } - - protected MockInvoker(int key, OptionalInt groupId) { - this(key, null, groupId); - } - - protected MockInvoker(int key, Coverage coverage) { - this(key, coverage, OptionalInt.empty()); } protected MockInvoker(int key) { - this(key, null, OptionalInt.empty()); + this(key, null); } MockInvoker setHits(List<Hit> hits) { @@ -45,18 +36,15 @@ class MockInvoker extends SearchInvoker { return this; } - /** Returns the group to be invoked, if known */ - public OptionalInt groupId() { return groupId; } - @Override - protected Object sendSearchRequest(Query query, Object context) throws IOException { + protected Object sendSearchRequest(Query query, Object context) { this.query = query; hitsRequested = query.getHits(); return context; } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult(Execution execution) { InvokerResult ret = new InvokerResult(query, 10); if (coverage != null) { ret.getResult().setCoverage(coverage); @@ -80,8 +68,7 @@ class MockInvoker extends SearchInvoker { @Override public String toString() { - return "invoker with key " + distributionKey() + - (groupId().isPresent() ? " of group " + groupId().getAsInt() : ""); + return "invoker with key " + distributionKey(); } }
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java index c9f7469acbb..8101aee74fd 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java @@ -101,7 +101,7 @@ public class SearchClusterCoverageTest { } @Test - public void one_group_few_docs_has_well_balanced_content() { + public void one_group_few_docs_unbalanced() { var tester = new SearchClusterTester(1, 2); Node node0 = tester.group(0).nodes().get(0); @@ -115,7 +115,27 @@ public class SearchClusterCoverageTest { node1.setActiveDocuments(0); tester.pingIterationCompleted(); - assertTrue(tester.group(0).isContentWellBalanced()); + assertFalse(tester.group(0).isBalanced()); + assertTrue(tester.group(0).isSparse()); + } + + @Test + public void one_group_many_docs_unbalanced() { + var tester = new SearchClusterTester(1, 2); + + Node node0 = tester.group(0).nodes().get(0); + Node node1 = tester.group(0).nodes().get(1); + + // 1 document + node0.setWorking(true); + node1.setWorking(true); + + node0.setActiveDocuments(1000000); + node1.setActiveDocuments(100000); + + tester.pingIterationCompleted(); + assertFalse(tester.group(0).isBalanced()); + assertFalse(tester.group(0).isSparse()); } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 48134094faf..f46717ce180 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -337,45 +337,45 @@ public class SearchClusterTest { @Test public void requireThatEmptyGroupIsInBalance() { Group group = new Group(0, new ArrayList<>()); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.aggregateNodeValues(); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); } @Test public void requireThatSingleNodeGroupIsInBalance() { Group group = new Group(0, Arrays.asList(new Node(1, "n", 1))); group.nodes().forEach(node -> node.setWorking(true)); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.aggregateNodeValues(); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.nodes().get(0).setActiveDocuments(1000); group.aggregateNodeValues(); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); } @Test public void requireThatMultiNodeGroupDetectsBalance() { Group group = new Group(0, Arrays.asList(new Node(1, "n1", 1), new Node(2, "n2", 1))); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.nodes().forEach(node -> node.setWorking(true)); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.aggregateNodeValues(); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); group.nodes().get(0).setActiveDocuments(1000); group.aggregateNodeValues(); - assertFalse(group.isContentWellBalanced()); + assertFalse(group.isBalanced()); group.nodes().get(1).setActiveDocuments(100); group.aggregateNodeValues(); - assertFalse(group.isContentWellBalanced()); + assertFalse(group.isBalanced()); group.nodes().get(1).setActiveDocuments(800); group.aggregateNodeValues(); - assertFalse(group.isContentWellBalanced()); + assertFalse(group.isBalanced()); group.nodes().get(1).setActiveDocuments(818); group.aggregateNodeValues(); - assertFalse(group.isContentWellBalanced()); + assertFalse(group.isBalanced()); group.nodes().get(1).setActiveDocuments(819); group.aggregateNodeValues(); - assertTrue(group.isContentWellBalanced()); + assertTrue(group.isBalanced()); } } diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java index 2eb5901b786..2b584c7b285 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.handler.test; +package com.yahoo.search.handler; import com.yahoo.container.Container; import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; @@ -10,12 +10,11 @@ import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.net.HostName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.handler.HttpSearchResponse; -import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; @@ -45,7 +44,7 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth */ -public class SearchHandlerTestCase { +public class SearchHandlerTest { private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; private static final String myHostnameHeader = "my-hostname-header"; @@ -59,6 +58,7 @@ public class SearchHandlerTestCase { private RequestHandlerTestDriver driver = null; private HandlersConfigurerTestWrapper configurer = null; + private MockMetric metric; private SearchHandler searchHandler; @Before @@ -72,6 +72,7 @@ public class SearchHandlerTestCase { configurer = new HandlersConfigurerTestWrapper(new Container(), configId); searchHandler = (SearchHandler)configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + metric = (MockMetric) searchHandler.metric(); driver = new RequestHandlerTestDriver(searchHandler); } @@ -289,6 +290,7 @@ public class SearchHandlerTestCase { assertEquals(expected, response.readAll()); assertEquals(200, response.getStatus()); assertEquals(selfHostname, response.getResponse().headers().get(myHostnameHeader).get(0)); + assertTrue(metric.metrics().containsKey(SearchHandler.RENDER_LATENCY_METRIC)); } @Test @@ -310,7 +312,7 @@ public class SearchHandlerTestCase { } private void assertHandlerResponse(int status, String responseData, String handlerName) throws Exception { - RequestHandler forwardingHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.test.SearchHandlerTestCase$" + handlerName + "Handler"); + RequestHandler forwardingHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.SearchHandlerTest$" + handlerName + "Handler"); try (RequestHandlerTestDriver forwardingDriver = new RequestHandlerTestDriver(forwardingHandler)) { RequestHandlerTestDriver.MockResponseHandler response = forwardingDriver.sendRequest("http://localhost/" + handlerName + "?query=test"); response.awaitResponse(); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg index 9a16c6ed1e7..f7eba221ef1 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg @@ -1,20 +1,20 @@ chains[4] chains[0].id default chains[0].components[1] -chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[0].components[0] com.yahoo.search.handler.SearchHandlerTest$TestSearcher chains[1].id classLoadingError chains[1].components[1] -chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher +chains[1].components[0] com.yahoo.search.handler.SearchHandlerTest$ClassLoadingErrorSearcher chains[2].id exceptionInPlugin chains[2].components[1] -chains[2].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher +chains[2].components[0] com.yahoo.search.handler.SearchHandlerTest$ExceptionInPluginSearcher chains[3].id echoingQuery chains[3].components[2] chains[3].components[0] com.yahoo.search.yql.MinimalQueryInserter -chains[3].components[1] com.yahoo.search.handler.test.SearchHandlerTestCase$EchoingQuerySearcher +chains[3].components[1] com.yahoo.search.handler.SearchHandlerTest$EchoingQuerySearcher components[5] -components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher -components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher -components[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher -components[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$EchoingQuerySearcher +components[0].id com.yahoo.search.handler.SearchHandlerTest$TestSearcher +components[1].id com.yahoo.search.handler.SearchHandlerTest$ClassLoadingErrorSearcher +components[2].id com.yahoo.search.handler.SearchHandlerTest$ExceptionInPluginSearcher +components[3].id com.yahoo.search.handler.SearchHandlerTest$EchoingQuerySearcher components[4].id com.yahoo.search.yql.MinimalQueryInserter diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg index 915da8dc037..12f218581d5 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -1,9 +1,9 @@ handler[8] handler[0].id com.yahoo.search.handler.SearchHandler -handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler -handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler -handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandler -handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler -handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler -handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler +handler[1].id com.yahoo.search.handler.SearchHandlerTest$NullReturningHandler +handler[2].id com.yahoo.search.handler.SearchHandlerTest$NullReturningAsyncHandler +handler[3].id com.yahoo.search.handler.SearchHandlerTest$ThrowingHandler +handler[4].id com.yahoo.search.handler.SearchHandlerTest$ThrowingAsyncHandler +handler[5].id com.yahoo.search.handler.SearchHandlerTest$ForwardingHandler +handler[6].id com.yahoo.search.handler.SearchHandlerTest$ForwardingAsyncHandler handler[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg index 2437efdec4f..83db7ef1cc9 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg @@ -1,10 +1,10 @@ chains[2] chains[0].id default chains[0].components[1] -chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[0].components[0] com.yahoo.search.handler.SearchHandlerTest$TestSearcher chains[1].id hello chains[1].components[1] -chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher +chains[1].components[0] com.yahoo.search.handler.SearchHandlerTest$HelloWorldSearcher components[2] -components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher -components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher +components[0].id com.yahoo.search.handler.SearchHandlerTest$TestSearcher +components[1].id com.yahoo.search.handler.SearchHandlerTest$HelloWorldSearcher diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg index 691b37b4955..9dd1aff9d06 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg @@ -1,3 +1,3 @@ handler[2] handler[0].id com.yahoo.search.handler.SearchHandler -handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ErrorOnInitializationHandler +handler[1].id com.yahoo.search.handler.SearchHandlerTest$ErrorOnInitializationHandler diff --git a/container-test-jars/OWNERS b/container-test-jars/OWNERS deleted file mode 100644 index 3b2ba1ede81..00000000000 --- a/container-test-jars/OWNERS +++ /dev/null @@ -1 +0,0 @@ -gjoranv diff --git a/container-test-jars/README.md b/container-test-jars/README.md deleted file mode 100644 index f4f4481efc1..00000000000 --- a/container-test-jars/README.md +++ /dev/null @@ -1,4 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Container test bundles - -Used by the 'application' module. diff --git a/container-test-jars/bundle-with-provided-bundle/pom.xml b/container-test-jars/bundle-with-provided-bundle/pom.xml deleted file mode 100644 index d9d97ddd2b0..00000000000 --- a/container-test-jars/bundle-with-provided-bundle/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>container-test-jars</artifactId> - <version>7-SNAPSHOT</version> - </parent> - - <artifactId>bundle-with-provided-bundle</artifactId> - <packaging>container-plugin</packaging> - <description> - This will trigger bugs in our bundle plugins that assumes that artifact.getFile is a jar file. - Normally, artifact.getFile points to a jar file, - but in multi-module projects when running phases not including package, - this will be a directory. - </description> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>jersey-resources</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa_jersey2</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - <type>pom</type> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${project.version}</version> - <extensions>true</extensions> - </plugin> - </plugins> - </build> -</project> diff --git a/container-test-jars/jersey-resources/pom.xml b/container-test-jars/jersey-resources/pom.xml deleted file mode 100644 index cb35be91cfd..00000000000 --- a/container-test-jars/jersey-resources/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>container-test-jars</artifactId> - <version>7-SNAPSHOT</version> - </parent> - - <artifactId>jersey-resources</artifactId> - <packaging>container-plugin</packaging> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa_jersey2</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - <type>pom</type> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <!-- Explicit for IntelliJ to detect correct language level from parent --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${project.version}</version> - <extensions>true</extensions> - </plugin> - </plugins> - </build> -</project> diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java deleted file mode 100644 index 59095d05567..00000000000 --- a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -@Path("bundle-plugin-test/test-resource") -public class TestResource extends TestResourceBase { -} diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java deleted file mode 100644 index c3724723252..00000000000 --- a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources; - -import javax.ws.rs.core.MediaType; -import javax.ws.rs.Produces; -import javax.ws.rs.GET; - -/** - * @author Tony Vaagenes - * @author ollivir - */ -public class TestResourceBase { - @GET - @Produces({MediaType.TEXT_PLAIN}) - public String get() { - return content(getClass()); - } - - public static String content(Class<? extends TestResourceBase> clazz) { - return "Response from " + clazz.getName(); - } -} diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java deleted file mode 100644 index ab1c1f8f229..00000000000 --- a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage1; - -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("bundle-plugin-test/nested-test-resource1") -public class NestedTestResource1 extends TestResourceBase { -} diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java deleted file mode 100644 index 0dfc9e1938b..00000000000 --- a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage2; - -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; - -import javax.ws.rs.Path; - -/** - * @author Tony Vaagenes - */ -@Path("bundle-plugin-test/nested-test-resource2") -public class NestedTestResource2 extends TestResourceBase { -} diff --git a/container-test-jars/pom.xml b/container-test-jars/pom.xml deleted file mode 100644 index 7176af3791c..00000000000 --- a/container-test-jars/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <groupId>com.yahoo.vespa.container-test-jars</groupId> - <artifactId>container-test-jars</artifactId> - <version>7-SNAPSHOT</version> - <packaging>pom</packaging> - <modules> - <module>jersey-resources</module> - <module>bundle-with-provided-bundle</module> - </modules> - - <properties> - <!-- This project only builds test artifacts --> - <maven.deploy.skip>true</maven.deploy.skip> - </properties> - -</project> diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 2f01f45edaa..02a7028b8ca 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -27,6 +27,13 @@ <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>serviceview</artifactId> <scope>provided</scope> <version>${project.version}</version> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java index a74a362330b..0f05fa60db7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java @@ -15,15 +15,15 @@ import java.util.Optional; */ public class MockResourceTagger implements ResourceTagger { - Map<ZoneId, Map<HostName, Optional<ApplicationId>>> values = new HashMap<>(); + Map<ZoneId, Map<HostName, ApplicationId>> values = new HashMap<>(); @Override - public int tagResources(ZoneApi zone, Map<HostName, Optional<ApplicationId>> ownerOfHosts) { + public int tagResources(ZoneApi zone, Map<HostName, ApplicationId> ownerOfHosts) { values.put(zone.getId(), ownerOfHosts); return 0; } - public Map<ZoneId, Map<HostName, Optional<ApplicationId>>> getValues() { + public Map<ZoneId, Map<HostName, ApplicationId>> getValues() { return values; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java index 61f8a57ac8b..988949456bb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java @@ -16,7 +16,7 @@ public interface ResourceTagger { /** * Returns number of tagged resources */ - int tagResources(ZoneApi zone, Map<HostName, Optional<ApplicationId>> ownerOfHosts); + int tagResources(ZoneApi zone, Map<HostName, ApplicationId> ownerOfHosts); static ResourceTagger empty() { return (zone, tenantOfHosts) -> 0; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 81c4d7be483..8d9f20a7cee 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -55,7 +55,7 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath); - String getClusterControllerStatus(DeploymentId deployment, String node, String subPath); + String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath); /** * Gets the Vespa logs of the given deployment. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java index 1d23cd52d23..01045eec020 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.time.Instant; import java.util.Map; @@ -20,10 +21,10 @@ public class ApplicationSummary { private final Optional<Instant> lastQueried; private final Optional<Instant> lastWritten; private final Optional<Instant> lastBuilt; - private final Map<ZoneId, Metric> metrics; + private final Map<DeploymentId, Metric> metrics; public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, - Optional<Instant> lastBuilt, Map<ZoneId, Metric> metrics) { + Optional<Instant> lastBuilt, Map<DeploymentId, Metric> metrics) { this.application = Objects.requireNonNull(application); this.lastQueried = Objects.requireNonNull(lastQueried); this.lastWritten = Objects.requireNonNull(lastWritten); @@ -47,7 +48,7 @@ public class ApplicationSummary { return lastBuilt; } - public Map<ZoneId, Metric> metrics() { + public Map<DeploymentId, Metric> metrics() { return metrics; } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 0c05f7d70bb..a0d05588a0d 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -20,6 +20,13 @@ <!-- provided --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>controller-api</artifactId> <version>${project.version}</version> @@ -128,16 +135,6 @@ </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> <groupId>com.auth0</groupId> <artifactId>java-jwt</artifactId> <exclusions> @@ -167,6 +164,14 @@ <scope>compile</scope> </dependency> + <dependency> + <!-- required by java-jwt --> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>${commons.codec.version}</version> + <scope>compile</scope> + </dependency> + <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a3a03eb942d..4e6df1921b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -202,7 +202,7 @@ public class DeploymentTrigger { return List.copyOf(jobs.keySet()); } - /** retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ + /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment) { JobType jobType = JobType.from(controller.system(), deployment.zoneId()) .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index a1566a1b5a1..c49e3c88df3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -66,7 +66,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -89,12 +88,15 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.logging.Level.INFO; @@ -188,11 +190,20 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { + Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate(); return deploy(() -> controller.applications().deploy(id.job(), setTheStage), controller.jobController().run(id).get() .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), - logger); + logger) + .filter(result -> { + // If no tester cert, or deployment failed, propagate original result. + if ( ! useTesterCertificate(id) || result != running) + return true; + // If tester cert, ensure real is deployed with the tester cert whose key was successfully deployed. + return controller.jobController().run(id).get().stepStatus(deployTester).get() == succeeded + && testerCertificate.equals(controller.jobController().run(id).get().testerCertificate()); + }); } private Optional<RunStatus> deployTester(RunId id, DualLogger logger) { @@ -249,7 +260,7 @@ public class InternalStepRunner implements StepRunner { case OUT_OF_CAPACITY: logger.log(e.message()); return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant()) - ? Optional.empty() + ? result : Optional.of(outOfCapacity); case INVALID_APPLICATION_PACKAGE: case BAD_REQUEST: @@ -636,6 +647,19 @@ public class InternalStepRunner implements StepRunner { try { controller.jobController().updateVespaLog(id); } + // Hitting a config server which doesn't have this particular app loaded causes a 404. + catch (ConfigServerException e) { + Instant doom = controller.jobController().run(id).get().stepInfo(copyVespaLogs).get().startTime().get() + .plus(Duration.ofMinutes(3)); + if (e.code() == ConfigServerException.ErrorCode.NOT_FOUND && controller.clock().instant().isBefore(doom)) { + logger.log(INFO, "Found no logs, but will retry"); + return Optional.empty(); + } + else { + logger.log(INFO, "Failure getting vespa logs for " + id, e); + return Optional.of(error); + } + } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); return Optional.of(error); @@ -685,6 +709,12 @@ public class InternalStepRunner implements StepRunner { logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e); return Optional.of(error); } + catch (RuntimeException e) { + Instant start = controller.jobController().run(id).get().stepInfo(report).get().startTime().get(); + return (controller.clock().instant().isAfter(start.plusSeconds(180))) + ? Optional.empty() + : Optional.of(error); + } return Optional.of(running); } @@ -798,16 +828,20 @@ public class InternalStepRunner implements StepRunner { return deployment.at().isBefore(controller.clock().instant().minus(timeout.minus(Duration.ofMinutes(1)))); } + private boolean useTesterCertificate(RunId id) { + return controller.system().isPublic() && id.type().environment().isTest(); + } + /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ private ApplicationPackage testerPackage(RunId id) { ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); ZoneId zone = id.type().zone(controller.system()); - boolean useTesterCertificate = controller.system().isPublic() && id.type().environment().isTest(); + boolean useTesterCertificate = useTesterCertificate(id); boolean useOsgiBasedTestRuntime = testerPlatformVersion(id).isAfter(new Version(7, 247, 11)); - byte[] servicesXml = servicesXml(! controller.system().isPublic(), + byte[] servicesXml = servicesXml( ! controller.system().isPublic(), useTesterCertificate, useOsgiBasedTestRuntime, testerResourcesFor(zone, spec.requireInstance(id.application().instance())), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 25bc21a0076..b622fc0bd75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -75,9 +76,9 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public class JobController { - public static final int historyLength = 64; public static final Duration maxHistoryAge = Duration.ofDays(60); + private final int historyLength; private final Controller controller; private final CuratorDb curator; private final BufferedLogStore logs; @@ -87,6 +88,7 @@ public class JobController { private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); public JobController(Controller controller) { + this.historyLength = controller.system().isCd() ? 256 : 64; this.controller = controller; this.curator = controller.curator(); this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); @@ -230,6 +232,14 @@ public class JobController { return runs(id.application(), id.type()); } + /** Lists the start time of non-redeployment runs of the given job, in order of increasing age. */ + public List<Instant> jobStarts(JobId id) { + return runs(id).descendingMap().values().stream() + .filter(run -> ! run.isRedeployment()) + .map(Run::start) + .collect(toUnmodifiableList()); + } + /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); @@ -359,7 +369,7 @@ public class JobController { List<Lock> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. - for (Step step : report.allPrerequisites()) + for (Step step : report.allPrerequisites(run(id).get().steps().keySet())) locks.add(curator.lock(id.application(), id.type(), step)); locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. @@ -440,18 +450,23 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { - start(id, type, versions, JobProfile.of(type)); + start(id, type, versions, false); + } + + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment) { + start(id, type, versions, isRedeployment, JobProfile.of(type)); } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) { + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile) { locked(id, type, __ -> { Optional<Run> last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile)); + curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile)); metric.jobStarted(newId.job()); }); } @@ -477,6 +492,7 @@ public class JobController { ApplicationVersion.unknown, Optional.empty(), Optional.empty()), + false, JobProfile.development); }); @@ -573,7 +589,7 @@ public class JobController { /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { try (Lock lock = curator.lock(id, type, step)) { - for (Step prerequisite : step.allPrerequisites()) // Check that no prerequisite is still running. + for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. try (Lock __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index d2481cd97ad..d93a0133c97 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -30,6 +30,7 @@ public class Run { private final RunId id; private final Map<Step, StepInfo> steps; private final Versions versions; + private final boolean isRedeployment; private final Instant start; private final Optional<Instant> end; private final RunStatus status; @@ -40,12 +41,13 @@ public class Run { private final Optional<X509Certificate> testerCertificate; // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end, + public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; + this.isRedeployment = isRedeployment; this.start = start; this.end = end; this.status = status; @@ -56,10 +58,10 @@ public class Run { this.testerCertificate = testerCertificate; } - public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) { + public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -73,7 +75,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, + return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } @@ -88,49 +90,49 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(startTime)); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run finished(Instant now) { requireActive(); - return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status, + return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty()); } public Run aborted() { requireActive(); - return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(long lastTestRecord) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.of(testerCertificate)); } @@ -222,6 +224,11 @@ public class Run { return testerCertificate; } + /** Whether this is a automatic redeployment. */ + public boolean isRedeployment() { + return isRedeployment; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index baba4771370..ce34a021218 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -1,10 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -34,7 +36,7 @@ public enum Step { installTester(false, deployTester), /** Download and deploy the initial real application, for staging tests. */ - deployInitialReal(false, deployTester), + deployInitialReal(false), /** See that the real application has had its nodes converge to the initial state. */ installInitialReal(false, deployInitialReal), @@ -46,7 +48,7 @@ public enum Step { endStagingSetup(false, startStagingSetup), /** Download and deploy real application, restarting services if required. */ - deployReal(false, endStagingSetup, deployTester), + deployReal(false, endStagingSetup), /** See that real application has had its nodes converge to the wanted version and generation. */ installReal(false, deployReal), @@ -72,24 +74,24 @@ public enum Step { private final boolean alwaysRun; private final List<Step> prerequisites; - private final List<Step> allPrerequisites; Step(boolean alwaysRun, Step... prerequisites) { this.alwaysRun = alwaysRun; this.prerequisites = List.of(prerequisites); - this.allPrerequisites = Stream.concat(Stream.of(prerequisites), - Stream.of(prerequisites).flatMap(pre -> pre.allPrerequisites().stream())) - .sorted() - .distinct() - .collect(toUnmodifiableList()); } /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */ public boolean alwaysRun() { return alwaysRun; } - /** Returns all prerequisite steps for this, recursively. */ - public List<Step> allPrerequisites() { - return allPrerequisites; + /** Returns all prerequisite steps for this, including transient ones, in a job profile containing the given steps. */ + public List<Step> allPrerequisites(Collection<Step> among) { + return prerequisites.stream() + .filter(among::contains) + .flatMap(pre -> Stream.concat(Stream.of(pre), + pre.allPrerequisites(among).stream())) + .sorted() + .distinct() + .collect(toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 69e0eb26f16..de087737320 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; @@ -17,7 +18,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.HashMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -78,14 +78,15 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { private ApplicationSummary summaryOf(TenantAndApplicationId application) { var app = applications.requireApplication(application); - var metrics = new HashMap<ZoneId, ApplicationSummary.Metric>(); + var metrics = new HashMap<DeploymentId, ApplicationSummary.Metric>(); for (Instance instance : app.instances().values()) { for (var kv : instance.deployments().entrySet()) { var zone = kv.getKey(); var deploymentMetrics = kv.getValue().metrics(); - metrics.put(zone, new ApplicationSummary.Metric(deploymentMetrics.documentCount(), - deploymentMetrics.queriesPerSecond(), - deploymentMetrics.writesPerSecond())); + metrics.put(new DeploymentId(instance.id(), zone), + new ApplicationSummary.Metric(deploymentMetrics.documentCount(), + deploymentMetrics.queriesPerSecond(), + deploymentMetrics.writesPerSecond())); } } return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), @@ -112,20 +113,29 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } private double updateConfirmedApplicationOwners() { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); applications() - .withProjectId() - .withProductionDeployment() - .asList() - .stream() - .filter(application -> application.ownershipIssueId().isPresent()) - .forEach(application -> { - IssueId ownershipIssueId = application.ownershipIssueId().get(); - ownershipIssues.getConfirmedOwner(ownershipIssueId).ifPresent(owner -> { - controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.withOwner(owner))); - }); - }); - return 1.0; + .withProjectId() + .withProductionDeployment() + .asList() + .stream() + .filter(application -> application.ownershipIssueId().isPresent()) + .forEach(application -> { + attempts.incrementAndGet(); + IssueId issueId = application.ownershipIssueId().get(); + try { + ownershipIssues.getConfirmedOwner(issueId).ifPresent(owner -> { + controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.withOwner(owner))); + }); + } + catch (RuntimeException e) { + failures.incrementAndGet(); + log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); + } + }); + return asSuccessFactor(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 91dfed500e3..56bf870c7fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -42,6 +42,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(upgrader); maintainers.addAll(osUpgraders(controller, intervals.osUpgrader)); maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); + maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); maintainers.add(new MetricsReporter(controller, metric)); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9e3da506ca8..40191190eff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import java.util.logging.Level; @@ -62,9 +63,8 @@ public class DeploymentExpirer extends ControllerMaintainer { .map(type -> new JobId(instance, type)); if (jobId.isEmpty()) return false; - return controller().jobController().last(jobId.get()) - .flatMap(Run::end) - .map(end -> end.plus(ttl.get()).isBefore(controller().clock().instant())) + return controller().jobController().jobStarts(jobId.get()).stream().findFirst() + .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant())) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java new file mode 100644 index 00000000000..a69af024b96 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -0,0 +1,92 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + +/** + * Upgrades instances in manually deployed zones to the system version, at a convenient time. + * + * @author jonmv + */ +public class DeploymentUpgrader extends ControllerMaintainer { + + public DeploymentUpgrader(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected double maintain() { + AtomicInteger attempts = new AtomicInteger(); + AtomicInteger failures = new AtomicInteger(); + Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty()); + for (Application application : controller().applications().readable()) + for (Instance instance : application.instances().values()) + for (Deployment deployment : instance.deployments().values()) + try { + attempts.incrementAndGet(); + JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + if ( ! deployment.zone().environment().isManuallyDeployed()) continue; + if ( ! deployment.version().isBefore(target.targetPlatform())) continue; + if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue; + if ( ! isLikelyNightFor(job)) continue; + + log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); + controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true); + } catch (Exception e) { + failures.incrementAndGet(); + log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + + ": " + Exceptions.toMessageString(e) + ". Retrying in " + + interval()); + } + return asSuccessFactor(attempts.get(), failures.get()); + } + + private boolean isLikelyNightFor(JobId job) { + int hour = hourOf(controller().clock().instant()); + int[] runStarts = controller().jobController().jobStarts(job).stream() + .mapToInt(DeploymentUpgrader::hourOf) + .toArray(); + int localNight = mostLikelyWeeHour(runStarts); + return Math.abs(hour - localNight) <= 1; + } + + static int mostLikelyWeeHour(int[] starts) { + double weight = 1; + double[] buckets = new double[24]; + for (int start : starts) + buckets[start] += weight *= 0.8; // Weight more recent deployments higher. + + int best = -1; + double min = Double.MAX_VALUE; + for (int i = 12; i < 36; i++) { + double sum = 0; + for (int j = -12; j < 12; j++) + sum += buckets[(i + j) % 24] / (Math.abs(j) + 1); + + if (sum < min) { + min = sum; + best = i; + } + } + return (best + 2) % 24; + } + + private static int hourOf(Instant instant) { + return (int) (instant.toEpochMilli() / 3_600_000 % 24); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 85a69b0f338..cd7ce8c3fa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -86,14 +86,14 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } /** - * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs. + * If it's been four days since the cert has been refreshed, re-trigger all prod deployment jobs. */ private void deployRefreshedCertificates() { var now = clock.instant(); curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); - if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) { + if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 5942291f2b0..d3b05922d26 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -47,10 +47,9 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (currentTarget.isEmpty()) return; if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete - controller().upgradeOsIn(cloud, - release.version(currentTarget.get(), now), - release.upgradeBudget(), - false); + Version version = release.version(currentTarget.get(), now); + if (!version.isAfter(currentTarget.get().osVersion().version())) return; + controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false); } private boolean upgradingToNewMajor(CloudName cloud) { @@ -112,7 +111,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { /** The cool-down period that must pass before a stable version can be used */ private Duration cooldown() { - return system.isCd() ? Duration.ZERO : Duration.ofDays(14); + return system.isCd() ? Duration.ZERO : Duration.ofDays(7); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index ab988bcf0ac..e0441df025b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -20,6 +21,9 @@ import java.util.stream.Collectors; */ public class ResourceTagMaintainer extends ControllerMaintainer { + static final ApplicationId SHARED_HOST_APPLICATION = ApplicationId.from("hosted-vespa", "shared-host", "default"); + static final ApplicationId INFRASTRUCTURE_APPLICATION = ApplicationId.from("hosted-vespa", "infrastructure", "default"); + private final ResourceTagger resourceTagger; public ResourceTagMaintainer(Controller controller, Duration interval, ResourceTagger resourceTagger) { @@ -33,7 +37,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { .ofCloud(CloudName.from("aws")) .reachable() .zones().forEach(zone -> { - Map<HostName, Optional<ApplicationId>> applicationOfHosts = getTenantOfParentHosts(zone.getId()); + Map<HostName, ApplicationId> applicationOfHosts = getTenantOfParentHosts(zone.getId()); int taggedResources = resourceTagger.tagResources(zone, applicationOfHosts); if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); @@ -41,15 +45,21 @@ public class ResourceTagMaintainer extends ControllerMaintainer { return 1.0; } - private Map<HostName, Optional<ApplicationId>> getTenantOfParentHosts(ZoneId zoneId) { + private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { return controller().serviceRegistry().configServer().nodeRepository() .list(zoneId, false) .stream() .filter(node -> node.type().isHost()) .collect(Collectors.toMap( Node::hostname, - Node::exclusiveTo, + this::getApplicationId, (node1, node2) -> node1 )); } + + private ApplicationId getApplicationId(Node node) { + if (node.type() == NodeType.host) + return node.exclusiveTo().orElse(SHARED_HOST_APPLICATION); + return INFRASTRUCTURE_APPLICATION; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 87527085237..8ffa4823ead 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -83,6 +83,7 @@ class RunSerializer { private static final String endField = "end"; private static final String statusField = "status"; private static final String versionsField = "versions"; + private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; private static final String repositoryField = "repository"; private static final String branchField = "branch"; @@ -131,6 +132,7 @@ class RunSerializer { runObject.field(numberField).asLong()), steps, versionsFromSlime(runObject.field(versionsField)), + runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), @@ -177,7 +179,7 @@ class RunSerializer { compileVersion, buildTime, sourceUrl, commit); } - // Don't change this — introduce a separate array instead. + // Don't change this — introduce a separate array instead. private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) { if ( ! summaryArray.valid()) return Optional.empty(); @@ -215,6 +217,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d022588c757..c1e6c362c83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -245,7 +245,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -546,28 +546,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (controller.tenants().get(tenantName).isEmpty()) return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); + List<Application> applications = applicationName.isEmpty() ? + controller.applications().asList(tenant) : + controller.applications().getApplication(TenantAndApplicationId.from(tenantName, applicationName.get())) + .map(List::of) + .orElseThrow(() -> new NotExistsException("Application '" + applicationName.get() + "' does not exist")); + Slime slime = new Slime(); Cursor applicationArray = slime.setArray(); - for (Application application : controller.applications().asList(tenant)) { - if (applicationName.map(application.id().application().value()::equals).orElse(true)) { - Cursor applicationObject = applicationArray.addObject(); - applicationObject.setString("tenant", application.id().tenant().value()); - applicationObject.setString("application", application.id().application().value()); - applicationObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), - request.getUri()).toString()); - Cursor instanceArray = applicationObject.setArray("instances"); - for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() - : application.instances().keySet()) { - Cursor instanceObject = instanceArray.addObject(); - instanceObject.setString("instance", instance.value()); - instanceObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value() + - "/instance/" + instance.value(), - request.getUri()).toString()); - } + for (Application application : applications) { + Cursor applicationObject = applicationArray.addObject(); + applicationObject.setString("tenant", application.id().tenant().value()); + applicationObject.setString("application", application.id().application().value()); + applicationObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value(), + request.getUri()).toString()); + Cursor instanceArray = applicationObject.setArray("instances"); + for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() + : application.instances().keySet()) { + Cursor instanceObject = instanceArray.addObject(); + instanceObject.setString("instance", instance.value()); + instanceObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + instance.value(), + request.getUri()).toString()); } } return new SlimeJsonResponse(slime); @@ -1383,11 +1387,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); response.setString("revision", deployment.applicationVersion().id()); - response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); + Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment); + response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) - .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); + .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); - DeploymentStatus status = controller.jobController().deploymentStatus(application); application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.applicationVersion().source(), response); @@ -1396,18 +1400,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) toSlime(instance.rotations(), instance.rotationStatus(), deployment, response); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) - .ifPresent(stepStatus -> { - JobControllerApiHandlerHelper.applicationVersionToSlime( - response.setObject("applicationVersion"), deployment.applicationVersion()); - if (!status.jobsToRun().containsKey(stepStatus.job().get())) - response.setString("status", "complete"); - else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) - response.setString("status", "pending"); - else response.setString("status", "running"); - }); + if (!deployment.zone().environment().isManuallyDeployed()) { + DeploymentStatus status = controller.jobController().deploymentStatus(application); + JobType.from(controller.system(), deployment.zone()) + .map(type -> new JobId(instance.id(), type)) + .map(status.jobSteps()::get) + .ifPresent(stepStatus -> { + JobControllerApiHandlerHelper.applicationVersionToSlime( + response.setObject("applicationVersion"), deployment.applicationVersion()); + if (!status.jobsToRun().containsKey(stepStatus.job().get())) + response.setString("status", "complete"); + else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) + response.setString("status", "pending"); + else response.setString("status", "running"); + }); + } } response.setDouble("quota", deployment.quota().rate()); @@ -1435,6 +1442,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } + private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { + return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get())) + .stream().findFirst().orElse(deployment.at()); + } + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { if ( ! applicationVersion.isUnknown()) { object.setLong("buildNumber", applicationVersion.buildNumber().getAsLong()); @@ -1666,9 +1678,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) { - String[] parts = restPath.split("/status/"); - String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, parts[0], parts[1]); + if (restPath.contains("/status/")) { + String[] parts = restPath.split("/status/", 2); + String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, serviceName, parts[0], parts[1]); return new HtmlResponse(result); } @@ -2079,12 +2091,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.serviceRegistry().roleService().getTenantRole(tenant.name()), cloudTenant.tenantSecretStores()); - var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name()); - var usedQuota = applications.stream() - .map(Application::quotaUsage) - .reduce(QuotaUsage.none, QuotaUsage::add); + try { + var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name()); + var usedQuota = applications.stream() + .map(Application::quotaUsage) + .reduce(QuotaUsage.none, QuotaUsage::add); - toSlime(tenantQuota, usedQuota, object.setObject("quota")); + toSlime(tenantQuota, usedQuota, object.setObject("quota")); + } catch (Exception e) { + log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e))); + } cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role)); @@ -2187,9 +2203,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) { Optional<Instant> lastDev = applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.deployments().values().stream()) - .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(Deployment::at) + .flatMap(instance -> instance.deployments().values().stream() + .filter(deployment -> deployment.zone().environment() == Environment.dev) + .map(deployment -> lastDeploymentStart(instance.id(), deployment))) .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index d246b20f2fe..eb9b0e03f3d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -40,10 +40,12 @@ import java.time.Instant; import java.time.format.TextStyle; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; @@ -84,7 +86,7 @@ class JobControllerApiHandlerHelper { Cursor jobObject = jobsArray.addObject(); jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, baseUriForJobs); + toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs); }); return new SlimeJsonResponse(slime); @@ -117,11 +119,19 @@ class JobControllerApiHandlerHelper { } /** Returns a response with the runs for the given job type. */ - static HttpResponse runResponse(Map<RunId, Run> runs, URI baseUriForJobType) { + static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); - runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); + // TODO (freva): Remove after console migrated to use new format + if (limitStr.isEmpty()) + runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); + else { + int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); + toSlime(cursor.setArray("runs"), runs.values().stream() + .sorted(Comparator.comparing((Run run) -> run.id().number()).reversed()) + .collect(Collectors.toUnmodifiableList()), limit, baseUriForJobType); + } return new SlimeJsonResponse(slime); } @@ -377,7 +387,7 @@ class JobControllerApiHandlerHelper { toSlime(runObject.setObject("versions"), versions); } - toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob); + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); }); } @@ -428,8 +438,8 @@ class JobControllerApiHandlerHelper { return Optional.of(versions.get(i)); } - private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) { - runs.stream().limit(10).forEach(run -> { + private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) { + runs.stream().limit(limit).forEach(run -> { Cursor runObject = runsArray.addObject(); runObject.setLong("id", run.id().number()); runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 4951f2ba33b..c13a9b42382 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -113,7 +113,7 @@ public class Badges { List<Run> runs = status.runs().descendingMap().values().stream() .filter(Run::hasEnded) .skip(1) - .limit(length + (lastTriggered.hasEnded() ? 0 : 1)) + .limit(length) .collect(toList()); boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 14244d7bdda..d8bddac4187 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -58,6 +57,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -383,28 +383,81 @@ public class InternalStepRunnerTest { @Test public void vespaLogIsCopied() { + // Tests fail. We should get logs. This fails too, on the first attempt. RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.ERROR); - tester.configServer().setLogStream(vespaLog); + tester.configServer().setLogStream(() -> { throw new ConfigServerException(ConfigServerException.ErrorCode.NOT_FOUND, "404", "context"); }); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.copyVespaLogs)); assertTestLogEntries(id, Step.copyVespaLogs, - new LogEntry(lastId + 2, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 2, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 4, tester.clock().instant(), info, + "Found no logs, but will retry")); + + // Config servers now provide the log, and we get it. + tester.configServer().setLogStream(() -> vespaLog); + tester.runner().run(); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertTestLogEntries(id, Step.copyVespaLogs, + new LogEntry(lastId + 2, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 4, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 3, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 6, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 4, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 7, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, + new LogEntry(lastId + 8, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); } @Test + public void realDeploymentRequiresForTesterCert() { + tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); + var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); + ZoneId testZone = JobType.systemTest.zone(tester.controller().system()); + RunId id = app.newRun(JobType.systemTest); + tester.configServer().throwOnPrepare(instanceId -> { + if (instanceId.instance().isTester()) + throw new ConfigServerException(ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY, "provisioning", "deploy tester"); + }); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); + + List<X509Certificate> oldTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); + X509Certificate oldCert = tester.jobs().run(id).get().testerCertificate().get(); + oldTrusted.add(oldCert); + assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + + tester.configServer().throwOnNextPrepare(null); + tester.runner().run(); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); + + List<X509Certificate> newTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); + X509Certificate newCert = tester.jobs().run(id).get().testerCertificate().get(); + newTrusted.add(newCert); + assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertNotEquals(oldCert, newCert); + } + + @Test public void certificateTimeoutAbortsJob() { tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index dca79c43afe..92c8cbc4889 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -63,6 +63,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -93,9 +98,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private List<ProtonMetrics> protonMetrics; private Version lastPrepareVersion = null; - private RuntimeException prepareException = null; + private Consumer<ApplicationId> prepareException = null; private ConfigChangeActions configChangeActions = null; - private String log = "INFO - All good"; + private Supplier<String> log = () -> "INFO - All good"; @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { @@ -205,9 +210,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Optional.ofNullable(lastPrepareVersion); } + /** Sets a function that may throw, determined by app id. */ + public void throwOnPrepare(Consumer<ApplicationId> prepareThrower) { + this.prepareException = prepareThrower; + } + /** The exception to throw on the next prepare run, or null to continue normally */ public void throwOnNextPrepare(RuntimeException prepareException) { - this.prepareException = prepareException; + this.prepareException = prepareException == null ? null : id -> { this.prepareException = null; throw prepareException; }; } /** Set version for an application in a given zone */ @@ -366,11 +376,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentData deployment) { lastPrepareVersion = deployment.platform(); - if (prepareException != null) { - RuntimeException prepareException = this.prepareException; - this.prepareException = null; - throw prepareException; - } + if (prepareException != null) + prepareException.accept(ApplicationId.from(deployment.instance().tenant(), + deployment.instance().application(), + deployment.instance().instance())); DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); @@ -523,7 +532,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public String getClusterControllerStatus(DeploymentId deployment, String node, String subPath) { + public String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath) { return "<h1>OK</h1>"; } @@ -554,7 +563,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters) { - return new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)); + return new ByteArrayInputStream(log.get().getBytes(StandardCharsets.UTF_8)); } @Override @@ -562,7 +571,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return new ProxyResponse("{\"path\":\"" + path + "\"}", "application/json", 200); } - public void setLogStream(String log) { + public void setLogStream(Supplier<String> log) { this.log = log; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index ffc82f90ad4..31f8aaf9e2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -70,7 +70,9 @@ public class DeploymentExpirerTest { assertEquals(1, permanentDeployments(prodApp.instance())); // Dev application expires when enough time has passed since most recent attempt + // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); + tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java new file mode 100644 index 00000000000..7a8f775e8b1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -0,0 +1,101 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour; +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author jonmv + */ +public class DeploymentUpgraderTest { + + private final DeploymentTester tester = new DeploymentTester(); + + @Test + public void testDeploymentUpgrading() { + ZoneId devZone = ZoneId.from(Environment.dev, RegionName.from("us-east-1")); + DeploymentUpgrader upgrader = new DeploymentUpgrader(tester.controller(), Duration.ofDays(1)); + var devApp = tester.newDeploymentContext("tenant1", "app1", "default"); + var prodApp = tester.newDeploymentContext("tenant2", "app2", "default"); + + ApplicationPackage appPackage = new ApplicationPackageBuilder().region("us-west-1").build(); + Version systemVersion = tester.controller().readSystemVersion(); + Instant start = tester.clock().instant().truncatedTo(MILLIS); + + devApp.runJob(devUsEast1, appPackage); + prodApp.submit(appPackage).deploy(); + assertEquals(systemVersion, tester.jobs().last(devApp.instanceId(), devUsEast1).get().versions().targetPlatform()); + assertEquals(systemVersion, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().versions().targetPlatform()); + + // Not upgraded initially + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // Not upgraded immediately after system upgrades + tester.controllerTester().upgradeSystem(new Version(7, 8, 9)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 14 hours pass, but not upgraded before a day has passed since last deployment + tester.clock().advance(Duration.ofHours(14)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 35 hours pass, but not upgraded since it's not likely in the middle of the night + tester.clock().advance(Duration.ofHours(21)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 38 hours pass, and the dev deployment, only, is upgraded + tester.clock().advance(Duration.ofHours(3)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertTrue(tester.jobs().last(devApp.instanceId(), devUsEast1).get().isRedeployment()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + devApp.runJob(devUsEast1); + + // After the upgrade, the dev app is mostly (re)deployed to at night, but this doesn't affect what is likely the night. + tester.controllerTester().upgradeSystem(new Version(7, 9, 11)); + tester.clock().advance(Duration.ofHours(48)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + } + + @Test + public void testNight() { + assertEquals(16, mostLikelyWeeHour(new int[]{ 0, 1, 2, 3, 4, 5, 6 })); + assertEquals(14, mostLikelyWeeHour(new int[]{ 22, 23, 0, 1, 2, 3, 4 })); + assertEquals(18, mostLikelyWeeHour(new int[]{ 6, 5, 4, 3, 2, 1, 0 })); + assertEquals(20, mostLikelyWeeHour(new int[]{ 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 11 })); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index ce219b8beed..3ed26b4fb6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -82,7 +82,7 @@ public class EndpointCertificateMaintainerTest { } @Test - public void refreshed_certificate_is_deployed_after_one_week() { + public void refreshed_certificate_is_deployed_after_four_days() { var appId = ApplicationId.from("tenant", "application", "default"); DeploymentTester deploymentTester = new DeploymentTester(tester); @@ -107,7 +107,7 @@ public class EndpointCertificateMaintainerTest { maintainer.maintain(); - tester.clock().advance(Duration.ofDays(8)); + tester.clock().advance(Duration.ofDays(4)); deploymentContext.assertNotRunning(productionUsWest1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index f677cc079cc..023c5671b60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -131,8 +131,8 @@ public class JobRunnerTest { Map<Step, Status> steps = run.get().stepStatuses(); runner.maintain(); assertEquals(steps, run.get().stepStatuses()); - assertEquals(List.of(deployTester), run.get().readySteps()); - assertStepsWithStartTime(run.get(), deployTester); + assertEquals(List.of(deployTester, deployReal), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal); outcomes.put(deployTester, running); runner.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index d87dfcfa315..770d0a898fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -116,7 +116,7 @@ public class OsUpgradeSchedulerTest { tester.controller().osVersionTarget(cloud).get().osVersion().version()); // Enough time passes since promotion of stable release - tester.clock().advance(Duration.ofDays(14).plus(Duration.ofSeconds(1))); + tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); scheduler.maintain(); OsVersionTarget target0 = tester.controller().osVersionTarget(cloud).get(); assertEquals(version1, target0.osVersion().version()); @@ -132,6 +132,16 @@ public class OsUpgradeSchedulerTest { assertEquals("Target is unchanged as not enough time has passed", version1, target1.osVersion().version()); assertEquals("Target is not re-scheduled", target0.scheduledAt(), target1.scheduledAt()); + + // A newer version is triggered manually + Version version3 = Version.fromString("8.3"); + tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); + + // Enough time passes for stable version to be promoted. Nothing happens as stable is now before the manually + // triggered version + tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); + scheduler.maintain(); + assertEquals(version3, tester.controller().osVersionTarget(cloud).get().osVersion().version()); } private static ZoneApi zone(String id, CloudName cloud) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 814dc2a3f50..516c28ab5cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.maintenance.ResourceTagMaintainer.SHARED_HOST_APPLICATION; import static org.junit.Assert.assertEquals; /** @@ -35,9 +36,9 @@ public class ResourceTagMaintainerTest { mockResourceTagger); resourceTagMaintainer.maintain(); assertEquals(2, mockResourceTagger.getValues().size()); - Map<HostName, Optional<ApplicationId>> applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); - assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.from("parentHostA.prod.region-2")).get()); - assertEquals(Optional.empty(), applicationForHost.get(HostName.from("parentHostB.prod.region-2"))); + Map<HostName, ApplicationId> applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); + assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.from("parentHostA.prod.region-2"))); + assertEquals(SHARED_HOST_APPLICATION, applicationForHost.get(HostName.from("parentHostB.prod.region-2"))); } private void setUpZones() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 6e12373640d..03a050db74e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -149,7 +149,7 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); - Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production); + Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 3fa593fcf64..a01097cfcb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -225,6 +225,10 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), new File("application-list.json")); + // GET tenant application instances for application that does not exist + tester.assertResponse(request("/application/v4/tenant/tenant1/application/fake-app/instance/", GET).userIdentity(USER_ID), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Application 'fake-app' does not exist\"}", 404); + // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), new File("application-list.json")); @@ -801,7 +805,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET system test job overview. tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET) - .userIdentity(USER_ID), + .userIdentity(USER_ID).properties(Map.of("limit", "100")), new File("system-test-job.json")); // GET system test run 1 details. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 1be7f16e85f..72295497c03 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -131,14 +131,14 @@ public class JobControllerApiHandlerHelperTest { // Only us-east-3 is verified, on revision1. // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2, // one success from revision2 to revision3 and two failures from revision1 to revision3. - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json"); var userApp = tester.newDeploymentContext(app.instanceId().tenant().value(), app.instanceId().application().value(), "user"); userApp.runJob(devAwsUsEast2a, applicationPackage); - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } @@ -151,10 +151,10 @@ public class JobControllerApiHandlerHelperTest { ZoneId zone = JobType.devUsEast1.zone(tester.controller().system()); tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); - tester.configServer().setLogStream("1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); + tester.configServer().setLogStream(() -> "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json"); - tester.configServer().setLogStream("Nope, this won't be logged"); + tester.configServer().setLogStream(() -> "Nope, this won't be logged"); tester.configServer().convergeServices(app.instanceId(), zone); tester.runner().run(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 3234333c092..c0988e8c301 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -1,69 +1,119 @@ { - "1": { - "id": 1, - "status": "success", - "start": "(ignore)", - "end": "(ignore)", - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + "runs": [ + { + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", + "start": "(ignore)", + "status": "running", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "steps": [ + { + "name": "deployTester", + "status": "unfinished" + }, + { + "name": "installTester", + "status": "unfinished" + }, + { + "name": "deployReal", + "status": "unfinished" + }, + { + "name": "installReal", + "status": "unfinished" + }, + { + "name": "startTests", + "status": "unfinished" + }, + { + "name": "endTests", + "status": "unfinished" + }, + { + "name": "copyVespaLogs", + "status": "unfinished" + }, + { + "name": "deactivateReal", + "status": "unfinished" + }, + { + "name": "deactivateTester", + "status": "unfinished" + }, + { + "name": "report", + "status": "unfinished" + } + ] }, - "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", - "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" - }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" - }, - "2": { - "id": 2, - "status": "running", - "start": "(ignore)", - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.4-commit1", - "build": 4, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", - "deployReal": "unfinished", - "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "copyVespaLogs": "unfinished", - "deactivateReal": "unfinished", - "deactivateTester": "unfinished", - "report": "unfinished" - }, - "tasks": {}, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2" - } + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "succeeded" + }, + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "startTests", + "status": "succeeded" + }, + { + "name": "endTests", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index e906df94023..668baa50cc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -43,6 +43,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "DeploymentUpgrader" + }, + { "name": "EndpointCertificateMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 766a84ca33c..63474ebb7c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -53,6 +53,8 @@ public class BadgeApiTest extends ControllerContainerTest { tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), Files.readString(Paths.get(responseFiles + "overview.svg")), 200); + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), + Files.readString(Paths.get(responseFiles + "single-running.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); @@ -66,6 +68,9 @@ public class BadgeApiTest extends ControllerContainerTest { tester.serviceRegistry().clock().advance(Duration.ofSeconds(1)); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history2.svg")), 200); + + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), + Files.readString(Paths.get(responseFiles + "single-done.svg")), 200); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg new file mode 100644 index 00000000000..3bcbed97499 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg @@ -0,0 +1,81 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f244' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='294.82606' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='294.82606' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text> + <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text> + </g> +</svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg new file mode 100644 index 00000000000..27e967f8e46 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg @@ -0,0 +1,81 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f244' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='294.82606' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#run-on-failure)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='294.82606' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text> + <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text> + </g> +</svg> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index a61410ebf31..152a0d3eef3 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -31,6 +31,16 @@ endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + if (VESPA_OS_DISTRO_NAME STREQUAL "CentOS Stream") + set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) + else() + set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) + endif() +endfunction() + +function(setup_vespa_default_build_settings_rocky_8_4) + message("-- Setting up default build settings for rocky 8.4") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) endfunction() @@ -187,6 +197,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_centos_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") setup_vespa_default_build_settings_centos_8() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rocky 8.4") + setup_vespa_default_build_settings_rocky_8_4() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") diff --git a/dist/vespa.spec b/dist/vespa.spec index 13ea9a733e1..d8e3653af39 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -27,8 +27,10 @@ License: Commercial URL: http://vespa.ai Source0: vespa-%{version}.tar.gz -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} BuildRequires: epel-release +%endif +%if 0%{?centos} %if 0%{?el7} && ! 0%{?amzn2} BuildRequires: centos-release-scl %endif @@ -62,6 +64,7 @@ BuildRequires: maven BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python36-devel +BuildRequires: glibc-langpack-en %endif %if 0%{?fedora} BuildRequires: gcc-c++ @@ -69,6 +72,7 @@ BuildRequires: libatomic BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python3-devel +BuildRequires: glibc-langpack-en %endif %if 0%{?el7} BuildRequires: cmake3 @@ -92,11 +96,18 @@ BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 +%if 0%{?centos} || 0%{?rocky} %if 0%{?centos} # Current cmake on CentOS 8 is broken and manually requires libarchive install BuildRequires: libarchive +%endif %define _command_cmake cmake +%global _centos_stream %(grep -qs '^NAME="CentOS Stream"' /etc/os-release && echo 1 || echo 0) +%if 0%{?_centos_stream} +BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) +%else BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12) +%endif %else BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) %endif @@ -219,8 +230,12 @@ Requires: vespa-valgrind >= 3.17.0-1 %endif %endif %if 0%{?el8} -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} +%if 0%{?_centos_stream} +%define _vespa_llvm_version 12 +%else %define _vespa_llvm_version 11 +%endif %else %define _vespa_llvm_version 10 %endif @@ -290,7 +305,7 @@ Vespa - The open big data serving engine - base Summary: Vespa - The open big data serving engine - base C++ libraries -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} Requires: epel-release %endif %if 0%{?amzn2} @@ -307,6 +322,7 @@ Requires: vespa-lz4 >= 1.9.2-2 Requires: vespa-libzstd >= 1.4.5-2 %if 0%{?el8} Requires: openblas +Requires: glibc-langpack-en %else %if 0%{?amzn2} Requires: vespa-openblas = 0.3.15 @@ -319,6 +335,9 @@ Requires: vespa-re2 = 20190801 %else Requires: re2 %endif +%if 0%{?fedora} +Requires: glibc-langpack-en +%endif %description base-libs @@ -347,8 +366,12 @@ Requires: libicu Requires: openssl-libs %endif %if 0%{?el8} -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} +%if 0%{?_centos_stream} +Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) +%else Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12) +%endif %else Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) %endif @@ -771,7 +794,6 @@ fi %{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar %{_prefix}/lib/jars/container-apache-http-client-bundle-jar-with-dependencies.jar %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar -%{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar %{_prefix}/lib/jars/defaults-jar-with-dependencies.jar diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 8bfa85d8b16..e922d878dd7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -20,6 +20,12 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> @@ -58,10 +64,6 @@ <artifactId>airline</artifactId> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 91cb1895a6d..4a104deaf55 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -137,12 +137,6 @@ public class Flags { "Takes effect on the next suspension request to the Orchestrator.", APPLICATION_ID); - public static final UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( - "encrypt-disk", false, - List.of("hakonhall"), "2021-05-05", "2021-08-05", - "Allow migrating an unencrypted data partition to being encrypted.", - "Takes effect on next host-admin tick."); - public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag( "encrypt-dirty-disk", false, List.of("hakonhall"), "2021-05-14", "2021-08-05", @@ -151,7 +145,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2021-07-01", + List.of("geirst"), "2021-01-27", "2021-09-01", "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -184,7 +178,7 @@ public class Flags { public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2021-07-01", + List.of("vekterli"), "2021-02-19", "2021-09-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -199,42 +193,42 @@ public class Flags { public static final UnboundIntFlag NUM_DISTRIBUTOR_STRIPES = defineIntFlag( "num-distributor-stripes", 0, - List.of("geirst", "vekterli"), "2021-04-20", "2021-07-01", + List.of("geirst", "vekterli"), "2021-04-20", "2021-09-01", "Specifies the number of stripes used by the distributor. When 0, legacy single stripe behavior is used.", "Takes effect after distributor restart", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 16, - List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 1024, - List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( "use-external-rank-expression", false, - List.of("baldersheim"), "2021-05-24", "2021-07-01", + List.of("baldersheim"), "2021-05-24", "2021-09-01", "Whether to use distributed external rank expression or inline in rankproperties", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundBooleanFlag DISTRIBUTE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( "distribute-external-rank-expression", false, - List.of("baldersheim"), "2021-05-27", "2021-07-01", + List.of("baldersheim"), "2021-05-27", "2021-09-01", "Whether to use distributed external rank expression files by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( "large-rank-expression-limit", 0x10000, - List.of("baldersheim"), "2021-06-09", "2021-07-01", + List.of("baldersheim"), "2021-06-09", "2021-09-01", "Limit for size of rank expressions distributed by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -281,7 +275,7 @@ public class Flags { public static final UnboundBooleanFlag DRY_RUN_ONNX_ON_SETUP = defineFeatureFlag( "dry-run-onnx-on-setup", false, - List.of("baldersheim"), "2021-06-23", "2021-08-01", + List.of("baldersheim"), "2021-06-23", "2021-09-01", "Whether to dry run onnx models on setup for better error checking", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -292,6 +286,12 @@ public class Flags { "List of applications where encryption of their host should be deferred", "Takes effect on next run of HostEncrypter"); + public static final UnboundBooleanFlag PODMAN3 = defineFeatureFlag( + "podman3", true, + List.of("mpolden"), "2021-07-05", "2021-09-01", + "Whether to use Podman 3 on supported hosts", + "Takes effect on host-admin restart"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index d497b9de6ba..cccd75fe7ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -169,6 +169,20 @@ public class PermanentFlags { APPLICATION_ID ); + public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( + "resource-limit-disk", 0.8, + "Resource limit (between 0.0 and 1.0) for disk used by cluster controller for when to block feed", + "Takes effect on next deployment", + APPLICATION_ID + ); + + public static final UnboundDoubleFlag RESOURCE_LIMIT_MEMORY = defineDoubleFlag( + "resource-limit-memory", 0.8, + "Resource limit (between 0.0 and 1.0) for memory used by cluster controller for when to block feed", + "Takes effect on next deployment", + APPLICATION_ID + ); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 1f85658430f..2a8ec1b9bb9 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -31,22 +31,21 @@ <artifactId>slf4j-api</artifactId> <scope>provided</scope> </dependency> - - <!-- compile scope --> + <!-- Apache client artifacts are provided by the jdisc container and are therefore scoped as such --> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents.client5</groupId> <artifactId>httpclient5</artifactId> - <scope>compile</scope> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java index 896799d4492..78a7a858aba 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java @@ -6,12 +6,16 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Transformer; +import java.util.logging.Logger; +import java.util.logging.Level; + /** * @author Simon Thoresen Hult */ public final class NormalizeExpression extends Expression { private final Linguistics linguistics; + private static final Logger logger = Logger.getLogger(NormalizeExpression.class.getName()); public NormalizeExpression(Linguistics linguistics) { super(DataType.STRING); @@ -22,9 +26,35 @@ public final class NormalizeExpression extends Expression { return linguistics; } + + private static String escape(String str) { + StringBuilder buf = new StringBuilder(); + for (char c : str.toCharArray()) { + if (c >= ' ') { + buf.append(c); + } else { + buf.append(String.format("U+%04X", (int)c)); + } + } + return buf.toString(); + } + @Override protected void doExecute(ExecutionContext context) { Transformer transformer = linguistics.getTransformer(); + var orig = String.valueOf(context.getValue()); + var lang = context.resolveLanguage(linguistics); + var transformed = transformer.accentDrop(orig, lang); + try { + context.setValue(new StringFieldValue(transformed)); + return; + } catch (IllegalArgumentException ex) { + String msg = ("bad normalize, \n" + + "original: >>> " + escape(orig) + " <<<\n" + + " -> accentDrop(" + lang + ") -> \n" + + "transformed: >>> " + escape(transformed) + " <<<"); + logger.log(Level.SEVERE, msg); + } context.setValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getValue()), context.resolveLanguage(linguistics)))); } @@ -45,14 +75,10 @@ public final class NormalizeExpression extends Expression { } @Override - public boolean equals(Object obj) { - if (!(obj instanceof NormalizeExpression)) { - return false; - } - NormalizeExpression rhs = (NormalizeExpression)obj; - if (linguistics != rhs.linguistics) { - return false; - } + public boolean equals(Object o) { + if (!(o instanceof NormalizeExpression)) return false; + NormalizeExpression other = (NormalizeExpression)o; + if (linguistics != other.linguistics) return false; return true; } @@ -60,4 +86,5 @@ public final class NormalizeExpression extends Expression { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java index 8b4f1a8b344..3ad1b129b4d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java @@ -6,8 +6,10 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Transformer; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; + import org.junit.Test; import org.mockito.Mockito; @@ -56,4 +58,37 @@ public class NormalizeTestCase { assertTrue(val instanceof StringFieldValue); assertEquals("beyonce", ((StringFieldValue)val).getString()); } + + class MyMockTransformer implements Transformer { + boolean first = true; + @Override + public String accentDrop(String input, Language language) { + if (first) { + first = false; + return input.replace(' ', '\u0008'); + } else { + return input.replace(' ', '/'); + } + } + } + + class MyMockLinguistics extends SimpleLinguistics { + private Transformer transformer = new MyMockTransformer(); + @Override + public Transformer getTransformer() { + return transformer; + } + } + + @Test + public void requireThatBadNormalizeRetries() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setLanguage(Language.ENGLISH); + ctx.setValue(new StringFieldValue("bad norm")); + var linguistics = new MyMockLinguistics(); + new NormalizeExpression(linguistics).execute(ctx); + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("bad/norm", ((StringFieldValue)val).getString()); + } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java index e9a0d0253e1..223c8191186 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -90,4 +90,5 @@ public class ExpressionTestCase { assertEquals(foo.hashCode(), bar.hashCode()); assertEquals(foo, bar); } + } diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 9f4572736c8..0c89872aa46 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -24,8 +24,20 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.amazonaws</groupId> diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp index 9259f992d6c..9ff352abcf5 100644 --- a/messagebus/src/tests/targetpool/targetpool.cpp +++ b/messagebus/src/tests/targetpool/targetpool.cpp @@ -37,7 +37,7 @@ TEST("targetpool_test") { FRT_Supervisor & orb = server.supervisor(); std::unique_ptr<PoolTimer> ptr(new PoolTimer()); PoolTimer &timer = *ptr; - RPCTargetPool pool(std::move(ptr), 0.666); + RPCTargetPool pool(std::move(ptr), 0.666, 1 ); // Assert that all connections expire. RPCTarget::SP target; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 34ba486d0c9..41298c28a88 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -82,7 +82,7 @@ struct TargetPoolTask : public FNET_Task { TransportConfig toFNETConfig(const RPCNetworkParams & params) { - return TransportConfig() + return TransportConfig(params.getNumNetworkThreads()) .maxInputBufferSize(params.getMaxInputBufferSize()) .maxOutputBufferSize(params.getMaxOutputBufferSize()) .tcpNoDelay(params.getTcpNoDelay()); @@ -125,7 +125,7 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _owner(nullptr), _ident(params.getIdentity()), - _threadPool(std::make_unique<FastOS_ThreadPool>(128000, 0)), + _threadPool(std::make_unique<FastOS_ThreadPool>(128_Ki, 0)), _transport(std::make_unique<FNET_Transport>(toFNETConfig(params))), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _scheduler(*_transport->GetScheduler()), @@ -133,7 +133,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)), _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)), _requestedPort(params.getListenPort()), - _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())), + _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs(), params.getNumRpcTargets())), _targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)), _servicePool(std::make_unique<RPCServicePool>(*_mirror, 4_Ki)), _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 64_Ki)), diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index d701358fc84..a3722376086 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -74,7 +74,6 @@ private: bool _allowDispatchForEncode; bool _allowDispatchForDecode; - /** * Resolves and assigns a service address for the given recipient using the * given address. This is called by the {@link diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index d440d3b012b..59802167c34 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -16,6 +16,8 @@ RPCNetworkParams::RPCNetworkParams(config::ConfigUri configUri) : _maxInputBufferSize(256_Ki), _maxOutputBufferSize(256_Ki), _numThreads(4), + _numNetworkThreads(1), + _numRpcTargets(1), _tcpNoDelay(true), _dispatchOnEncode(true), _dispatchOnDecode(false), diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index ddb4df1a3a3..37739ee5189 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -20,6 +20,8 @@ private: uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; uint32_t _numThreads; + uint32_t _numNetworkThreads; + uint32_t _numRpcTargets; bool _tcpNoDelay; bool _dispatchOnEncode; bool _dispatchOnDecode; @@ -34,6 +36,26 @@ public: ~RPCNetworkParams(); /** + * Sets number of threads for the network. + * + * @param numNetworkThreads number of threads for the network + * @return This, to allow chaining. + */ + RPCNetworkParams &setNumNetworkThreads(uint32_t numNetworkThreads) { + _numNetworkThreads = numNetworkThreads; + return *this; + } + + uint32_t getNumNetworkThreads() const { return _numNetworkThreads; } + + RPCNetworkParams &setNumRpcTargets(uint32_t numRpcTargets) { + _numRpcTargets = numRpcTargets; + return *this; + } + + uint32_t getNumRpcTargets() const { return _numRpcTargets; } + + /** * Returns the identity to use for the network. * * @return The identity. diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp index b42ac47e54d..cfb1e120a2c 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp @@ -4,23 +4,45 @@ namespace mbus { -RPCTargetPool::Entry::Entry(RPCTarget::SP target, uint64_t lastUse) : - _target(target), - _lastUse(lastUse) +RPCTargetPool::Entry::Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse) + : _targets(std::move(targets)), + _lastUse(lastUse), + _next(0) { } -RPCTargetPool::RPCTargetPool(double expireSecs) : - _lock(), - _targets(), - _timer(new SteadyTimer()), - _expireMillis(static_cast<uint64_t>(expireSecs * 1000)) +RPCTarget::SP +RPCTargetPool::Entry::getTarget(const LockGuard &, uint64_t now) { + if (_next >= _targets.size()) { + _next = 0; + } + RPCTarget::SP target = _targets[_next++]; + if ( ! target->isValid()) { + return RPCTarget::SP(); + } + _lastUse = now; + return target; +} + +bool +RPCTargetPool::Entry::inUse(const LockGuard &) const { + for (const auto & target : _targets) { + if (target.use_count() > 1) { + return true; + } + } + return false; +} + +RPCTargetPool::RPCTargetPool(double expireSecs, size_t numTargetsPerSpec) + : RPCTargetPool(std::make_unique<SteadyTimer>(), expireSecs, numTargetsPerSpec) { } -RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs) : +RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec) : _lock(), _targets(), _timer(std::move(timer)), - _expireMillis(static_cast<uint64_t>(expireSecs * 1000)) + _expireMillis(static_cast<uint64_t>(expireSecs * 1000)), + _numTargetsPerSpec(numTargetsPerSpec) { } RPCTargetPool::~RPCTargetPool() @@ -35,21 +57,12 @@ RPCTargetPool::flushTargets(bool force) LockGuard guard(_lock); TargetMap::iterator it = _targets.begin(); while (it != _targets.end()) { - Entry &entry = it->second; - if (entry._target.get() != nullptr) { - if (entry._target.use_count() > 1) { - entry._lastUse = currentTime; - ++it; - continue; // someone is using this - } - if (!force) { - if (entry._lastUse + _expireMillis > currentTime) { - ++it; - continue; // not sufficiently idle - } - } + const Entry &entry = it->second; + if ( ! entry.inUse(guard) && (force || ((entry.lastUse() + _expireMillis) < currentTime))) { + _targets.erase(it++); // postfix increment to move the iterator + } else { + ++it; } - _targets.erase(it++); // postfix increment to move the iterator } } @@ -68,16 +81,19 @@ RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address) LockGuard guard(_lock); auto it = _targets.find(spec); if (it != _targets.end()) { - Entry &entry = it->second; - if (entry._target->isValid()) { - entry._lastUse = currentTime; - return entry._target; + RPCTarget::SP target = it->second.getTarget(guard, currentTime); + if (target) { + return target; } _targets.erase(it); } - auto ret = std::make_shared<RPCTarget>(spec, orb); - _targets.insert(TargetMap::value_type(spec, Entry(ret, currentTime))); - return ret; + std::vector<RPCTarget::SP> targets; + targets.reserve(_numTargetsPerSpec); + for (size_t i(0); i < _numTargetsPerSpec; i++) { + targets.push_back(std::make_shared<RPCTarget>(spec, orb)); + } + _targets.insert(TargetMap::value_type(spec, Entry(std::move(targets), currentTime))); + return _targets.find(spec)->second.getTarget(guard, currentTime); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h index bc9e1a4b19f..2a6fb75d511 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h @@ -16,24 +16,30 @@ namespace mbus { */ class RPCTargetPool { private: + using LockGuard = std::lock_guard<std::mutex>; /** * Implements a helper class holds the necessary reference and token counter * for a JRT target to keep connections open as long as they get used from * time to time. */ - struct Entry { - RPCTarget::SP _target; - uint64_t _lastUse; - - Entry(RPCTarget::SP target, uint64_t lastUse); + class Entry { + public: + Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse); + RPCTarget::SP getTarget(const LockGuard & guard, uint64_t now); + uint64_t lastUse() const { return _lastUse; } + bool inUse(const LockGuard & guard) const; + private: + std::vector<RPCTarget::SP> _targets; + uint64_t _lastUse; + size_t _next; }; using TargetMap = std::map<string, Entry>; - using LockGuard = std::lock_guard<std::mutex>; std::mutex _lock; TargetMap _targets; ITimer::UP _timer; uint64_t _expireMillis; + size_t _numTargetsPerSpec; public: RPCTargetPool(const RPCTargetPool &) = delete; @@ -46,7 +52,7 @@ public: * @param expireSecs The number of seconds until an idle connection is * closed. */ - RPCTargetPool(double expireSecs); + RPCTargetPool(double expireSecs, size_t numTargetsPerSpec); /** * Constructs a new instance of this class, using the given {@link Timer} @@ -57,7 +63,7 @@ public: * @param expireSecs The number of seconds until an idle connection is * closed. */ - RPCTargetPool(ITimer::UP timer, double expireSecs); + RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec); /** * Destructor. Frees any allocated resources. diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index a7579aeb2ea..19b545df616 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -60,6 +60,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-core</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java index 58ff1af4681..75069fa815e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java @@ -25,7 +25,7 @@ public interface ContainerEngine { /** Start a created container */ void startContainer(NodeAgentContext context); - /** Update an existing container */ + /** Update an existing container with new resources */ void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources); /** Remove given container. The container will be stopped if necessary */ @@ -34,25 +34,25 @@ public interface ContainerEngine { /** Get container for given context */ Optional<Container> getContainer(NodeAgentContext context); - /** List containers managed by this */ + /** Returns all containers known by this */ List<PartialContainer> listContainers(TaskContext context); /** Returns the network interface used by container in given context */ String networkInterface(NodeAgentContext context); - /** Executes a command in inside container as root user, throws on non-zero exit code */ + /** Execute command inside container as root. Ignores non-zero exit code */ CommandResult executeAsRoot(NodeAgentContext context, Duration timeout, String... command); - /** Executes a command in inside containers network namespace, throws on non-zero exit code */ + /** Execute command inside the container's network namespace. Throws on non-zero exit code */ CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command); - /** Download giving image */ + /** Download given image */ void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials); /** Returns whether given image is already downloaded */ boolean hasImage(TaskContext context, DockerImage image); - /** Remove given image */ + /** Remove image by id */ void removeImage(TaskContext context, String id); /** Returns images available in this */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 444e92e793f..49216ddfdaf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -70,17 +70,16 @@ public class ContainerOperations { return executeCommandInContainerAsRoot(context, CommandLine.DEFAULT_TIMEOUT.toSeconds(), command); } - /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */ + /** Execute command inside container identified by given context. Does NOT throw on non-zero exit code */ public CommandResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) { return containerEngine.executeAsRoot(context, Duration.ofSeconds(timeoutSeconds), command); } - /** Executes a command in inside containers network namespace, throws on non-zero exit code */ + /** Execute command in inside containers network namespace, identified by given context. Throws on non-zero exit code */ public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) { return containerEngine.executeInNetworkNamespace(context, command); } - /** Resume node. Resuming a node means that it is ready to receive traffic */ public String resumeNode(NodeAgentContext context) { return executeNodeCtlInContainer(context, "resume"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java index c3be670c009..f960201b2a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java @@ -104,8 +104,8 @@ public class PartialContainer { stopped, paused, exited, - removing; - + removing, + stopping; public boolean isRunning() { return this == running; @@ -121,6 +121,7 @@ public class PartialContainer { case "paused": return paused; case "exited": return exited; case "removing": return removing; + case "stopping": return stopping; } throw new IllegalArgumentException("Invalid state '" + state + "'"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java index c65209698f6..066114dc41f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java @@ -13,6 +13,8 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Download a container image asynchronously. @@ -21,6 +23,8 @@ import java.util.concurrent.Executors; */ public class ContainerImageDownloader { + private static final Logger LOG = Logger.getLogger(ContainerImageDownloader.class.getName()); + private final ContainerEngine containerEngine; private final ExecutorService executorService = Executors.newSingleThreadExecutor( @@ -42,6 +46,8 @@ public class ContainerImageDownloader { executorService.submit(() -> { try { containerEngine.pullImage(context, image, registryCredentials); + } catch (RuntimeException e) { + LOG.log(Level.SEVERE, "Failed to download container image " + image, e); } finally { pendingDownloads.remove(image); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java index bb33dfee663..37601055d00 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java @@ -40,11 +40,10 @@ import java.util.stream.Stream; * <li>Deleting a tag of an image with multiple tags will only remove the tag, the image with the * remaining tags will remain</li> * <li>Deleting the last tag of an image will delete the entire image.</li> - * <li>Image cannot be deleted if:</li> - * <ol> - * <li>It has 1 or more children</li> - * <li>A container uses it</li> - * </ol> + * <li>Image cannot be deleted if: + * <p>- It has 1 or more children + * <p>- A container uses it + * </li> * </ol> * * @author freva diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java index 2a01a5ebcb4..ba9ba80ccb7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; @@ -57,16 +56,6 @@ public abstract class YumCommand<T extends YumCommand<T>> { public abstract boolean converge(TaskContext context); - /** Returns the version of Yum itself */ - protected final Version version(TaskContext context) { - return terminal.newCommandLine(context).add("yum", "--version") - .executeSilently() - .getOutputLinesStream() - .findFirst() - .map(Version::fromString).orElseThrow(() -> new IllegalStateException("Failed to detect Yum version")); - } - - public static class GenericYumCommand extends YumCommand<GenericYumCommand> { private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$"); @@ -111,11 +100,10 @@ public abstract class YumCommand<T extends YumCommand<T>> { if (yumCommand == CommandType.remove) if (packages.stream().noneMatch(pkg -> isInstalled(context, pkg))) return false; - Version yumVersion = version(context); CommandLine commandLine = terminal.newCommandLine(context); commandLine.add("yum", yumCommand.name()); addParametersToCommandLine(commandLine); - commandLine.add(packages.stream().map(pkg -> pkg.toName(yumVersion)).collect(Collectors.toList())); + commandLine.add(packages.stream().map(pkg -> pkg.toName()).collect(Collectors.toList())); // There's no way to figure out whether a yum command would have been a no-op. // Therefore, run the command and parse the output to decide. @@ -168,15 +156,10 @@ public abstract class YumCommand<T extends YumCommand<T>> { @Override public boolean converge(TaskContext context) { - Version yumVersion = version(context); - String targetVersionLockName = yumPackage.toVersionLockName(yumVersion); + String targetVersionLockName = yumPackage.toVersionLockName(); List<String> command = new ArrayList<>(4); command.add("yum"); - // Using --quiet on Yum 4 always results in an empty list, even if locks exist... - if (yumVersion.getMajor() < 4) { - command.add("--quiet"); - } command.add("versionlock"); command.add("list"); @@ -193,7 +176,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { if (packageName.getName().equals(yumPackage.getName())) { // If existing lock doesn't exactly match the full package name, // it means it's locked to another version and we must remove that lock. - String versionLockName = packageName.toVersionLockName(yumVersion); + String versionLockName = packageName.toVersionLockName(); if (versionLockName.equals(targetVersionLockName)) { return true; } else { @@ -231,7 +214,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { var installCommand = terminal.newCommandLine(context).add("yum", "install"); addParametersToCommandLine(installCommand); - installCommand.add(yumPackage.toName(yumVersion)); + installCommand.add(yumPackage.toName()); String output = installCommand.executeSilently().getUntrimmedOutput(); @@ -240,7 +223,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { // case 3. var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade"); addParametersToCommandLine(upgradeCommand); - upgradeCommand.add(yumPackage.toName(yumVersion)).execute(); + upgradeCommand.add(yumPackage.toName()).execute(); modified = true; } else { // case 2. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java index f3fa6bc65e4..5b32f50ad49 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.yum; import com.google.common.base.Strings; -import com.yahoo.component.Version; import java.util.Arrays; import java.util.Objects; @@ -12,7 +11,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * YUM package name. + * A YUM/DNF package name. * * <p>From yum(8): YUM package names are used with install, update, remove, list, info etc * with any of the following as well as globs of any of the following, with any of the @@ -42,9 +41,10 @@ public class YumPackageName { Arrays.stream(Architecture.values()).map(Architecture::name).collect(Collectors.joining("|")); private static final Pattern ARCHITECTURE_PATTERN = Pattern.compile("\\.(" + ARCHITECTURES_OR + "|\\*)$"); private static final Pattern EPOCH_PATTERN = Pattern.compile("^((.+)-)?([0-9]+)$"); - private static final Pattern NAME_VER_REL_PATTERN = Pattern.compile("^((.+)-)?" + - "([+()a-z0-9._]*[0-9][a-z0-9._]*)-" + // ver contains at least one digit - "([+()a-z0-9._]*[0-9][a-z0-9._]*)$"); // rel contains at least one digit + private static final Pattern NAME_VER_REL_PATTERN = + Pattern.compile("^(.+?)?" + // name + "-([+()a-z0-9._]*[0-9]+\\.[a-z0-9._]*)" + // ver: contains at least one digit and dot + "(?:-([+()a-z0-9._]*[0-9][a-z0-9._]*))?$"); // rel: contains at least one digit private static final Pattern NAME_PATTERN = Pattern.compile("^[+()a-zA-Z0-9._-]+$"); private final Optional<String> epoch; @@ -74,15 +74,6 @@ public class YumPackageName { architecture = packageName.architecture; } - /** - * Set the epoch of the YUM package. - * - * <p>WARNING: Should only be invoked if the YUM package actually has an epoch. Typically - * YUM packages doesn't have one explicitly set, and in case "0" will be used with - * {@link #toVersionLockName(Version)} (otherwise it fails), but it will be absent from an - * install with {@link #toName(Version)} (otherwise it fails). This typically means that - * you should set this only if the epoch is != "0".</p> - */ public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; } public Builder setName(String name) { this.name = name; return this; } public Builder setVersion(String version) { this.version = Optional.of(version); return this; } @@ -100,7 +91,6 @@ public class YumPackageName { Optional<String> architecture) { if (Strings.isNullOrEmpty(name)) throw new IllegalArgumentException("name cannot be null or empty"); - this.epoch = epoch; this.name = name; this.version = version; @@ -116,6 +106,7 @@ public class YumPackageName { * <ol> * <li>name * <li>name.arch + * <li>name-ver * <li>name-ver-rel * <li>name-ver-rel.arch * <li>name-epoch:ver-rel.arch @@ -125,19 +116,12 @@ public class YumPackageName { * @throws IllegalArgumentException if spec does not specify a package name. * @see #parseString(String) */ - public static YumPackageName fromString(final String packageSpec) { + public static YumPackageName fromString(String packageSpec) { String spec = packageSpec; Optional<String> epoch = Optional.empty(); String name = null; - // packageSpec spec - // name name - // name.arch name.arch - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel.arch - // name-epoch:ver-rel.arch name-epoch:ver-rel.arch - // epoch:name-ver-rel.arch epoch:name-ver-rel.arch - + // Parse epoch and remove it from spec int epochColon = spec.indexOf(':'); if (epochColon >= 0) { Matcher epochMatcher = EPOCH_PATTERN.matcher(spec.substring(0, epochColon)); @@ -148,17 +132,12 @@ public class YumPackageName { name = epochMatcher.group(2); epoch = Optional.of(epochMatcher.group(3)); - spec = spec.substring(epochColon + 1); + spec = epochColon == 0 + ? spec.substring(epochColon + 1) + : spec.substring(0, epochColon - 1) + spec.substring(epochColon + 1); } - // packageSpec spec - // name name - // name.arch name.arch - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel.arch - // name-epoch:ver-rel.arch ver-rel.arch (non-null name) - // epoch:name-ver-rel.arch name-ver-rel.arch - + // Parse architecture and remove it from spec Optional<String> architecture = Optional.empty(); Matcher architectureMatcher = ARCHITECTURE_PATTERN.matcher(spec); if (architectureMatcher.find()) { @@ -166,44 +145,27 @@ public class YumPackageName { spec = spec.substring(0, architectureMatcher.start()); } - // packageSpec spec - // name name - // name.arch name - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel - // name-epoch:ver-rel.arch ver-rel (non-null name) - // epoch:name-ver-rel.arch name-ver-rel - + // Parse name, version and release and remove the latter two from spec Optional<String> version = Optional.empty(); Optional<String> release = Optional.empty(); Matcher matcher = NAME_VER_REL_PATTERN.matcher(spec); if (matcher.find()) { - // spec format one of: - // 1. name-ver-rel - // 2. ver-rel - - spec = matcher.group(2); + spec = matcher.group(1); if (spec == null) { if (name == null) { throw new IllegalArgumentException("No package name was found: " + packageSpec); } spec = name; // makes spec hold the package name in all cases below. - } else if (name != null) { - throw new IllegalArgumentException("Ambiguous package names were found for " + - packageSpec + ": '" + name + "' and '" + spec + "'"); } - version = Optional.of(matcher.group(3)); - release = Optional.of(matcher.group(4)); + version = Optional.of(matcher.group(2)); + release = Optional.ofNullable(matcher.group(3)); } - // packageSpec spec - // name name - // name.arch name - // name-ver-rel name - // name-ver-rel.arch name - // name-epoch:ver-rel.arch name - // epoch:name-ver-rel.arch name + // Set default epoch if we have a version + if (version.isPresent() && epoch.isEmpty()) { + epoch = Optional.of("0"); + } if (!NAME_PATTERN.matcher(spec).find()) { throw new IllegalArgumentException("Bad package name in " + packageSpec + ": '" + spec + "'"); @@ -229,21 +191,14 @@ public class YumPackageName { public Optional<String> getArchitecture() { return architecture; } /** Return package name, omitting components that are not specified. */ - public String toName(Version yumVersion) { + public String toName() { StringBuilder builder = new StringBuilder(); - boolean isBare = version.isEmpty() && release.isEmpty() && architecture.isEmpty(); char nextDelimiter; - if (yumVersion.getMajor() < 4) { - epoch.ifPresent(ep -> builder.append(ep).append(':')); - builder.append(name); - nextDelimiter = '-'; - } else { - builder.append(name); - // Fully versioned package names must always include epoch in Yum 4 - epoch.or(() -> Optional.of("0").filter(v -> !isBare)) - .ifPresent(ep -> builder.append('-').append(ep)); - nextDelimiter = ':'; - } + builder.append(name); + // Fully versioned package names must always include epoch in Yum 4 + epoch.or(() -> Optional.of("0").filter(v -> version.isPresent())) + .ifPresent(ep -> builder.append('-').append(ep)); + nextDelimiter = ':'; version.ifPresent(s -> builder.append(nextDelimiter).append(s)); release.ifPresent(s -> builder.append('-').append(s)); architecture.ifPresent(arch -> builder.append('.').append(arch)); @@ -255,7 +210,7 @@ public class YumPackageName { * * @throws IllegalStateException if any field required for the version lock spec is missing */ - public String toVersionLockName(Version yumVersion) { + public String toVersionLockName() { Builder b = new Builder(this).setArchitecture("*"); if (epoch.isEmpty()) { b.setEpoch("0"); @@ -263,7 +218,7 @@ public class YumPackageName { YumPackageName lockSpec = b.build(); if (lockSpec.getVersion().isEmpty()) throw new IllegalStateException("Version is missing for YUM package " + name); if (lockSpec.getRelease().isEmpty()) throw new IllegalStateException("Release is missing for YUM package " + name); - return lockSpec.toName(yumVersion); + return lockSpec.toName(); } public boolean isSubsetOf(YumPackageName other) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java index e47d71cbdf7..e48a0fa2683 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; @@ -17,20 +16,10 @@ import java.util.stream.Stream; public class YumTester extends Yum { private final TestTerminal terminal; - private final Version yumVersion; public YumTester(TestTerminal terminal) { - this(terminal, YumVersion.rhel7); - } - - public YumTester(TestTerminal terminal, YumVersion yumVersion) { super(terminal); this.terminal = terminal; - this.yumVersion = yumVersion.asVersion(); - } - - public Version yumVersion() { - return yumVersion; } public GenericYumCommandExpectation expectInstall(String... packages) { @@ -81,10 +70,6 @@ public class YumTester extends Yum { } } - protected void expectYumVersion() { - terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n"); - } - private YumTester execute(String output) { if (commandType == CommandType.install) terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(1, "Not installed")); @@ -106,7 +91,7 @@ public class YumTester extends Yum { if (commandType == CommandType.upgrade && packages.size() > 1) cmd.append(" --setopt skip_missing_names_on_update=False"); packages.forEach(pkg -> { - String name = pkg.toName(yumVersion); + String name = pkg.toName(); if (name.contains("(") || name.contains(")")) { // Ugly hack to handle implicit quoting done in com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine name = "\"" + name + "\""; } @@ -114,27 +99,20 @@ public class YumTester extends Yum { }); cmd.append(" 2>&1"); - expectYumVersion(); terminal.expectCommand(cmd.toString(), 0, output); return YumTester.this; } } public class InstallFixedCommandExpectation extends GenericYumCommandExpectation { + private InstallFixedCommandExpectation(String yumPackage) { super(CommandType.installFixed, yumPackage); } @Override - protected void expectYumVersion() {} - - @Override public YumTester andReturn(boolean value) { - // Pretend package is already correctly version locked to simplify expectations - terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n"); - - String quiet = yumVersion.getMajor() < 4 ? " --quiet" : ""; - terminal.expectCommand("yum" + quiet +" versionlock list 2>&1", 0, packages.get(0).toVersionLockName(yumVersion)); + terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName()); return super.andReturn(value); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java deleted file mode 100644 index b0c2805d620..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.component.Version; - -/** - * Red Hat versions and their associated Yum major version. - * - * @author mpolden - */ -public enum YumVersion { - - rhel7(3), - rhel8(4); - - private final Version version; - - YumVersion(int yumMajor) { - this.version = new Version(yumMajor, 0, 0); - } - - public Version asVersion() { - return version; - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 4e2a5052ea6..1d077449ed6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import java.time.Duration; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -19,19 +18,12 @@ import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; /** - * A mock implementation of {@link ContainerEngine}. - * - * Container operations are multi-thread safe. Note that this is a requirement since tests may use this through a real - * (multi-threaded) node-admin instance. - * * @author mpolden */ public class ContainerEngineMock implements ContainerEngine { - private final Map<ContainerName, Container> containers = new HashMap<>(); + private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); - private final Object monitor = new Object(); - private boolean asyncImageDownload = false; public ContainerEngineMock asyncImageDownload(boolean enabled) { @@ -58,72 +50,56 @@ public class ContainerEngineMock implements ContainerEngine { } public ContainerEngineMock addContainers(List<Container> containers) { - synchronized (monitor) { - for (var container : containers) { - if (this.containers.containsKey(container.name())) { - throw new IllegalArgumentException("Container " + container.name() + " already exists"); - } - this.containers.put(container.name(), container); + for (var container : containers) { + if (this.containers.containsKey(container.name())) { + throw new IllegalArgumentException("Container " + container.name() + " already exists"); } - return this; + this.containers.put(container.name(), container); } + return this; } public ContainerEngineMock addContainer(Container container) { - synchronized (monitor) { - return addContainers(List.of(container)); - } + return addContainers(List.of(container)); } @Override public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { - synchronized (monitor) { - addContainer(createContainer(context, PartialContainer.State.created, containerResources)); - } + addContainer(createContainer(context, PartialContainer.State.created, containerResources)); } @Override public void startContainer(NodeAgentContext context) { - synchronized (monitor) { - Container container = requireContainer(context.containerName(), PartialContainer.State.created); - Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); - containers.put(newContainer.name(), newContainer); - } + Container container = requireContainer(context.containerName(), PartialContainer.State.created); + Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); + containers.put(newContainer.name(), newContainer); } @Override public void removeContainer(TaskContext context, PartialContainer container) { - synchronized (monitor) { - requireContainer(container.name()); - containers.remove(container.name()); - } + requireContainer(container.name()); + containers.remove(container.name()); } @Override public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { - synchronized (monitor) { - Container container = requireContainer(context.containerName()); - containers.put(container.name(), new Container(containerId, container.name(), container.state(), - container.imageId(), container.image(), - container.labels(), container.pid(), - container.conmonPid(), container.hostname(), - containerResources, container.networks(), - container.managed())); - } + Container container = requireContainer(context.containerName()); + containers.put(container.name(), new Container(containerId, container.name(), container.state(), + container.imageId(), container.image(), + container.labels(), container.pid(), + container.conmonPid(), container.hostname(), + containerResources, container.networks(), + container.managed())); } @Override public Optional<Container> getContainer(NodeAgentContext context) { - synchronized (monitor) { - return Optional.ofNullable(containers.get(context.containerName())); - } + return Optional.ofNullable(containers.get(context.containerName())); } @Override public List<PartialContainer> listContainers(TaskContext context) { - synchronized (monitor) { - return List.copyOf(containers.values()); - } + return List.copyOf(containers.values()); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java index 04d86a69057..898d7ebe901 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -8,6 +8,8 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import org.junit.Test; +import java.util.List; + import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; @@ -20,8 +22,8 @@ public class ContainerFailTest { @Test public void test() { - try (ContainerTester tester = new ContainerTester()) { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage"); + DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage"); + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { ContainerName containerName = new ContainerName("host1"); String hostname = "host1.test.yahoo.com"; NodeSpec nodeSpec = NodeSpec.Builder diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 4179f53370b..3f2083638dc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integration; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -28,6 +29,7 @@ import java.nio.file.FileSystem; import java.time.Clock; import java.time.Duration; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.logging.Logger; @@ -50,7 +52,8 @@ public class ContainerTester implements AutoCloseable { private final Thread loopThread; - final ContainerOperations containerOperations = spy(new ContainerOperations(new ContainerEngineMock(), TestFileSystem.create())); + private final ContainerEngineMock containerEngine = new ContainerEngineMock(); + final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, TestFileSystem.create())); final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); final Orchestrator orchestrator = mock(Orchestrator.class); final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); @@ -64,7 +67,8 @@ public class ContainerTester implements AutoCloseable { private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED; - ContainerTester() { + ContainerTester(List<DockerImage> images) { + images.forEach(image -> containerEngine.pullImage(null, image, RegistryCredentials.none)); when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); IPAddressesMock ipAddresses = new IPAddressesMock(); @@ -110,6 +114,12 @@ public class ContainerTester implements AutoCloseable { /** Adds a node to node-repository mock that is running on this host */ void addChildNodeRepositoryNode(NodeSpec nodeSpec) { + if (nodeSpec.wantedDockerImage().isPresent()) { + if (!containerEngine.hasImage(null, nodeSpec.wantedDockerImage().get())) { + throw new IllegalArgumentException("Want to use image " + nodeSpec.wantedDockerImage().get() + + ", but that image does not exist in the container engine"); + } + } nodeRepository.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec) .parentHostname(HOST_HOSTNAME.value()) .build()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java index 86db3ae092e..fdfb3457330 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.hosted.node.admin.integration; import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import org.junit.Test; +import java.util.List; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -19,11 +21,12 @@ public class MultiContainerTest { @Test public void test() { - try (ContainerTester tester = new ContainerTester()) { - DockerImage image1 = DockerImage.fromString("registry.example.com/image1"); + DockerImage image1 = DockerImage.fromString("registry.example.com/image1"); + DockerImage image2 = DockerImage.fromString("registry.example.com/image2"); + try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { addAndWaitForNode(tester, "host1.test.yahoo.com", image1); NodeSpec nodeSpec2 = addAndWaitForNode( - tester, "host2.test.yahoo.com", DockerImage.fromString("registry.example.com/image2")); + tester, "host2.test.yahoo.com", image2); tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java index dad02f46d88..109fb61d0c9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -27,7 +27,7 @@ public class RebootTest { @Test public void test() { - try (ContainerTester tester = new ContainerTester()) { + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build()); ContainerName host1 = new ContainerName("host1"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java index 160948d7996..b848a5a91d9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import org.junit.Test; +import java.util.List; + import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -20,10 +22,9 @@ public class RestartTest { @Test public void test() { - try (ContainerTester tester = new ContainerTester()) { + DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage:1.2.3"); + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { String hostname = "host1.test.yahoo.com"; - DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage:1.2.3"); - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build(); tester.addChildNodeRepositoryNode(nodeSpec); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java index 0880640b84d..a4dc3ac359a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import org.junit.Test; import java.util.Optional; @@ -26,8 +25,7 @@ public class YumPackageNameTest { .setRelease("71.git3e8e77d.el7.centos.1") .setArchitecture("x86_64") .build(); - assertEquals("2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName(Version.fromString("3"))); - assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName(Version.fromString("4"))); + assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName()); } @Test @@ -65,25 +63,35 @@ public class YumPackageNameTest { "docker-engine-selinux.x86_64", null); + // name-ver + verifyPackageName("docker-engine-selinux-1.12.6", + "0", + "docker-engine-selinux", + "1.12.6", + null, + null, + "docker-engine-selinux-0:1.12.6", + null); + // name-ver-rel verifyPackageName("docker-engine-selinux-1.12.6-1.el7", - null, + "0", "docker-engine-selinux", "1.12.6", "1.el7", null, - "docker-engine-selinux-1.12.6-1.el7", - "0:docker-engine-selinux-1.12.6-1.el7.*"); + "docker-engine-selinux-0:1.12.6-1.el7", + "docker-engine-selinux-0:1.12.6-1.el7.*"); // name-ver-rel.arch verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", - null, + "0", "docker-engine-selinux", "1.12.6", "1.el7", "x86_64", - "docker-engine-selinux-1.12.6-1.el7.x86_64", - "0:docker-engine-selinux-1.12.6-1.el7.*"); + "docker-engine-selinux-0:1.12.6-1.el7.x86_64", + "docker-engine-selinux-0:1.12.6-1.el7.*"); // name-epoch:ver-rel.arch verifyPackageName( @@ -93,8 +101,8 @@ public class YumPackageNameTest { "1.12.6", "71.git3e8e77d.el7.centos.1", "x86_64", - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.*"); + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); // epoch:name-ver-rel.arch verifyPackageName( @@ -104,22 +112,11 @@ public class YumPackageNameTest { "1.12.6", "71.git3e8e77d.el7.centos.1", "x86_64", - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.*"); - - // name-ver-rel.arch (RHEL 8) - verifyPackageName("podman-1.9.3-2.module+el8.2.1+6867+366c07d6.x86_64", - null, - "podman", - "1.9.3", - "2.module+el8.2.1+6867+366c07d6", - "x86_64", - "podman-0:1.9.3-2.module+el8.2.1+6867+366c07d6.x86_64", - "podman-0:1.9.3-2.module+el8.2.1+6867+366c07d6.*", - YumVersion.rhel8); + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); } - private void verifyPackageName(String packageName, + private void verifyPackageName(String input, String epoch, String name, String version, @@ -127,43 +124,33 @@ public class YumPackageNameTest { String architecture, String toName, String toVersionName) { - verifyPackageName(packageName, epoch, name, version, release, architecture, toName, toVersionName, YumVersion.rhel7); - } - - private void verifyPackageName(String packageName, - String epoch, - String name, - String version, - String release, - String architecture, - String toName, - String toVersionName, - YumVersion yumVersion) { - YumPackageName yumPackageName = YumPackageName.fromString(packageName); - verifyValue(epoch, yumPackageName.getEpoch()); - verifyValue(name, Optional.of(yumPackageName.getName())); - verifyValue(version, yumPackageName.getVersion()); - verifyValue(release, yumPackageName.getRelease()); - verifyValue(architecture, yumPackageName.getArchitecture()); - verifyValue(toName, Optional.of(yumPackageName.toName(yumVersion.asVersion()))); + YumPackageName yumPackageName = YumPackageName.fromString(input); + assertPackageField("epoch", epoch, yumPackageName.getEpoch()); + assertPackageField("name", name, Optional.of(yumPackageName.getName())); + assertPackageField("version", version, yumPackageName.getVersion()); + assertPackageField("release", release, yumPackageName.getRelease()); + assertPackageField("architecture", architecture, yumPackageName.getArchitecture()); + assertPackageField("toName()", toName, Optional.of(yumPackageName.toName())); if (toVersionName == null) { try { - yumPackageName.toVersionLockName(yumVersion.asVersion()); + yumPackageName.toVersionLockName(); fail(); } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsStringIgnoringCase("Version is missing ")); + assertTrue("Exception message contains expected substring: " + e.getMessage(), + e.getMessage().contains("Version is missing ") || + e.getMessage().contains("Release is missing ")); } } else { - assertEquals(toVersionName, yumPackageName.toVersionLockName(yumVersion.asVersion())); + assertEquals(toVersionName, yumPackageName.toVersionLockName()); } } - private void verifyValue(String value, Optional<String> actual) { - if (value == null) { - assertFalse(actual.isPresent()); + private void assertPackageField(String field, String expected, Optional<String> actual) { + if (expected == null) { + assertFalse(field + " is not present", actual.isPresent()); } else { - assertEquals(value, actual.get()); + assertEquals(field + " has expected value", expected, actual.get()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java index 92f8f78d255..62fd7410e95 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -79,20 +79,6 @@ public class YumTest { @Test public void testAlreadyInstalled() { mockRpmQuery("package-1", null); - mockYumVersion(); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do\n"); - - assertFalse(yum - .install("package-1", "package-2") - .enableRepo("repo1", "repo2") - .converge(taskContext)); - - // RHEL 8 - mockRpmQuery("package-1", null); - mockYumVersion(YumVersion.rhel8); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -104,18 +90,6 @@ public class YumTest { @Test public void testAlreadyUpgraded() { - mockYumVersion(); - terminal.expectCommand( - "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", - 0, - "foobar\nNo packages marked for update\n"); - - assertFalse(yum - .upgrade("package-1", "package-2") - .converge(taskContext)); - - // RHEL 8 - mockYumVersion(YumVersion.rhel8); terminal.expectCommand( "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", 0, @@ -128,19 +102,6 @@ public class YumTest { @Test public void testAlreadyRemoved() { mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockYumVersion(); - terminal.expectCommand( - "yum remove --assumeyes package-1 package-2 2>&1", - 0, - "foobar\nNo Packages marked for removal\n"); - - assertFalse(yum - .remove("package-1", "package-2") - .converge(taskContext)); - - // RHEL 8 - mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockYumVersion(YumVersion.rhel8); terminal.expectCommand( "yum remove --assumeyes package-1 package-2 2>&1", 0, @@ -160,7 +121,6 @@ public class YumTest { @Test public void testInstall() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -181,7 +141,6 @@ public class YumTest { @Test public void testInstallWithEnablerepo() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -195,23 +154,6 @@ public class YumTest { @Test public void testWithVersionLock() { - mockYumVersion(); - terminal.expectCommand("yum --quiet versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "0:chef-12.21.1-1.el7.*\n"); - terminal.expectCommand("yum versionlock add --assumeyes \"0:package-1-0.10-654.el7.*\" 2>&1"); - terminal.expectCommand( - "yum install --assumeyes 0:package-1-0.10-654.el7.x86_64 2>&1", - 0, - "installing"); - - assertTrue(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7.x86_64")).converge(taskContext)); - } - - @Test - public void testWithVersionLockYum4() { - mockYumVersion(YumVersion.rhel8); terminal.expectCommand("yum versionlock list 2>&1", 0, "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n"); @@ -232,69 +174,65 @@ public class YumTest { @Test public void testWithDifferentVersionLock() { - mockYumVersion(); - terminal.expectCommand("yum --quiet versionlock list 2>&1", + terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + - "0:chef-12.21.1-1.el7.*\n" + - "0:package-1-0.1-8.el7.*\n"); + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.1-8.el7.*\n"); - terminal.expectCommand("yum versionlock delete \"0:package-1-0.1-8.el7.*\" 2>&1"); + terminal.expectCommand("yum versionlock delete \"package-0:0.1-8.el7.*\" 2>&1"); - terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"0:package-1-0.10-654.el7.*\" 2>&1"); + terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"package-0:0.10-654.el7.*\" 2>&1"); terminal.expectCommand( - "yum install --assumeyes --enablerepo=somerepo 0:package-1-0.10-654.el7 2>&1", + "yum install --assumeyes --enablerepo=somerepo package-0:0.10-654.el7 2>&1", 0, "Nothing to do\n"); assertTrue(yum - .installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7")) + .installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")) .enableRepo("somerepo") .converge(taskContext)); } @Test public void testWithExistingVersionLock() { - mockYumVersion(); - terminal.expectCommand("yum --quiet versionlock list 2>&1", + terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + - "0:chef-12.21.1-1.el7.*\n" + - "0:package-1-0.10-654.el7.*\n"); + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.10-654.el7.*\n"); terminal.expectCommand( - "yum install --assumeyes 0:package-1-0.10-654.el7 2>&1", + "yum install --assumeyes package-0:0.10-654.el7 2>&1", 0, "Nothing to do\n"); - assertFalse(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7")).converge(taskContext)); + assertFalse(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); } @Test public void testWithDowngrade() { - mockYumVersion(); - terminal.expectCommand("yum --quiet versionlock list 2>&1", + terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + - "0:chef-12.21.1-1.el7.*\n" + - "0:package-1-0.10-654.el7.*\n"); + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.10-654.el7.*\n"); terminal.expectCommand( - "yum install --assumeyes 0:package-1-0.10-654.el7 2>&1", + "yum install --assumeyes package-0:0.10-654.el7 2>&1", 0, - "Package matching package-1-0.10-654.el7 already installed. Checking for update.\n" + + "Package matching package-=.0.10-654.el7 already installed. Checking for update.\n" + "Nothing to do\n"); - terminal.expectCommand("yum downgrade --assumeyes 0:package-1-0.10-654.el7 2>&1"); + terminal.expectCommand("yum downgrade --assumeyes package-0:0.10-654.el7 2>&1"); - assertTrue(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7")).converge(taskContext)); + assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); } @Test(expected = ChildProcessFailureException.class) public void testFailedInstall() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 1, @@ -310,7 +248,6 @@ public class YumTest { @Test public void testUnknownPackages() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1", 0, @@ -337,19 +274,10 @@ public class YumTest { @Test public void allowToCallUpgradeWithNoPackages() { - mockYumVersion(); terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); yum.upgrade().converge(taskContext); } - private void mockYumVersion(YumVersion yumVersion) { - terminal.expectCommand("yum --version 2>&1", 0, yumVersion.asVersion().toFullString() + "\ntrailing garbage\n"); - } - - private void mockYumVersion() { - mockYumVersion(YumVersion.rhel7); - } - private void mockRpmQuery(String packageName, YumPackageName installedOrNull) { new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java index bb4bb2686a3..93fffffb2fa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java @@ -37,7 +37,7 @@ public class YumTesterTest { assertYumMethod(yum -> yum.expectRemove(packages).withEnableRepo(repos), yum -> yum.remove(List.of(packages)).enableRepo(repos).converge(context)); - assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName(yum.yumVersion())).withEnableRepo(repos), + assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withEnableRepo(repos), yum -> yum.installFixedVersion(minimalPackage).enableRepo(repos).converge(context)); } diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 30aa76658fd..e7fdc560bc5 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -26,6 +26,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -79,10 +85,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> <version>${project.version}</version> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index b465917e9c9..b799e0056f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -107,8 +107,13 @@ public class AllocatableClusterResources { } public boolean preferableTo(AllocatableClusterResources other) { - if (this.fulfilment < 1 || other.fulfilment < 1) - return this.fulfilment > other.fulfilment; // we always want to fulfil as much as possible + if (this.fulfilment < 1 || other.fulfilment < 1) // always fulfil as much as possible + return this.fulfilment > other.fulfilment; + + if (clusterSpec.type().isContent() // always prefer local storage on content nodes + && this.realResources.storageType() != other.realResources().nodeResources().storageType()) + return this.realResources.storageType() == NodeResources.StorageType.local; + return this.cost() < other.cost(); // otherwise, prefer lower cost } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 8d97e14fc7c..3a655062f7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.io.IOUtils; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import io.questdb.cairo.CairoEngine; import io.questdb.cairo.CairoException; import io.questdb.cairo.DefaultCairoConfiguration; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -56,9 +58,9 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private final Clock clock; private final String dataDir; private final CairoEngine engine; - private final ThreadLocal<SqlCompiler> sqlCompiler; + private final ConcurrentResourcePool<SqlCompiler> sqlCompilerPool; + private final AtomicBoolean closed = new AtomicBoolean(false); - private volatile int nullRecords = 0; @Inject public QuestMetricsDb() { @@ -80,12 +82,18 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { this.dataDir = dataDir; engine = new CairoEngine(new DefaultCairoConfiguration(dataDir)); - sqlCompiler = ThreadLocal.withInitial(() -> new SqlCompiler(engine)); + sqlCompilerPool = new ConcurrentResourcePool<>(() -> new SqlCompiler(engine())); nodeTable = new Table(dataDir, "metrics", clock); clusterTable = new Table(dataDir, "clusterMetrics", clock); ensureTablesExist(); } + private CairoEngine engine() { + if (closed.get()) + throw new IllegalStateException("Attempted to access QuestDb after calling close"); + return engine; + } + @Override public Clock clock() { return clock; } @@ -182,11 +190,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } } - public int getNullRecordsCount() { return nullRecords; } - @Override public void gc() { - nullRecords = 0; nodeTable.gc(); clusterTable.gc(); } @@ -196,8 +201,14 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { @Override public void close() { - if (engine != null) - engine.close(); + if (closed.getAndSet(true)) return; + synchronized (nodeTable.writeLock) { + synchronized (clusterTable.writeLock) { + for (SqlCompiler sqlCompiler : sqlCompilerPool) + sqlCompiler.close(); + engine.close(); + } + } } private void ensureTablesExist() { @@ -222,7 +233,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private void ensureClusterTableIsUpdated() { try { - if (0 == engine.getStatus(newContext().getCairoSecurityContext(), new Path(), clusterTable.name)) { + if (0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), clusterTable.name)) { // Example: clusterTable.ensureColumnExists("write_rate", "float"); } } catch (Exception e) { @@ -277,10 +288,6 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { try (RecordCursor cursor = factory.getCursor(context)) { Record record = cursor.getRecord(); while (cursor.hasNext()) { - if (record == null || record.getStr(0) == null) { // Observed to happen. QuestDb bug? - nullRecords++; - continue; - } String hostname = record.getStr(0).toString(); if (hostnames.isEmpty() || hostnames.contains(hostname)) { snapshots.put(hostname, @@ -323,11 +330,19 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { /** Issues an SQL statement against the QuestDb engine */ private CompiledQuery issue(String sql, SqlExecutionContext context) throws SqlException { - return sqlCompiler.get().compile(sql, context); + SqlCompiler sqlCompiler = sqlCompilerPool.alloc(); + try { + return sqlCompiler.compile(sql, context); + } catch (SqlException e) { + log.log(Level.WARNING, "Could not execute SQL statement '" + sql + "'"); + throw e; + } finally { + sqlCompilerPool.free(sqlCompiler); + } } private SqlExecutionContext newContext() { - return new SqlExecutionContextImpl(engine, 1); + return new SqlExecutionContextImpl(engine(), 1); } /** A questDb table */ @@ -349,11 +364,11 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } boolean exists() { - return 0 == engine.getStatus(newContext().getCairoSecurityContext(), new Path(), name); + return 0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), name); } TableWriter getWriter() { - return engine.getWriter(newContext().getCairoSecurityContext(), name); + return engine().getWriter(newContext().getCairoSecurityContext(), name); } void gc() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index d671900d08c..4d16c90c002 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -53,9 +53,6 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { Thread.sleep(pauseMs); } - if (nodeRepository().metricsDb().getNullRecordsCount() > 0) - log.warning(nodeRepository().metricsDb().getNullRecordsCount() + " records returned null"); - nodeRepository().metricsDb().gc(); return asSuccessFactor(attempts, failures.get()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index fa5a72eea52..9beee666a74 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -54,7 +54,7 @@ public class CuratorDatabase { public CuratorDatabase(Curator curator, Path root, boolean useCache) { this.useCache = useCache; this.curator = curator; - changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter").getAbsolute()); + changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter")); cache.set(newCache(changeGenerationCounter.get())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index c37db2fa526..da4ab528630 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -82,7 +82,7 @@ public class CuratorDatabaseClient { this.nodeSerializer = new NodeSerializer(flavors, nodeCacheSize); this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; - this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); + this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter")); initZK(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 59556418fb5..2b64cc86c9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Locale; @@ -40,7 +41,10 @@ public class NodeResourceLimits { /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ public boolean isWithinRealLimits(NodeCandidate candidateNode, ClusterSpec cluster) { if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes - return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, cluster.isExclusive()), + // This node is allocated exclusively if that has been explicitly requested, or if the host of the node was + // provisioned exclusively + boolean exclusive = cluster.isExclusive() || candidateNode.parent.flatMap(Node::exclusiveTo).isPresent(); + return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, exclusive), cluster.type()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 761afed23cc..5c452989f56 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; @@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; -import org.junit.Ignore; import org.junit.Test; import java.time.Instant; @@ -150,7 +148,7 @@ public class DynamicAllocationTest { } @Test - public void test_allocation_balancing() { + public void allocation_balancing() { // Here we test balancing between cpu and memory and ignore disk ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); @@ -234,14 +232,12 @@ public class DynamicAllocationTest { fail("Two groups have been allocated to the same parent host"); } - @Ignore // TODO: Re-enable if we reintroduce spare capacity requirement @Test public void spare_capacity_used_only_when_replacement() { - // Use spare capacity only when replacement (i.e one node is failed) - // Test should allocate as much capacity as possible, verify that it is not possible to allocate one more unit - // Verify that there is still capacity (available spare) - // Fail one node and redeploy, Verify that one less node is empty. - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavorsConfig(flavorsConfig()) + .spareCount(2) + .build(); // Setup test ApplicationId application1 = ProvisioningTester.applicationId(); @@ -252,7 +248,7 @@ public class DynamicAllocationTest { // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1"); List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); assertEquals(2, initialSpareCapacity.size()); @@ -264,10 +260,9 @@ public class DynamicAllocationTest { tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> finalSpareCapacity = findSpareCapacity(tester); - assertEquals(1, finalSpareCapacity.size()); } @@ -278,7 +273,7 @@ public class DynamicAllocationTest { tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); assertEquals(0, initialSpareCapacity.size()); @@ -291,7 +286,7 @@ public class DynamicAllocationTest { tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); } @Test(expected = OutOfCapacityException.class) @@ -315,8 +310,8 @@ public class DynamicAllocationTest { tester.activate(application, hosts); NodeList activeNodes = tester.nodeRepository().nodes().list().owner(application); - assertEquals(ImmutableSet.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary()); - assertEquals(ImmutableSet.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary()); + assertEquals(Set.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary()); + assertEquals(Set.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary()); } @Test @@ -374,7 +369,7 @@ public class DynamicAllocationTest { } @Test - public void nodeResourcesAreRelaxedInDev() { + public void node_resources_are_relaxed_in_dev() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -394,7 +389,7 @@ public class DynamicAllocationTest { } @Test - public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { + public void switching_from_legacy_flavor_syntax_to_resources_does_not_cause_reallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); tester.activateTenantHosts(); @@ -413,7 +408,7 @@ public class DynamicAllocationTest { } @Test - public void testPreferExclusiveNetworkSwitch() { + public void prefer_exclusive_network_switch() { // Hosts are provisioned, without switch information ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); NodeResources hostResources = new NodeResources(32, 128, 2000, 10); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 6f1e2630434..0b7b9f2fa13 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -364,7 +364,7 @@ public class DynamicProvisioningTest { } @Test - public void test_any_disk_prefers_remote() { + public void test_any_disk_prefers_remote_for_container() { int memoryTax = 3; int localDiskTax = 55; // Disk tax is not included in flavor resources but memory tax is @@ -383,7 +383,7 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), resources(6, 3, 3, 25, 400, fast, StorageType.any))); @@ -392,6 +392,35 @@ public class DynamicProvisioningTest { app1, cluster1); } + @Test + public void test_any_disk_prefers_local_for_content() { + int memoryTax = 3; + int localDiskTax = 55; + // Disk tax is not included in flavor resources but memory tax is + List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, local)), + new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, local)), + new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), + new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote))); + + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + .flavors(flavors) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) + .nameResolver(nameResolver) + .resourcesCalculator(memoryTax, localDiskTax) + .build(); + + tester.activateTenantHosts(); + + ApplicationId app1 = ProvisioningTester.applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), + resources(6, 3, 3, 25, 400, fast, StorageType.any))); + tester.assertNodes("'any' selects a flavor with local storage", + 6, 2, 2, 20, 200, fast, local, + app1, cluster1); + } + private void prepareAndActivate(ApplicationId application, ClusterSpec clusterSpec, int nodes, int groups, NodeResources resources) { List<HostSpec> prepared = tester.prepare(application, clusterSpec, nodes, groups, resources); NodeList provisionedHosts = tester.nodeRepository().nodes().list(Node.State.provisioned).nodeType(NodeType.host); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java index 7ee65ebcd0b..467ee90176d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java @@ -1,6 +1,7 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; +import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; @@ -15,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong; * @author hakonhall */ public class HostInfosCache implements HostInfosService { - final static String HOST_STATUS_CACHE_COUNTER_PATH = "/vespa/host-status-service-cache-counter"; + final static Path HOST_STATUS_CACHE_COUNTER_PATH = Path.fromString("/vespa/host-status-service-cache-counter"); private final CuratorCounter counter; private final HostInfosService wrappedService; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index daafc81e0f3..8902bf1641c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.orchestrator.status; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; import com.yahoo.path.Path; @@ -54,16 +53,10 @@ public class ZkStatusService implements StatusService { private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>(); @Inject - public ZkStatusService( - @Component Curator curator, - @Component Metric metric, - @Component Timer timer, - @Component AntiServiceMonitor antiServiceMonitor) { - this(curator, - metric, - timer, - new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)), - antiServiceMonitor); + public ZkStatusService(Curator curator, Metric metric, Timer timer, AntiServiceMonitor antiServiceMonitor) { + this(curator, metric, timer, + new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)), + antiServiceMonitor); } /** Non-private for testing only. */ diff --git a/parent/pom.xml b/parent/pom.xml index 908491835ea..2a8527f80ea 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -863,6 +863,7 @@ --> <curator.version>5.1.0</curator.version> <jna.version>4.5.2</jna.version> + <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> <junit.version>5.7.0</junit.version> <maven-assembly-plugin.version>3.1.1</maven-assembly-plugin.version> @@ -61,13 +61,11 @@ <module>container-disc</module> <module>container-documentapi</module> <module>container-integration-test</module> - <module>container-jersey2</module> <module>container-messagebus</module> <module>container-search-and-docproc</module> <module>container-search</module> <module>container-search-gui</module> <module>container-test</module> - <module>container-test-jars</module> <module>controller-api</module> <module>controller-server</module> <module>defaults</module> diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 55b3b66a75c..dfcc7542e09 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -143,6 +143,7 @@ vespa_define_module( src/tests/proton/reprocessing/reprocessing_runner src/tests/proton/server src/tests/proton/server/disk_mem_usage_filter + src/tests/proton/server/disk_mem_usage_metrics src/tests/proton/server/disk_mem_usage_sampler src/tests/proton/server/health_adapter src/tests/proton/server/memory_flush_config_updater diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index d5823a8e055..13dcfb45dc9 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -117,6 +117,7 @@ public: search::SerialNum _oldestSerial; search::SerialNum _currentSerial; uint32_t _pendingDone; + uint32_t _taskDone; std::mutex _lock; vespalib::CountDownLatch _done; FlushDoneHistory _flushDoneHistory; @@ -131,6 +132,7 @@ public: _oldestSerial(0), _currentSerial(currentSerial), _pendingDone(0u), + _taskDone(0u), _lock(), _done(targets.size()), _flushDoneHistory() @@ -143,6 +145,11 @@ public: std::vector<IFlushTarget::SP> getFlushTargets() override { + { + std::lock_guard<std::mutex> guard(_lock); + _pendingDone += _taskDone; + _taskDone = 0; + } LOG(info, "SimpleHandler(%s)::getFlushTargets()", getName().c_str()); std::vector<IFlushTarget::SP> wrappedTargets; for (const auto &target : _targets) { @@ -154,7 +161,7 @@ public: // Called once by flush engine thread for each task done void taskDone() { std::lock_guard<std::mutex> guard(_lock); - ++_pendingDone; + ++_taskDone; } // Called by flush engine master thread after flush handler is diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt new file mode 100644 index 00000000000..fbd557991c8 --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_disk_mem_usage_metrics_test_app TEST + SOURCES + disk_mem_usage_metrics_test.cpp + DEPENDS + searchcore_server + GTest::GTest +) +vespa_add_test(NAME searchcore_disk_mem_usage_metrics_test_app COMMAND searchcore_disk_mem_usage_metrics_test_app) diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp new file mode 100644 index 00000000000..569633c518b --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/server/disk_mem_usage_metrics.h> +#include <vespa/searchcore/proton/server/disk_mem_usage_state.h> +#include <vespa/vespalib/gtest/gtest.h> + + +using proton::DiskMemUsageMetrics; +using proton::DiskMemUsageState; +using proton::ResourceUsageState; + +bool +expect_metrics(double disk_usage, double disk_utilization, double memory_usage, double memory_utilization, const DiskMemUsageMetrics &dm_metrics) +{ + bool result = true; + EXPECT_DOUBLE_EQ(disk_usage, dm_metrics.get_disk_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(disk_utilization, dm_metrics.get_disk_utilization()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_usage, dm_metrics.get_memory_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_utilization, dm_metrics.get_memory_utilization()) << (result = false, ""); + return result; +} + +TEST(DiskMemUsageMetricsTest, default_value_is_zero) +{ + DiskMemUsageMetrics dm_metrics; + EXPECT_TRUE(expect_metrics(0.0, 0.0, 0.0, 0.0, dm_metrics)); +} + +TEST(DiskMemUsageMetricsTest, merging_uses_max) +{ + DiskMemUsageMetrics dm_metrics({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); + EXPECT_TRUE(expect_metrics(0.4, 0.8, 0.3, 0.6, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.4, 0.4), ResourceUsageState(0.5, 0.4)}); + EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); + EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 7dff74c4fdd..3a51176b0d8 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -349,9 +349,6 @@ pruneremoveddocumentsinterval double default=0.0 ## Default value is 2 weeks (1209600 seconds). pruneremoveddocumentsage double default=1209600.0 -## Set to true to enable bucket locking via content layer -pruneremoveddocuments.usebucketexecutor bool default=true - ## Minimum size of packets to compress (0 means no compression) ## packetcompresslimit int default=1024 @@ -412,17 +409,9 @@ lidspacecompaction.removebatchblockrate double default=0.5 ## It is considered again at the next regular interval (see above). lidspacecompaction.removeblockrate double default=100.0 -## Set to true to enable bucket locking via content layer -## DEPRECATED and unused -lidspacecompaction.usebucketexecutor bool default=true - ## Maximum docs to move in single operation per bucket bucketmove.maxdocstomoveperbucket int default=1 -## Set to true to enable bucket locking via content layer -## DEPRECATED and unused -bucketmove.usebucketexecutor bool default=true - ## This is the maximum value visibilitydelay you can have. ## A to higher value here will cost more memory while not improving too much. maxvisibilitydelay double default=1.0 diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index c0b5bcab791..b8d1d2f17e6 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -71,6 +71,7 @@ DocsumContext::createReply() reply->docsums.resize(_docsumState._docsumcnt); SymbolTable::UP symbols = std::make_unique<SymbolTable>(); IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId()); + _docsumState._omit_summary_features = rci.outputClass->omit_summary_features(); for (uint32_t i = 0; i < _docsumState._docsumcnt; ++i) { buf.reset(); uint32_t docId = _docsumState._docsumbuf[i]; @@ -114,6 +115,7 @@ DocsumContext::createSlimeReply() const Symbol docsumSym = response->insert(DOCSUM); IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId()); + _docsumState._omit_summary_features = rci.outputClass->omit_summary_features(); uint32_t i(0); for (i = 0; (i < _docsumState._docsumcnt) && !_request.expired(); ++i) { uint32_t docId = _docsumState._docsumbuf[i]; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index d2ebb7a58a2..285e259e1a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchcore_server STATIC ddbstate.cpp disk_mem_usage_filter.cpp disk_mem_usage_forwarder.cpp + disk_mem_usage_metrics.cpp disk_mem_usage_sampler.cpp docstorevalidator.cpp document_db_config_owner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index ef77081132b..cbd378da9a0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -154,13 +154,14 @@ DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), _hwInfo(hwInfo), + _acceptWrite(true), _memoryStats(), _diskUsedSizeBytes(), _transient_memory_usage(0u), _config(), _state(), - _acceptWrite(true), _dmstate(), + _disk_mem_usage_metrics(), _listeners() { } @@ -254,6 +255,15 @@ DiskMemUsageFilter::usageState() const return _dmstate; } +DiskMemUsageMetrics +DiskMemUsageFilter::get_metrics() const +{ + Guard guard(_lock); + DiskMemUsageMetrics result(_disk_mem_usage_metrics); + _disk_mem_usage_metrics = DiskMemUsageMetrics(_dmstate); + return result; +} + bool DiskMemUsageFilter::acceptWriteOperation() const { @@ -295,6 +305,7 @@ DiskMemUsageFilter::notifyDiskMemUsage(const Guard &guard, DiskMemUsageState sta return; } _dmstate = state; + _disk_mem_usage_metrics.merge(state); for (const auto &listener : _listeners) { listener->notifyDiskMemUsage(_dmstate); } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index cdb9fc5f4cb..5bec34e51d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -4,6 +4,7 @@ #include "i_disk_mem_usage_notifier.h" #include "disk_mem_usage_state.h" +#include "disk_mem_usage_metrics.h" #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> @@ -40,16 +41,18 @@ public: }; private: - mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state + mutable Mutex _lock; HwInfo _hwInfo; + std::atomic<bool> _acceptWrite; + // Following member variables are protected by _lock vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; size_t _transient_memory_usage; size_t _transient_disk_usage; Config _config; State _state; - std::atomic<bool> _acceptWrite; DiskMemUsageState _dmstate; + mutable DiskMemUsageMetrics _disk_mem_usage_metrics; std::vector<IDiskMemUsageListener *> _listeners; void recalcState(const Guard &guard); // called with _lock held @@ -73,6 +76,7 @@ public: Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; + DiskMemUsageMetrics get_metrics() const; bool acceptWriteOperation() const override; State getAcceptState() const override; void addDiskMemUsageListener(IDiskMemUsageListener *listener) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp new file mode 100644 index 00000000000..2a97c56e63f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "disk_mem_usage_metrics.h" +#include "disk_mem_usage_state.h" +#include <algorithm> + +namespace proton { + +DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept + : DiskMemUsageMetrics(DiskMemUsageState()) +{ +} + +DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept + : _disk_usage(usage_state.diskState().usage()), + _disk_utilization(usage_state.diskState().utilization()), + _memory_usage(usage_state.memoryState().usage()), + _memory_utilization(usage_state.memoryState().utilization()) +{ +} + +void +DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept +{ + _disk_usage = std::max(_disk_usage, usage_state.diskState().usage()); + _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); + _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); + _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h new file mode 100644 index 00000000000..34d3bb36b40 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace proton { + +class DiskMemUsageState; + +/** + * Class containing disk and memory usage in a form suitable for + * metrics reporting. + */ +class DiskMemUsageMetrics +{ + double _disk_usage; + double _disk_utilization; + double _memory_usage; + double _memory_utilization; + +public: + DiskMemUsageMetrics() noexcept; + DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept; + void merge(const DiskMemUsageState &usage_state) noexcept; + double get_disk_usage() const noexcept { return _disk_usage; } + double get_disk_utilization() const noexcept { return _disk_utilization; } + double get_memory_usage() const noexcept { return _memory_usage; } + double get_memory_utilization() const noexcept { return _memory_utilization; } +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index adae323c1d9..ef72a52eb02 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -756,11 +756,11 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) } const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter(); - DiskMemUsageState usageState = usageFilter.usageState(); - metrics.resourceUsage.disk.set(usageState.diskState().usage()); - metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization()); - metrics.resourceUsage.memory.set(usageState.memoryState().usage()); - metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization()); + auto dm_metrics = usageFilter.get_metrics(); + metrics.resourceUsage.disk.set(dm_metrics.get_disk_usage()); + metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization()); + metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); + metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h index 29f9959aea9..0b6f34ba055 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h @@ -18,7 +18,7 @@ private: public: ResourceUsageState() - : _limit(0), + : _limit(1.0), _usage(0) { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index b600a612bb5..d41d4e0010c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -210,9 +210,9 @@ MultiAttrDFW<DataType>::insertField(uint32_t docid, GetDocsumsState* state, ResT return; // Don't insert empty fields } - Cursor &arr = target.insertArray(); std::vector<DataType> elements(entries); entries = std::min(entries, attr.get(docid, elements.data(), entries)); + Cursor &arr = target.insertArray(entries); if (_filter_elements) { const auto& matching_elems = state->get_matching_elements(*_matching_elems_fields) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 99a372d9676..6f042ee3907 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -36,6 +36,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _parsedLocations(), _summaryFeatures(nullptr), _summaryFeaturesCached(false), + _omit_summary_features(false), _rankFeatures(nullptr), _matching_elements(), _jsonStringer() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index db0f8e6e8ad..88a95d0446c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -84,6 +84,7 @@ public: // used by SummaryFeaturesDFW FeatureSet::SP _summaryFeatures; bool _summaryFeaturesCached; + bool _omit_summary_features; // used by RankFeaturesDFW FeatureSet::SP _rankFeatures; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index 6991d3acb29..aa891680a09 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -107,7 +107,7 @@ filter_matching_elements_in_input_field(const Slime& input_field, const std::vec { SlimeInserter output_inserter(output_field); Inspector& input_inspector = input_field.get(); - ArrayInserter array_inserter(output_inserter.insertArray()); + ArrayInserter array_inserter(output_inserter.insertArray(matching_elems.size())); auto elems_itr = matching_elems.begin(); for (size_t i = 0; (i < input_inspector.entries()) && (elems_itr != matching_elems.end()); ++i) { assert(*elems_itr >= i); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 8066a5e65db..f52cc62af92 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -15,7 +15,8 @@ ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & field _nameMap(), _fieldEnum(fieldEnum), _enumMap(), - _dynInfo(NULL) + _dynInfo(NULL), + _omit_summary_features(false) { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 52e331cd365..88c3552387b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -141,6 +141,9 @@ private: util::StringEnum &_fieldEnum; // fieldname -> f.n. enum value [SHARED] std::vector<int> _enumMap; // fieldname enum value -> entry index DynamicInfo *_dynInfo; // fields overridden and generated + // Whether or not summary features should be omitted when filling this summary class. + // As default, summary features are always included. + bool _omit_summary_features; public: typedef std::unique_ptr<ResultClass> UP; @@ -278,6 +281,14 @@ public: { return (offset < _entries.size()) ? &_entries[offset] : NULL; } + + void set_omit_summary_features(bool value) { + _omit_summary_features = value; + } + + bool omit_summary_features() const { + return _omit_summary_features; + } }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index ead3a4a2f9d..579779efb73 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -127,25 +127,27 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const int maxclassID = 0x7fffffff; // avoid negative classids _defaultSummaryId = cfg.defaultsummaryid; for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) { - if (cfg.classes[i].name.empty()) { + const auto& cfg_class = cfg.classes[i]; + if (cfg_class.name.empty()) { LOG(warning, "%s classes[%d]: empty name", configId, i); } - int classID = cfg.classes[i].id; + int classID = cfg_class.id; if (classID < 0 || classID > maxclassID) { LOG(error, "%s classes[%d]: bad id %d", configId, i, classID); rc = false; break; } - ResultClass *resClass = AddResultClass(cfg.classes[i].name.c_str(), classID); + ResultClass *resClass = AddResultClass(cfg_class.name.c_str(), classID); if (resClass == nullptr) { - LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg.classes[i].name.c_str()); + LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg_class.name.c_str()); rc = false; break; } - for (unsigned int j = 0; rc && (j < cfg.classes[i].fields.size()); j++) { - const char *fieldtype = cfg.classes[i].fields[j].type.c_str(); - const char *fieldname = cfg.classes[i].fields[j].name.c_str(); - LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg.classes[i].name.c_str(), fieldname, fieldtype); + resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures); + for (unsigned int j = 0; rc && (j < cfg_class.fields.size()); j++) { + const char *fieldtype = cfg_class.fields[j].type.c_str(); + const char *fieldname = cfg_class.fields[j].name.c_str(); + LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg_class.name.c_str(), fieldname, fieldtype); if (strcmp(fieldtype, "integer") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_INT); } else if (strcmp(fieldtype, "short") == 0) { @@ -179,12 +181,12 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const } else if (strcmp(fieldtype, "featuredata") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_FEATUREDATA); } else { - LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg.classes[i].name.c_str(), j, fieldtype); + LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg_class.name.c_str(), j, fieldtype); rc = false; break; } if (!rc) { - LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg.classes[i].name.c_str(), j, fieldname); + LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg_class.name.c_str(), j, fieldname); break; } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 425faff6a67..4d81f4505a7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -30,6 +30,9 @@ static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { + if (state->_omit_summary_features) { + return; + } if ( ! state->_summaryFeatures) { state->_callback.FillSummaryFeatures(state, _env); if ( !state->_summaryFeatures) { // still no summary features to write diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 578fcc83006..df28737f352 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -16,14 +16,6 @@ <description>Service monitor component for hosted vespa.</description> <dependencies> - <!-- compile scope --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.5</version> - <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new --> - <scope>compile</scope> - </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> @@ -34,6 +26,12 @@ <!-- provided scope --> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/slobrok/src/vespa/slobrok/server/cmd.cpp b/slobrok/src/vespa/slobrok/server/cmd.cpp index 99e86814852..60060b9868f 100644 --- a/slobrok/src/vespa/slobrok/server/cmd.cpp +++ b/slobrok/src/vespa/slobrok/server/cmd.cpp @@ -8,7 +8,7 @@ #include "sbenv.h" #include <vespa/log/log.h> -LOG_SETUP(".cmd"); +LOG_SETUP(".slobrok.server.cmd"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index c4a4acd1ced..c7e3c3b32a6 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -6,7 +6,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.exchange_manager"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp index 3220a9e5d69..930cdbc962a 100644 --- a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp +++ b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp @@ -7,7 +7,7 @@ #include <vespa/fnet/frt/target.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.managed_rpc_server"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/named_service.cpp b/slobrok/src/vespa/slobrok/server/named_service.cpp index c1aeb83a8a5..9747bbbce52 100644 --- a/slobrok/src/vespa/slobrok/server/named_service.cpp +++ b/slobrok/src/vespa/slobrok/server/named_service.cpp @@ -3,7 +3,7 @@ #include "named_service.h" #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.named_service"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp index 3bb9841d791..5107b5844c8 100644 --- a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp +++ b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp @@ -8,7 +8,7 @@ #include <vespa/log/log.h> #include <vespa/config/common/exceptions.h> -LOG_SETUP(".reconfigurable_stateserver"); +LOG_SETUP(".slobrok.server.reconfigurable_stateserver"); using namespace std::chrono_literals; diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index f08a1646b88..157b959dbfe 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -9,7 +9,7 @@ #include "exchange_manager.h" #include <vespa/log/log.h> -LOG_SETUP(".remcheck"); +LOG_SETUP(".slobrok.server.remote_check"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp index d50aee21a29..539a901fd9d 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp @@ -9,7 +9,7 @@ #include <vespa/fnet/frt/target.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.remote_slobrok"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp index ead062e9783..dfcdc0ef9d1 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp @@ -9,7 +9,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.rpc_server_manager"); using vespalib::make_string_short::fmt; diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp index 0f0fae61d4c..5f26608c294 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp @@ -6,7 +6,7 @@ #include "sbenv.h" #include <vespa/log/log.h> -LOG_SETUP(".rpcsrvmap"); +LOG_SETUP(".slobrok.server.rpc_server_map"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 7388e7f7e22..51cd3d65857 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -12,7 +12,7 @@ #include <vespa/vespalib/component/vtag.h> #include <vespa/log/log.h> -LOG_SETUP(".rpchooks"); +LOG_SETUP(".slobrok.server.rpchooks"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp index 7139dd47838..be533d9b30f 100644 --- a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp @@ -5,7 +5,7 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcmirror"); +LOG_SETUP(".slobrok.server.rpcmirror"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 525ba0c0aa7..2bf8e57cb26 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -13,7 +13,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".sbenv"); +LOG_SETUP(".slobrok.server.sbenv"); using namespace std::chrono_literals; diff --git a/slobrok/src/vespa/slobrok/server/service_map_history.cpp b/slobrok/src/vespa/slobrok/server/service_map_history.cpp index f9de736c093..3882a41bde5 100644 --- a/slobrok/src/vespa/slobrok/server/service_map_history.cpp +++ b/slobrok/src/vespa/slobrok/server/service_map_history.cpp @@ -3,7 +3,7 @@ #include "service_map_history.h" #include <vespa/log/log.h> -LOG_SETUP(".slobrok.publisher"); +LOG_SETUP(".slobrok.server.service_map_history"); namespace slobrok { diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h index 537b98a0618..5c251d14e17 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h @@ -11,7 +11,6 @@ #include "dimension.h" #include "dummy_metrics_manager.h" #include "gauge.h" -#include "json_formatter.h" #include "label.h" #include "metric_id.h" #include "metrics_manager.h" diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 11d0406fcc5..4aa66e9b1a4 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -90,6 +90,8 @@ config-model-api-jar-with-dependencies.jar, config-model-jar-with-dependencies.jar, container-disc-jar-with-dependencies.jar, + model-evaluation-jar-with-dependencies.jar, + model-integration-jar-with-dependencies.jar, vespajlib.jar </discPreInstallBundle> </configuration> diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index 3f3937658d2..b30a7ad4cd0 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -167,6 +167,7 @@ StartCommand() { --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.library.path="$VESPA_HOME/lib64" \ diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 799490ba114..70b8d3ca2b7 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -29,7 +29,16 @@ mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## TTL for rpc target cache mbus.rpctargetcache.ttl double default = 600 restart -## Number of threads for mbus threadpool +## Number of rpc targets per spec +mbus.num_rpc_targets int default=1 restart + +## Use tcpNoDelay for mbus network writes +mbus.tcp_no_delay bool default=true restart + +## Number of threads for network. +mbus.num_network_threads int default=1 restart + +## Number of workers threads for messagebus ## Any value below 1 will be 1. mbus.num_threads int default=4 restart diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 25942926155..7de1b7d869e 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -17,14 +17,11 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/stringfmt.h> - -#include <vespa/log/bufferedlogger.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> #include <string_view> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".communication.manager"); using vespalib::make_string; @@ -396,9 +393,11 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> mbus::RPCNetworkParams params(_configUri); params.setConnectionExpireSecs(config->mbus.rpctargetcache.ttl); params.setNumThreads(std::max(1, config->mbus.numThreads)); + params.setNumNetworkThreads(std::max(1, config->mbus.numNetworkThreads)); + params.setNumRpcTargets(std::max(1, config->mbus.numRpcTargets)); params.setDispatchOnDecode(config->mbus.dispatchOnDecode); params.setDispatchOnEncode(config->mbus.dispatchOnEncode); - params.setTcpNoDelay(config->mbus.optimizeFor == CommunicationManagerConfig::Mbus::OptimizeFor::LATENCY); + params.setTcpNoDelay(config->mbus.tcpNoDelay); params.setIdentity(mbus::Identity(_component.getIdentity())); if (config->mbusport != -1) { diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 90ab2a81e0c..1b03891f313 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -25,6 +25,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -131,14 +137,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> <exclusions> @@ -154,6 +152,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-annotations</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -170,7 +172,7 @@ <!-- required by java-jwt --> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> - <version>1.15</version> + <version>${commons.codec.version}</version> <scope>compile</scope> </dependency> </dependencies> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java index 97bc7dc838e..f80c28424f2 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java @@ -35,7 +35,7 @@ public class VespaOutputFormat extends OutputFormat { public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { VespaCounters counters = VespaCounters.get(context); VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride); - return configuration.useLegacyClient() + return configuration.useLegacyClient().orElse(true) ? new LegacyVespaRecordWriter(configuration, counters) : new VespaRecordWriter(configuration, counters); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 7219e621486..1be794f8e11 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -6,6 +6,7 @@ import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.io.StringReader; +import java.util.Optional; import java.util.Properties; public class VespaConfiguration { @@ -131,7 +132,11 @@ public class VespaConfiguration { return getInt(PROGRESS_REPORT, 1000); } - public boolean useLegacyClient() { return getBoolean(USE_LEGACY_CLIENT, true); } + public Optional<Boolean> useLegacyClient() { + String raw = getString(USE_LEGACY_CLIENT); + if (raw == null || raw.trim().isEmpty()) return Optional.empty(); + return Optional.of(Boolean.parseBoolean(raw)); + } public String getString(String name) { if (override != null && override.containsKey(name)) { @@ -191,6 +196,7 @@ public class VespaConfiguration { sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n"); sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n"); sb.append(NUM_RETRIES + ": " + numRetries() +"\n"); + sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n"); return sb.toString(); } diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java index 259ae2602c4..69c0e343872 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java @@ -39,6 +39,8 @@ public class GenerateTestDescriptorMojo extends AbstractMojo { } private void analyzeTestClasses(TestAnnotationAnalyzer analyzer) throws MojoExecutionException { + if (! Files.exists(testClassesDirectory())) return; + try (Stream<Path> files = Files.walk(testClassesDirectory())) { files .filter(f -> f.toString().endsWith(".class")) diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 834c3d7c988..17443b11b6a 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -38,11 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-http-client</artifactId> <version>${project.version}</version> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 9db296e33cd..770aa9bdb81 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1133,7 +1133,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { default: response.writeMessage(error.get() != null ? error.get() : message != null ? message : "Visiting failed"); if (getVisitorStatistics() != null) - response.writeDocumentCount(getVisitorStatistics().getDocumentsReturned()); + response.writeDocumentCount(getVisitorStatistics().getDocumentsVisited()); response.respond(Response.Status.BAD_GATEWAY); } diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index e58b1599b8f..6e7b3689004 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -2,6 +2,8 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/object_value.h> +#include <vespa/vespalib/data/slime/array_value.h> #include <vespa/vespalib/data/slime/strfmt.h> #include <vespa/vespalib/data/simple_buffer.h> #include <type_traits> diff --git a/vespalib/src/vespa/vespalib/data/slime/array_value.cpp b/vespalib/src/vespa/vespalib/data/slime/array_value.cpp index 1d04bfe9c7c..e34c352b4fd 100644 --- a/vespalib/src/vespa/vespalib/data/slime/array_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/array_value.cpp @@ -3,9 +3,18 @@ #include "array_value.h" #include "array_traverser.h" #include "empty_value_factory.h" +#include "symbol_table.h" namespace vespalib::slime { +ArrayValue::ArrayValue(SymbolTable &table, Stash & stash) + : _symbolTable(table), + _stash(stash), + _values() +{} + +ArrayValue::~ArrayValue() = default; + void ArrayValue::traverse(ArrayTraverser &at) const { for (size_t i = 0; i < _values.size(); ++i) { @@ -15,8 +24,8 @@ ArrayValue::traverse(ArrayTraverser &at) const { Cursor & -ArrayValue::addArray() { - return addLeaf(ArrayValueFactory(_symbolTable)); +ArrayValue::addArray(size_t reserve) { + return addLeaf(ArrayValueFactory(_symbolTable, reserve)); } Cursor & diff --git a/vespalib/src/vespa/vespalib/data/slime/array_value.h b/vespalib/src/vespa/vespalib/data/slime/array_value.h index 5561b93aca2..99b00b884b0 100644 --- a/vespalib/src/vespa/vespalib/data/slime/array_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/array_value.h @@ -5,9 +5,7 @@ #include "value.h" #include "nix_value.h" #include "value_factory.h" -#include "symbol_table.h" #include <vector> -#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { @@ -15,7 +13,7 @@ namespace vespalib::slime { * Class representing a collection of ordered values that can be * looked up by index. **/ -class ArrayValue : public Value +class ArrayValue final : public Value { private: SymbolTable &_symbolTable; @@ -30,9 +28,10 @@ protected: } public: - ArrayValue(SymbolTable &table, Stash & stash) : _symbolTable(table), _stash(stash), _values() {} + ArrayValue(SymbolTable &table, Stash & stash); ArrayValue(const ArrayValue &) = delete; ArrayValue &operator=(const ArrayValue &) = delete; + void reserve(size_t sz) { _values.reserve(sz); } Type type() const override { return ARRAY::instance; } size_t children() const override { return _values.size(); } @@ -46,11 +45,11 @@ public: return *NixValue::invalid(); } - Cursor &addArray() override; + Cursor &addArray(size_t reserve) override; Cursor &addObject() override; Symbol resolve(Memory symbol_name) override; - ~ArrayValue() override = default; + ~ArrayValue() override; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp index d09ac003526..9e96cd7bcf6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "basic_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.h b/vespalib/src/vespa/vespalib/data/slime/basic_value.h index 0b962085b78..20183999d8f 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.h @@ -5,40 +5,41 @@ #include "value.h" #include "memory.h" #include <vespa/vespalib/util/traits.h> -#include <vespa/vespalib/util/stash.h> + +namespace vespalib { class Stash; } namespace vespalib::slime { /** * Classes representing a single basic value. **/ -class BasicBoolValue : public Value { +class BasicBoolValue final : public Value { bool _value; public: - BasicBoolValue(bool bit) : _value(bit) {} + BasicBoolValue(bool bit) noexcept : _value(bit) {} bool asBool() const override { return _value; } Type type() const override { return BOOL::instance; } }; -class BasicLongValue : public Value { +class BasicLongValue final : public Value { int64_t _value; public: - BasicLongValue(int64_t l) : _value(l) {} + BasicLongValue(int64_t l) noexcept : _value(l) {} int64_t asLong() const override { return _value; } double asDouble() const override { return _value; } Type type() const override { return LONG::instance; } }; -class BasicDoubleValue : public Value { +class BasicDoubleValue final : public Value { double _value; public: - BasicDoubleValue(double d) : _value(d) {} + BasicDoubleValue(double d) noexcept : _value(d) {} double asDouble() const override { return _value; } int64_t asLong() const override { return _value; } Type type() const override { return DOUBLE::instance; } }; -class BasicStringValue : public Value { +class BasicStringValue final : public Value { Memory _value; public: BasicStringValue(Memory str, Stash & stash); @@ -48,7 +49,7 @@ public: Type type() const override { return STRING::instance; } }; -class BasicDataValue : public Value { +class BasicDataValue final : public Value { Memory _value; public: BasicDataValue(Memory data, Stash & stash); diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h index 5562415e33e..e7b2a9dd806 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h @@ -8,33 +8,33 @@ namespace vespalib::slime { -struct BoolValueFactory : public ValueFactory { +struct BoolValueFactory final : public ValueFactory { bool input; - BoolValueFactory(bool in) : input(in) {} + BoolValueFactory(bool in) noexcept : input(in) {} Value *create(Stash & stash) const override { return & stash.create<BasicBoolValue>(input); } }; -struct LongValueFactory : public ValueFactory { +struct LongValueFactory final : public ValueFactory { int64_t input; - LongValueFactory(int64_t in) : input(in) {} + LongValueFactory(int64_t in) noexcept : input(in) {} Value *create(Stash & stash) const override { return & stash.create<BasicLongValue>(input); } }; -struct DoubleValueFactory : public ValueFactory { +struct DoubleValueFactory final : public ValueFactory { double input; - DoubleValueFactory(double in) : input(in) {} + DoubleValueFactory(double in) noexcept : input(in) {} Value *create(Stash & stash) const override { return & stash.create<BasicDoubleValue>(input); } }; -struct StringValueFactory : public ValueFactory { +struct StringValueFactory final : public ValueFactory { Memory input; - StringValueFactory(Memory in) : input(in) {} + StringValueFactory(Memory in) noexcept : input(in) {} Value *create(Stash & stash) const override { return & stash.create<BasicStringValue>(input, stash); } }; -struct DataValueFactory : public ValueFactory { +struct DataValueFactory final : public ValueFactory { Memory input; - DataValueFactory(Memory in) : input(in) {} + DataValueFactory(Memory in) noexcept : input(in) {} Value *create(Stash & stash) const override { return & stash.create<BasicDataValue>(input, stash); } }; diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp index 1956dde689c..2eb6173b1e5 100644 --- a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp @@ -8,6 +8,25 @@ LOG_SETUP(".vespalib.data.slime.binary_format"); namespace vespalib::slime::binary_format { +namespace { + +void +write_cmpr_ulong_impl(OutputWriter &out, uint64_t value) { + out.commit(encode_cmpr_ulong(out.reserve(10), value)); +} + +void +write_type_and_size_impl(OutputWriter &out, uint32_t type, uint64_t size) { + char *start = out.reserve(11); // max size + char *pos = start; + if (size <= 30) { + *pos++ = encode_type_and_meta(type, size + 1); + } else { + *pos++ = encode_type_and_meta(type, 0); + pos += encode_cmpr_ulong(pos, size); + } + out.commit(pos - start); +} struct BinaryEncoder : public ArrayTraverser, public ObjectSymbolTraverser @@ -27,21 +46,21 @@ struct BinaryEncoder : public ArrayTraverser, write_type_and_bytes<true>(out, DOUBLE::ID, encode_double(value)); } void encodeString(const Memory &memory) { - write_type_and_size(out, STRING::ID, memory.size); + write_type_and_size_impl(out, STRING::ID, memory.size); out.write(memory.data, memory.size); } void encodeData(const Memory &memory) { - write_type_and_size(out, DATA::ID, memory.size); + write_type_and_size_impl(out, DATA::ID, memory.size); out.write(memory.data, memory.size); } void encodeArray(const Inspector &inspector) { ArrayTraverser &array_traverser = *this; - write_type_and_size(out, ARRAY::ID, inspector.children()); + write_type_and_size_impl(out, ARRAY::ID, inspector.children()); inspector.traverse(array_traverser); } void encodeObject(const Inspector &inspector) { ObjectSymbolTraverser &object_traverser = *this; - write_type_and_size(out, OBJECT::ID, inspector.children()); + write_type_and_size_impl(out, OBJECT::ID, inspector.children()); inspector.traverse(object_traverser); } void encodeValue(const Inspector &inspector) { @@ -59,10 +78,10 @@ struct BinaryEncoder : public ArrayTraverser, } void encodeSymbolTable(const Slime &slime) { size_t numSymbols = slime.symbols(); - write_cmpr_ulong(out, numSymbols); + write_cmpr_ulong_impl(out, numSymbols); for (size_t i = 0; i < numSymbols; ++i) { Memory image = slime.inspect(Symbol(i)); - write_cmpr_ulong(out, image.size); + write_cmpr_ulong_impl(out, image.size); out.write(image.data, image.size); } } @@ -79,7 +98,7 @@ BinaryEncoder::entry(size_t, const Inspector &inspector) void BinaryEncoder::field(const Symbol &symbol, const Inspector &inspector) { - write_cmpr_ulong(out, symbol.getValue()); + write_cmpr_ulong_impl(out, symbol.getValue()); encodeValue(inspector); } @@ -108,8 +127,8 @@ struct MappedSymbols { } Symbol map_symbol(Symbol symbol) const { return (symbol.getValue() < symbol_mapping.size()) - ? symbol_mapping[symbol.getValue()] - : symbol; + ? symbol_mapping[symbol.getValue()] + : symbol; } }; @@ -175,9 +194,7 @@ struct BinaryDecoder : SymbolHandler<remap_symbols>::type { void decodeValue(const Inserter &inserter) { char byte = in.read(); - Cursor &cursor = decodeValue(inserter, - decode_type(byte), - decode_meta(byte)); + Cursor &cursor = decodeValue(inserter, decode_type(byte), decode_meta(byte)); if (!cursor.valid()) { in.fail("failed to decode value"); } @@ -239,22 +256,16 @@ size_t decode(const Memory &memory, Slime &slime, const Inserter &inserter) { return input.failed() ? 0 : input.get_offset(); } +} + void write_cmpr_ulong(OutputWriter &out, uint64_t value) { - out.commit(encode_cmpr_ulong(out.reserve(10), value)); + write_cmpr_ulong_impl(out, value); } void write_type_and_size(OutputWriter &out, uint32_t type, uint64_t size) { - char *start = out.reserve(11); // max size - char *pos = start; - if (size <= 30) { - *pos++ = encode_type_and_meta(type, size + 1); - } else { - *pos++ = encode_type_and_meta(type, 0); - pos += encode_cmpr_ulong(pos, size); - } - out.commit(pos - start); + write_type_and_size_impl(out, type, size); } } diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h index 34d7028a027..6f926de85b6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/cursor.h +++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h @@ -19,7 +19,7 @@ struct Cursor : public Inspector { virtual Cursor &addString(Memory str) = 0; virtual Cursor &addData(Memory data) = 0; virtual Cursor &addData(ExternalMemory::UP data) = 0; - virtual Cursor &addArray() = 0; + virtual Cursor &addArray(size_t reserved_size) = 0; virtual Cursor &addObject() = 0; virtual Cursor &setNix(Symbol sym) = 0; @@ -29,7 +29,7 @@ struct Cursor : public Inspector { virtual Cursor &setString(Symbol sym, Memory str) = 0; virtual Cursor &setData(Symbol sym, Memory data) = 0; virtual Cursor &setData(Symbol sym, ExternalMemory::UP data) = 0; - virtual Cursor &setArray(Symbol sym) = 0; + virtual Cursor &setArray(Symbol sym, size_t reserved_size) = 0; virtual Cursor &setObject(Symbol sym) = 0; virtual Cursor &setNix(Memory name) = 0; @@ -39,10 +39,20 @@ struct Cursor : public Inspector { virtual Cursor &setString(Memory name, Memory str) = 0; virtual Cursor &setData(Memory name, Memory data) = 0; virtual Cursor &setData(Memory name, ExternalMemory::UP data) = 0; - virtual Cursor &setArray(Memory name) = 0; + virtual Cursor &setArray(Memory name, size_t reserved_size) = 0; virtual Cursor &setObject(Memory name) = 0; virtual Symbol resolve(Memory symbol_name) = 0; + + Cursor &addArray() { + return addArray(0); + } + Cursor &setArray(Memory name) { + return setArray(name, 0); + } + Cursor &setArray(Symbol sym) { + return setArray(sym, 0); + } }; } diff --git a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp index 3be492b7bcc..f6573ad26c4 100644 --- a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp @@ -1,12 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "empty_value_factory.h" +#include "object_value.h" +#include "array_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { Value * ArrayValueFactory::create(Stash & stash) const { - return & stash.create<ArrayValue>(symbolTable, stash); + ArrayValue & arr = stash.create<ArrayValue>(symbolTable, stash); + arr.reserve(_reserve); + return &arr; } Value * diff --git a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h index 5fc22b31df9..c92b7ed1c68 100644 --- a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h @@ -4,25 +4,23 @@ #include "value_factory.h" #include "nix_value.h" -#include "array_value.h" -#include "object_value.h" -#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { -struct NixValueFactory : public ValueFactory { +struct NixValueFactory final : public ValueFactory { Value *create(Stash &) const override { return NixValue::instance(); } }; -struct ArrayValueFactory : public ValueFactory { +struct ArrayValueFactory final : public ValueFactory { SymbolTable &symbolTable; - ArrayValueFactory(SymbolTable &table) : symbolTable(table) {} + size_t _reserve; + ArrayValueFactory(SymbolTable &table, size_t reserve) noexcept : symbolTable(table), _reserve(reserve) {} Value *create(Stash & stash) const override; }; -struct ObjectValueFactory : public ValueFactory { +struct ObjectValueFactory final : public ValueFactory { SymbolTable &symbolTable; - ObjectValueFactory(SymbolTable &table) : symbolTable(table) {} + ObjectValueFactory(SymbolTable &table) noexcept : symbolTable(table) {} Value *create(Stash & stash) const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp index 0042319d8a1..7e64800eefc 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp @@ -14,7 +14,7 @@ Cursor &SlimeInserter::insertDouble(double value) const { return slime.setDouble Cursor &SlimeInserter::insertString(Memory value) const { return slime.setString(value); } Cursor &SlimeInserter::insertData(Memory value) const { return slime.setData(value); } Cursor &SlimeInserter::insertData(ExtMemUP value) const { return slime.setData(std::move(value)); } -Cursor &SlimeInserter::insertArray() const { return slime.setArray(); } +Cursor &SlimeInserter::insertArray(size_t resv) const { return slime.setArray(resv); } Cursor &SlimeInserter::insertObject() const { return slime.setObject(); } Cursor &ArrayInserter::insertNix() const { return cursor.addNix(); } @@ -24,7 +24,7 @@ Cursor &ArrayInserter::insertDouble(double value) const { return cursor.addDoubl Cursor &ArrayInserter::insertString(Memory value) const { return cursor.addString(value); } Cursor &ArrayInserter::insertData(Memory value) const { return cursor.addData(value); } Cursor &ArrayInserter::insertData(ExtMemUP value) const { return cursor.addData(std::move(value)); } -Cursor &ArrayInserter::insertArray() const { return cursor.addArray(); } +Cursor &ArrayInserter::insertArray(size_t resv) const { return cursor.addArray(resv); } Cursor &ArrayInserter::insertObject() const { return cursor.addObject(); } Cursor &ObjectSymbolInserter::insertNix() const { return cursor.setNix(symbol); } @@ -34,7 +34,7 @@ Cursor &ObjectSymbolInserter::insertDouble(double value) const { return cursor.s Cursor &ObjectSymbolInserter::insertString(Memory value) const { return cursor.setString(symbol, value); } Cursor &ObjectSymbolInserter::insertData(Memory value) const { return cursor.setData(symbol, value); } Cursor &ObjectSymbolInserter::insertData(ExtMemUP value) const { return cursor.setData(symbol, std::move(value)); } -Cursor &ObjectSymbolInserter::insertArray() const { return cursor.setArray(symbol); } +Cursor &ObjectSymbolInserter::insertArray(size_t resv) const { return cursor.setArray(symbol, resv); } Cursor &ObjectSymbolInserter::insertObject() const { return cursor.setObject(symbol); } Cursor &ObjectInserter::insertNix() const { return cursor.setNix(name); } @@ -44,7 +44,7 @@ Cursor &ObjectInserter::insertDouble(double value) const { return cursor.setDoub Cursor &ObjectInserter::insertString(Memory value) const { return cursor.setString(name, value); } Cursor &ObjectInserter::insertData(Memory value) const { return cursor.setData(name, value); } Cursor &ObjectInserter::insertData(ExtMemUP value) const { return cursor.setData(name, std::move(value)); } -Cursor &ObjectInserter::insertArray() const { return cursor.setArray(name); } +Cursor &ObjectInserter::insertArray(size_t resv) const { return cursor.setArray(name, resv); } Cursor &ObjectInserter::insertObject() const { return cursor.setObject(name); } } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index 8206e67a79a..deddb5671c3 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -27,9 +27,13 @@ struct Inserter { virtual Cursor &insertString(Memory value) const = 0; virtual Cursor &insertData(Memory value) const = 0; virtual Cursor &insertData(ExternalMemory::UP value) const = 0; - virtual Cursor &insertArray() const = 0; + virtual Cursor &insertArray(size_t reserved) const = 0; virtual Cursor &insertObject() const = 0; - virtual ~Inserter() {} + virtual ~Inserter() = default; + + Cursor &insertArray() const { + return insertArray(0); + } }; //----------------------------------------------------------------------------- @@ -45,7 +49,7 @@ struct SlimeInserter : Inserter { Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; Cursor &insertData(ExternalMemory::UP value) const override; - Cursor &insertArray() const override; + Cursor &insertArray(size_t reserved) const override; Cursor &insertObject() const override; }; @@ -60,7 +64,7 @@ struct ArrayInserter : Inserter { Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; Cursor &insertData(ExternalMemory::UP value) const override; - Cursor &insertArray() const override; + Cursor &insertArray(size_t reserved) const override; Cursor &insertObject() const override; }; @@ -76,7 +80,7 @@ struct ObjectSymbolInserter : Inserter { Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; Cursor &insertData(ExternalMemory::UP value) const override; - Cursor &insertArray() const override; + Cursor &insertArray(size_t reserved) const override; Cursor &insertObject() const override; }; @@ -92,7 +96,7 @@ struct ObjectInserter : Inserter { Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; Cursor &insertData(ExternalMemory::UP value) const override; - Cursor &insertArray() const override; + Cursor &insertArray(size_t reserved) const override; Cursor &insertObject() const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h index aac35b2e386..9752040d3e8 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h @@ -10,14 +10,14 @@ namespace vespalib::slime { /** * Class used to insert the name of a field into a symbol table. **/ -class NamedSymbolInserter : public SymbolInserter +class NamedSymbolInserter final : public SymbolInserter { private: SymbolTable &_table; const Memory &_name; public: - NamedSymbolInserter(SymbolTable &table, const Memory &name) + NamedSymbolInserter(SymbolTable &table, const Memory &name) noexcept : _table(table), _name(name) {} Symbol insert() override { return _table.insert(_name); diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.cpp b/vespalib/src/vespa/vespalib/data/slime/object_value.cpp index f3436ac0978..25a5d762825 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.cpp @@ -24,18 +24,16 @@ ObjectValue::setLeaf(Memory name, const ValueFactory &input) { void ObjectValue::traverse(ObjectSymbolTraverser &ot) const { - typedef SymbolValueMap::const_iterator ITR; - for (ITR pos = _fields.begin(); pos != _fields.end(); ++pos) { - ot.field(pos->first, *pos->second); + for (const auto & field : _fields) { + ot.field(field.first, *field.second); } } void ObjectValue::traverse(ObjectTraverser &ot) const { - typedef SymbolValueMap::const_iterator ITR; - for (ITR pos = _fields.begin(); pos != _fields.end(); ++pos) { - Memory symbol = _symbolTable.inspect(pos->first); - ot.field(symbol, *pos->second); + for (const auto & field : _fields) { + Memory symbol = _symbolTable.inspect(field.first); + ot.field(symbol, *field.second); } } @@ -53,8 +51,8 @@ ObjectValue::operator[](Memory name) const { Cursor & -ObjectValue::setArray(Symbol symbol) { - return setLeaf(symbol, ArrayValueFactory(_symbolTable)); +ObjectValue::setArray(Symbol symbol, size_t reserve) { + return setLeaf(symbol, ArrayValueFactory(_symbolTable, reserve)); } Cursor & @@ -63,8 +61,8 @@ ObjectValue::setObject(Symbol symbol) { } Cursor & -ObjectValue::setArray(Memory name) { - return setLeaf(name, ArrayValueFactory(_symbolTable)); +ObjectValue::setArray(Memory name, size_t reserve) { + return setLeaf(name, ArrayValueFactory(_symbolTable, reserve)); } Cursor & diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h index 651f3a156d2..a26027f9e0d 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h @@ -6,25 +6,24 @@ #include "nix_value.h" #include "symbol.h" #include "symbol_lookup.h" -#include "symbol_table.h" #include "value_factory.h" #include "symbol_inserter.h" #include <vespa/vespalib/stllike/vector_map.h> -#include <vespa/vespalib/util/stash.h> +namespace vespalib { class Stash; } namespace vespalib::slime { /** * Class representing a collection of unordered values that can be * looked up by symbol. **/ -class ObjectValue : public Value +class ObjectValue final : public Value { private: struct hasher { size_t operator () (const Symbol & s) const { return s.getValue(); } }; - typedef vector_map<Symbol, Value*> SymbolValueMap; + using SymbolValueMap = vector_map<Symbol, Value*>; SymbolTable &_symbolTable; Stash &_stash; SymbolValueMap _fields; @@ -73,10 +72,10 @@ public: Cursor &operator[](Symbol sym) const override; Cursor &operator[](Memory name) const override; - Cursor &setArray(Symbol sym) override; + Cursor &setArray(Symbol sym, size_t reserve) override; Cursor &setObject(Symbol sym) override; - Cursor &setArray(Memory name) override; + Cursor &setArray(Memory name, size_t reserve) override; Cursor &setObject(Memory name) override; Symbol resolve(Memory symbol_name) override; diff --git a/vespalib/src/vespa/vespalib/data/slime/resolved_symbol.h b/vespalib/src/vespa/vespalib/data/slime/resolved_symbol.h index 2b8c39708c9..714230bdf94 100644 --- a/vespalib/src/vespa/vespalib/data/slime/resolved_symbol.h +++ b/vespalib/src/vespa/vespalib/data/slime/resolved_symbol.h @@ -13,14 +13,14 @@ namespace vespalib::slime { * appropriate symbol table. Thus, this class can satisfy both the * symbol lookup and inserter interfaces. **/ -class ResolvedSymbol : public SymbolLookup, - public SymbolInserter +class ResolvedSymbol final : public SymbolLookup, + public SymbolInserter { private: Symbol _symbol; public: - ResolvedSymbol(const Symbol &symbol) : _symbol(symbol) {} + ResolvedSymbol(const Symbol &symbol) noexcept : _symbol(symbol) {} Symbol lookup() const override { return _symbol; } diff --git a/vespalib/src/vespa/vespalib/data/slime/root_value.cpp b/vespalib/src/vespa/vespalib/data/slime/root_value.cpp index d99fc90fdee..f910c6ac4cf 100644 --- a/vespalib/src/vespa/vespalib/data/slime/root_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/root_value.cpp @@ -1,7 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "root_value.h" +#include "object_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { +Value * +RootValue::wrap(SymbolTable &table, SymbolInserter &symbol) { + Value *value = & _stash->create<ObjectValue>(table, *_stash, symbol, _value); + _value = value; + return _value; +} + } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/root_value.h b/vespalib/src/vespa/vespalib/data/slime/root_value.h index 314b386de6e..ed1c97b7d60 100644 --- a/vespalib/src/vespa/vespalib/data/slime/root_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/root_value.h @@ -3,7 +3,6 @@ #pragma once #include "nix_value.h" -#include "object_value.h" #include "value_factory.h" namespace vespalib::slime { @@ -35,12 +34,8 @@ public: _value = value; return *value; } - Value *wrap(SymbolTable &table, SymbolInserter &symbol) { - Value *value = & _stash->create<ObjectValue>(table, *_stash, symbol, _value); - _value = value; - return _value; - } - ~RootValue() { } + Value *wrap(SymbolTable &table, SymbolInserter &symbol); + ~RootValue() = default; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index 3ee608799a6..ee3222b2e8d 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -3,7 +3,6 @@ #pragma once #include "array_traverser.h" -#include "array_value.h" #include "basic_value.h" #include "basic_value_factory.h" #include "binary_format.h" @@ -18,7 +17,6 @@ #include "named_symbol_lookup.h" #include "nix_value.h" #include "object_traverser.h" -#include "object_value.h" #include "resolved_symbol.h" #include "root_value.h" #include "symbol.h" @@ -157,7 +155,10 @@ public: return _root.set(slime::ExternalDataValueFactory(std::move(data))); } Cursor &setArray() { - return _root.set(slime::ArrayValueFactory(*_names)); + return setArray(0); + } + Cursor &setArray(size_t reserve) { + return _root.set(slime::ArrayValueFactory(*_names, reserve)); } Cursor &setObject() { return _root.set(slime::ObjectValueFactory(*_names)); diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol.h b/vespalib/src/vespa/vespalib/data/slime/symbol.h index 77620f3ff92..020fc8bc827 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol.h @@ -12,13 +12,13 @@ namespace vespalib::slime { class Symbol { private: - static const uint32_t UNDEFINED = (uint32_t)-1; + static constexpr uint32_t UNDEFINED = (uint32_t)-1; uint32_t _value; public: - Symbol() : _value(UNDEFINED) {} - Symbol(uint32_t v) : _value(v) {} + Symbol() noexcept : _value(UNDEFINED) {} + Symbol(uint32_t v) noexcept : _value(v) {} bool undefined() const { return (_value == UNDEFINED); } uint32_t getValue() const { return _value; } bool operator<(const Symbol &rhs) const noexcept { return (_value < rhs._value); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp index 300eeba47e5..f7c37d73652 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp @@ -130,16 +130,16 @@ Value::setData(Memory name, ExternalMemory::UP data) { return setLeaf(name, Exte // nop defaults for array/objects Cursor & -Value::addArray() { return *NixValue::invalid(); } +Value::addArray(size_t) { return *NixValue::invalid(); } Cursor & Value::addObject() { return *NixValue::invalid(); } Cursor & -Value::setArray(Symbol) { return *NixValue::invalid(); } +Value::setArray(Symbol, size_t) { return *NixValue::invalid(); } Cursor & Value::setObject(Symbol) { return *NixValue::invalid(); } Cursor & -Value::setArray(Memory) { return *NixValue::invalid(); } +Value::setArray(Memory, size_t) { return *NixValue::invalid(); } Cursor & Value::setObject(Memory) { return *NixValue::invalid(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value.h b/vespalib/src/vespa/vespalib/data/slime/value.h index 786f2bede91..930c182924e 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.h +++ b/vespalib/src/vespa/vespalib/data/slime/value.h @@ -21,7 +21,7 @@ struct ObjectTraverser; class Value : public Cursor { protected: - virtual ~Value() {} + virtual ~Value() = default; virtual Cursor &addLeaf(const ValueFactory &input); virtual Cursor &setLeaf(Symbol symbol, const ValueFactory &input); @@ -57,7 +57,7 @@ public: Cursor &addString(Memory str) override; Cursor &addData(Memory data) override; Cursor &addData(ExternalMemory::UP data) override; - Cursor &addArray() override; + Cursor &addArray(size_t reserved_size) override; Cursor &addObject() override; Cursor &setNix(Symbol sym) override; @@ -67,7 +67,7 @@ public: Cursor &setString(Symbol sym, Memory str) override; Cursor &setData(Symbol sym, Memory data) override; Cursor &setData(Symbol sym, ExternalMemory::UP data) override; - Cursor &setArray(Symbol sym) override; + Cursor &setArray(Symbol sym, size_t reserved_size) override; Cursor &setObject(Symbol sym) override; Cursor &setNix(Memory name) override; @@ -77,7 +77,7 @@ public: Cursor &setString(Memory name, Memory str) override; Cursor &setData(Memory name, Memory str) override; Cursor &setData(Memory name, ExternalMemory::UP data) override; - Cursor &setArray(Memory name) override; + Cursor &setArray(Memory name, size_t reserved_size) override; Cursor &setObject(Memory name) override; Symbol resolve(Memory symbol_name) override; diff --git a/vespalib/src/vespa/vespalib/data/slime/value_factory.h b/vespalib/src/vespa/vespalib/data/slime/value_factory.h index 484246cf9e7..b3072778f2a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/value_factory.h @@ -14,7 +14,7 @@ class Value; **/ struct ValueFactory { virtual Value *create(Stash & stash) const = 0; - virtual ~ValueFactory() {} + virtual ~ValueFactory() = default; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/stllike/vector_map.h b/vespalib/src/vespa/vespalib/stllike/vector_map.h index f0cc97ecfae..d9c5fd44bea 100644 --- a/vespalib/src/vespa/vespalib/stllike/vector_map.h +++ b/vespalib/src/vespa/vespalib/stllike/vector_map.h @@ -28,18 +28,23 @@ public: typedef K key_type; typedef V mapped_type; private: - typedef std::vector< value_type> OrderedList; + using OrderedList = std::vector<value_type>; friend bool operator < (const std::pair<K, V> & a, const std::pair<K, V> & b) { LT lt; return lt(a.first, b.first); } - LT _lt; + struct KeyOrder { + bool operator () (const std::pair<K, V> & a, const K & b) { + LT lt; + return lt(a.first, b); + } + }; OrderedList _ht; public: - typedef typename OrderedList::iterator iterator; - typedef typename OrderedList::const_iterator const_iterator; + using iterator = typename OrderedList::iterator; + using const_iterator = typename OrderedList::const_iterator; public: - vector_map(size_t reserveSize=0) : _ht(reserveSize) { } + vector_map() : _ht() { } iterator begin() { return _ht.begin(); } iterator end() { return _ht.end(); } const_iterator begin() const { return _ht.begin(); } @@ -47,26 +52,17 @@ public: size_t capacity() const { return _ht.capacity(); } size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } - V & operator [] (const K & key) const { return _ht.find(key)->second; } - V & operator [] (const K & key) { - value_type v(key, V()); - LT lt; - iterator f = std::lower_bound(begin(), end(), v); - if ((f == end()) || lt(key, f->first)) { - f = _ht.insert(f, v); - } - return f->second; - } + V & operator [] (const K & key); void erase(const K & key) { return _ht.erase(find(key)); } void erase(iterator it) { return _ht.erase(it); } void erase(const_iterator it) { return _ht.erase(it); } iterator find(const K & key) { - iterator f = std::lower_bound(begin(), end(), value_type(key, V())); + iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); LT lt; return ((f != end()) && !lt(key, f->first)) ? f : end(); } const_iterator find(const K & key) const { - const_iterator f = std::lower_bound(begin(), end(), value_type(key, V())); + const_iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); LT lt; return ((f != end()) && !lt(key, f->first)) ? f : end(); } @@ -82,6 +78,19 @@ void swap(vector_map<K, V, LT> & a, vector_map<K, V, LT> & b) a.swap(b); } +template< typename K, typename V, typename LT > +V & +vector_map<K, V, LT>::operator[](const K &key) { + LT lt; + iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); + if (f == end()) { + _ht.template emplace_back(key, V()); + return _ht.rbegin()->second; + } else if (lt(key, f->first)) { + f = _ht.template emplace(f, key, V()); + } + return f->second; +} } diff --git a/yolean/abi-spec.json b/yolean/abi-spec.json index 82bf59ebf87..85aaaf5f64e 100644 --- a/yolean/abi-spec.json +++ b/yolean/abi-spec.json @@ -169,6 +169,7 @@ ], "methods": [ "public void <init>(com.yahoo.yolean.concurrent.ResourceFactory)", + "public void <init>(java.util.function.Supplier)", "public final java.lang.Object alloc()", "public final void free(java.lang.Object)", "public java.util.Iterator iterator()" @@ -209,7 +210,8 @@ ], "methods": [ "public void <init>()", - "public abstract java.lang.Object create()" + "public abstract java.lang.Object create()", + "public final java.util.function.Supplier asSupplier()" ], "fields": [] }, diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java index 24d4cfe4318..a00de4866d0 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java @@ -4,24 +4,41 @@ package com.yahoo.yolean.concurrent; import java.util.Iterator; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Supplier; /** + * A pool of a resource. This create new instances of the resource on request until enough are created + * to deliver a unique one to all threads needing one concurrently and then reuse those instances + * in subsequent requests. + * * @author baldersheim */ public class ConcurrentResourcePool<T> implements Iterable<T> { private final Queue<T> pool = new ConcurrentLinkedQueue<>(); - private final ResourceFactory<T> factory; + private final Supplier<T> factory; + // TODO: Deprecate public ConcurrentResourcePool(ResourceFactory<T> factory) { + this.factory = factory.asSupplier(); + } + + public ConcurrentResourcePool(Supplier<T> factory) { this.factory = factory; } + /** + * Allocates an instance of the resource to the requestor. + * The resource will be allocated exclusively to the requestor until it calls free(instance). + * + * @return a reused or newly created instance of the resource + */ public final T alloc() { - final T e = pool.poll(); - return e != null ? e : factory.create(); + T e = pool.poll(); + return e != null ? e : factory.get(); } + /** Frees an instance previously acquired bty alloc */ public final void free(T e) { pool.offer(e); } @@ -30,4 +47,5 @@ public class ConcurrentResourcePool<T> implements Iterable<T> { public Iterator<T> iterator() { return pool.iterator(); } + } diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java index f926283a47f..3a99b189ed8 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java @@ -1,11 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.yolean.concurrent; +import java.util.function.Supplier; + /** * @author baldersheim - * @since 5.2 */ +// TODO: Deprecate public abstract class ResourceFactory<T> { public abstract T create(); + + public final Supplier<T> asSupplier() { + return () -> create(); + } + } diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 25cc678fe66..7140c490460 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -50,6 +50,12 @@ <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -57,19 +63,27 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <!-- Needed to have the same version as slf4j-api --> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> - </dependency> - <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.2 due to @@ -79,6 +93,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 9a6ef3f74f1..414a18eb56d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; import com.google.inject.Inject; @@ -54,7 +54,10 @@ public class Curator implements VespaCurator, AutoCloseable { private static final Logger LOG = Logger.getLogger(Curator.class.getName()); private static final File ZK_CLIENT_CONFIG_FILE = new File(Defaults.getDefaults().underVespaHome("conf/zookeeper/zookeeper-client.cfg")); - private static final Duration ZK_SESSION_TIMEOUT = Duration.ofSeconds(30); + + // Note that session timeout has min and max values are related to tickTime defined by server, see configserver.def + private static final Duration ZK_SESSION_TIMEOUT = Duration.ofSeconds(120); + private static final Duration ZK_CONNECTION_TIMEOUT = Duration.ofSeconds(30); private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1); private static final int MAX_RETRIES = 10; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java index dd49b2595a4..2167b45bc02 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java @@ -1,6 +1,7 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.recipes; +import com.yahoo.path.Path; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.atomic.AtomicValue; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; @@ -9,15 +10,14 @@ import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; * A distributed atomic counter. * * @author Ulf Lilleengen - * @since 5.1 */ public class CuratorCounter { private final DistributedAtomicLong counter; - private final String counterPath; + private final Path counterPath; - public CuratorCounter(Curator curator, String counterPath) { - this.counter = curator.createAtomicCounter(counterPath); + public CuratorCounter(Curator curator, Path counterPath) { + this.counter = curator.createAtomicCounter(counterPath.getAbsolute()); this.counterPath = counterPath; } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java index 6b85953a1ff..3d465c6b71f 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; import org.junit.Test; + import static org.junit.Assert.assertEquals; /** @@ -14,9 +15,9 @@ public class CuratorCounterTest { @Test public void testCounter() throws Exception { DistributedAtomicLong counter = new MockCurator().createAtomicCounter("/mycounter"); - counter.initialize(4l); - assertEquals(4l, counter.get().postValue().longValue()); - assertEquals(5l, counter.increment().postValue().longValue()); + counter.initialize(4L); + assertEquals(4L, counter.get().postValue().longValue()); + assertEquals(5L, counter.increment().postValue().longValue()); } } diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml index 2f0029169ea..e40ba8fa161 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml @@ -33,15 +33,24 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.6.2</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.2 due to @@ -51,6 +60,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml index a4568d3585f..3074f3d2117 100644 --- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.3/pom.xml @@ -33,15 +33,24 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.6.3</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.3 due to @@ -51,6 +60,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java index c40b7cb7b52..ae84ff12b3b 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java @@ -70,6 +70,7 @@ public class ConfiguratorTest { builder.server(newServer(2, "baz", 345, 543, true)); builder.myidFile(idFile.getAbsolutePath()); builder.myid(1); + builder.tickTime(1234); new Configurator(builder.build()).writeConfigToDisk(VespaTlsConfig.tlsDisabled()); validateConfigFileMultipleHosts(cfgFile); validateIdFile(idFile, "1\n"); @@ -127,6 +128,7 @@ public class ConfiguratorTest { builder.myidFile(idFile.getAbsolutePath()); builder.server(newServer(0, "foo", 123, 321, false)); builder.myid(0); + builder.tickTime(1234); return builder; } @@ -146,7 +148,7 @@ public class ConfiguratorTest { } private String commonConfig() { - return "tickTime=2000\n" + + return "tickTime=1234\n" + "initLimit=20\n" + "syncLimit=15\n" + "maxClientCnxns=0\n" + |
