diff options
32 files changed, 139 insertions, 115 deletions
diff --git a/application-model/pom.xml b/application-model/pom.xml index 2143f3a5ffd..c47fccc51bc 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -22,6 +22,10 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> @@ -39,6 +43,11 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java index f8faf655415..87665efc1ef 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.compress; +package com.yahoo.vespa.archive; import com.yahoo.path.Path; import com.yahoo.yolean.Exceptions; diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java b/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java new file mode 100644 index 00000000000..51b9d930ae6 --- /dev/null +++ b/application-model/src/main/java/com/yahoo/vespa/archive/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.archive; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java index b7f019282b7..78ff2a805e5 100644 --- a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java +++ b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java @@ -1,6 +1,6 @@ -package com.yahoo.compress; +package com.yahoo.vespa.archive; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.yolean.Exceptions; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 6b63182627b..3fce00c3b16 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -187,7 +187,6 @@ <include>org.antlr:antlr-runtime:3.5.2:test</include> <include>org.antlr:antlr4-runtime:4.9.3:test</include> <include>org.apache.commons:commons-exec:1.3:test</include> - <include>org.apache.commons:commons-compress:1.21:test</include> <include>org.apache.commons:commons-math3:3.6.1:test</include> <include>org.apache.felix:org.apache.felix.framework:${felix.version}:test</include> <include>org.apache.felix:org.apache.felix.log:1.0.1:test</include> diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 43d4d80bb9d..023169a1c4a 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -201,7 +201,6 @@ <i>com.google.inject:guice:jar:no_aop:*:*</i> <i>com.google.j2objc:j2objc-annotations:*:*</i> <i>com.google.protobuf:protobuf-java:*:*</i> - <i>com.microsoft.onnxruntime:onnxruntime:*:*</i> <i>com.sun.activation:javax.activation:*:*</i> <i>com.sun.xml.bind:jaxb-core:*:*</i> <i>com.sun.xml.bind:jaxb-impl:*:*</i> @@ -211,13 +210,9 @@ <i>io.prometheus:simpleclient_common:*:*</i> <i>javax.inject:javax.inject:*:*</i> <i>javax.servlet:javax.servlet-api:*:*</i> - <i>net.java.dev.jna:jna:*:*</i> <i>net.openhft:zero-allocation-hashing:*:*</i> <i>org.antlr:antlr-runtime:*:*</i> <i>org.antlr:antlr4-runtime:*:*</i> - <i>org.apache.commons:commons-compress:*:*</i> - <i>org.apache.commons:commons-exec:*:*</i> - <i>org.apache.commons:commons-math3:*:*</i> <i>org.apache.felix:org.apache.felix.framework:*:*</i> <i>org.bouncycastle:bcpkix-jdk18on:*:*</i> <i>org.bouncycastle:bcprov-jdk18on:*:*</i> diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java index 27e6d6e57a6..c254385a96e 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java @@ -123,9 +123,13 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ rawRankProfiles.put(rawRank.getName(), rawRank); } return rawRankProfiles; - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException e) { throw new IllegalStateException(e); } + catch (ExecutionException e) { + throw e.getCause() instanceof IllegalArgumentException ? (IllegalArgumentException)e.getCause() + : new IllegalStateException(e); + } } private static FileDistributedConstants deriveFileDistributedConstants(Schema schema, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index f7d9c2f4a0d..93b67407933 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -226,8 +226,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private Optional<Admin> getAdmin() { if (parent != null) { AbstractConfigProducerRoot r = parent.getRoot(); - if (r instanceof VespaModel) { - VespaModel model = (VespaModel) r; + if (r instanceof VespaModel model) { return Optional.ofNullable(model.getAdmin()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 01dd47765d2..3bad813ef4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 1e45771d988..711afe72c62 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -84,10 +84,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - <exclusion> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> @@ -164,10 +160,6 @@ <artifactId>zero-allocation-hashing</artifactId> </exclusion> <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </exclusion> - <exclusion> <groupId>org.lz4</groupId> <artifactId>lz4-java</artifactId> </exclusion> @@ -188,14 +180,6 @@ <artifactId>icu4j</artifactId> </exclusion> <exclusion> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - <exclusion> <groupId>org.antlr</groupId> <artifactId>antlr4-runtime</artifactId> </exclusion> @@ -244,17 +228,6 @@ Excluded artifacts should be added explicitly to the application module to make then visible in users' test classpath. --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>document</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>opennlp-linguistics</artifactId> <version>${project.version}</version> <exclusions> diff --git a/container-test/pom.xml b/container-test/pom.xml index 7dcbc794b77..103c118b083 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -99,11 +99,6 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <scope>compile</scope> - </dependency> <dependency> <!-- TODO: Remove on Vespa 9 --> <!-- not used by Vespa, but was historically on test classpath --> <groupId>org.json</groupId> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 53c78d7c8ec..8dbec87828a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -5,9 +5,9 @@ import com.google.common.hash.Funnel; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.yahoo.component.Version; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.file.NoSuchFileException; @@ -38,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index 185c97f866e..d22a41f74a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -1,27 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; /** * A list of entries read from a ZIP archive, and their contents. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 526bf0c53b3..64dd51ce304 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -138,6 +138,7 @@ public class Notifier { .map(Notifier::linkify) .map(m -> "<li>" + m + "</li>") .collect(Collectors.joining())) + .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source())) .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant())) .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html") .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html")) @@ -170,4 +171,23 @@ public class Notifier { return new UriBuilder(dashboardUri).append(path).toString(); } + private String notificationLink(NotificationSource source) { + var uri = new UriBuilder(dashboardUri); + uri = uri.append("tenant").append(source.tenant().value()); + if (source.application().isPresent()) + uri = uri.append("application").append(source.application().get().value()); + if (source.isProduction()) { + uri = uri.append("prod/instance"); + if (source.jobType().isPresent()) { + uri = uri.append(source.instance().get().value()); + } + } + else { + uri = uri.append("dev/instance/").append(source.instance().get().value()); + } + if (source.jobType().isPresent()) { + uri = uri.append("job").append(source.jobType().get().jobName()).append("run").append(String.valueOf(source.runNumber().getAsLong())); + } + return uri.toString(); + } } diff --git a/controller-server/src/main/resources/mail/mail-notification.tmpl b/controller-server/src/main/resources/mail/mail-notification.tmpl index 868a3a76bab..8c02c05723f 100644 --- a/controller-server/src/main/resources/mail/mail-notification.tmpl +++ b/controller-server/src/main/resources/mail/mail-notification.tmpl @@ -486,7 +486,7 @@ valign="middle" > <a - href="#" + href="[[LINK_TO_NOTIFICATION]]" style=" display: inline-block; background: #005a8e; diff --git a/controller-server/src/test/resources/mail/notification.txt b/controller-server/src/test/resources/mail/notification.txt index 5ca26cb4753..9f1583d7674 100644 --- a/controller-server/src/test/resources/mail/notification.txt +++ b/controller-server/src/test/resources/mail/notification.txt @@ -486,7 +486,7 @@ valign="middle" > <a - href="#" + href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance" style=" display: inline-block; background: #005a8e; diff --git a/docprocs/pom.xml b/docprocs/pom.xml index e001d83a00f..6d654deea44 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -57,10 +57,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> </exclusion> - <exclusion> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </exclusion> </exclusions> <!-- End Workaround --> diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 5bb4b806ffb..ff77b5e9601 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -73,6 +73,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>model-integration</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (60 MB) and not needed for config generation --> + <groupId>com.microsoft.onnxruntime</groupId> + <artifactId>onnxruntime</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -93,6 +100,16 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -161,6 +178,11 @@ <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> </exclusion> + <exclusion> + <!-- OPTIMIZATION: large (4 MB) and only used for query dispatch TDistribution--> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 7916698b62c..97b17cc00d8 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -22,7 +22,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-apache-http-client-bundle</artifactId> <version>${project.version}</version> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -34,11 +34,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -55,6 +57,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>junit</groupId> @@ -64,6 +67,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>io.airlift</groupId> @@ -72,6 +76,7 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.slf4j</groupId> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index f9ecaa29153..67c888a8d77 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -23,7 +23,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions; -import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors; import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toList; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java index def61006f15..0c25f31f266 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java @@ -62,7 +62,7 @@ public class ExternalMetrics { private Set<MetricId> metricsToRetain() { return consumers.getConsumersByMetric().keySet().stream() - .map(configuredMetric -> configuredMetric.id()) + .map(ConfiguredMetric::id) .collect(toCollection(LinkedHashSet::new)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java index fb789874acf..e0e11ad0a3a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java @@ -19,15 +19,17 @@ public class Container extends PartialContainer { private final ContainerResources resources; private final int conmonPid; private final List<Network> networks; + private final List<String> createCommand; public Container(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image, Map<String, String> labels, int pid, int conmonPid, String hostname, - ContainerResources resources, List<Network> networks, boolean managed) { + ContainerResources resources, List<Network> networks, boolean managed, List<String> createCommand) { super(id, name, createdAt, state, imageId, image, labels, pid, managed); this.hostname = Objects.requireNonNull(hostname); this.resources = Objects.requireNonNull(resources); this.conmonPid = conmonPid; this.networks = List.copyOf(Objects.requireNonNull(networks)); + this.createCommand = List.copyOf(Objects.requireNonNull(createCommand)); } /** The hostname of this, if any */ @@ -50,18 +52,23 @@ public class Container extends PartialContainer { return networks; } + /** The command used to create this */ + public List<String> createCommand() { + return createCommand; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; - Container that = (Container) o; - return conmonPid == that.conmonPid && hostname.equals(that.hostname) && resources.equals(that.resources) && networks.equals(that.networks); + Container container = (Container) o; + return conmonPid == container.conmonPid && hostname.equals(container.hostname) && resources.equals(container.resources) && networks.equals(container.networks) && createCommand.equals(container.createCommand); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks); + return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks, createCommand); } /** The network of a container */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java index 2aa1d12c491..630509cf482 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java @@ -35,6 +35,9 @@ public interface ContainerEngine { /** Remove given container. The container will be stopped if necessary */ void removeContainer(TaskContext context, PartialContainer container); + /** Returns whether the given container should be re-created to apply new configuration */ + boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources); + /** Get container for given context */ Optional<Container> getContainer(NodeAgentContext context); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 9060261b806..e632acd5223 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -57,6 +57,10 @@ public class ContainerOperations { containerEngine.updateContainer(context, containerId, containerResources); } + public boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources) { + return containerEngine.shouldRecreate(context, container, wantedResources); + } + public Optional<Container> getContainer(NodeAgentContext context) { return containerEngine.getContainer(context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 20ea29381f3..d46325e4bca 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -332,34 +332,34 @@ public class NodeAgentImpl implements NodeAgent { } private List<String> shouldRemoveContainer(NodeAgentContext context, Container existingContainer) { - final NodeState nodeState = context.node().state(); + NodeState nodeState = context.node().state(); List<String> reasons = new ArrayList<>(); - if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) + if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) { reasons.add("Node in state " + nodeState + ", container should no longer be running"); - + } if (context.node().wantedDockerImage().isPresent() && !context.node().wantedDockerImage().get().equals(existingContainer.image())) { - reasons.add("The node is supposed to run a new Docker image: " + reasons.add("The node is supposed to run a new image: " + existingContainer.image().asString() + " -> " + context.node().wantedDockerImage().get().asString()); } - - if (!existingContainer.state().isRunning()) + if (!existingContainer.state().isRunning()) { reasons.add("Container no longer running"); - + } if (currentRebootGeneration < context.node().wantedRebootGeneration()) { reasons.add(String.format("Container reboot wanted. Current: %d, Wanted: %d", currentRebootGeneration, context.node().wantedRebootGeneration())); } - ContainerResources wantedContainerResources = getContainerResources(context); if (!wantedContainerResources.equalsMemory(existingContainer.resources())) { reasons.add("Container should be running with different memory allocation, wanted: " + wantedContainerResources.toStringMemory() + ", actual: " + existingContainer.resources().toStringMemory()); } - - if (containerState == STARTING) + if (containerOperations.shouldRecreate(context, existingContainer, wantedContainerResources)) { + reasons.add("Container should be re-created to apply new configuration"); + } + if (containerState == STARTING) { reasons.add("Container failed to start"); - + } return reasons; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 2d3a4976fe5..744d78d83da 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -28,6 +28,7 @@ public class ContainerEngineMock implements ContainerEngine { private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); + private boolean asyncImageDownload = false; public ContainerEngineMock asyncImageDownload(boolean enabled) { @@ -112,6 +113,11 @@ public class ContainerEngineMock implements ContainerEngine { } @Override + public boolean shouldRecreate(NodeAgentContext context, Container container, ContainerResources wantedResources) { + return false; + } + + @Override public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { Container container = requireContainer(context.containerName()); containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), @@ -119,7 +125,8 @@ public class ContainerEngineMock implements ContainerEngine { container.labels(), container.pid(), container.conmonPid(), container.hostname(), containerResources, container.networks(), - container.managed())); + container.managed(), + container.createCommand())); } @Override @@ -200,7 +207,8 @@ public class ContainerEngineMock implements ContainerEngine { context.hostname().value(), containerResources, List.of(), - true); + true, + List.of()); } private static class ImageDownload { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index 9a5ca8c805e..8fb65e4bd47 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -59,7 +59,7 @@ public class ContainerOperationsTest { private Container createContainer(String name, boolean managed) { return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, - ContainerResources.UNLIMITED, List.of(), managed); + ContainerResources.UNLIMITED, List.of(), managed, List.of()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java index 2ef6780dff6..0d83a397d33 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java @@ -123,7 +123,7 @@ public class ContainerImagePrunerTest { return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, Container.State.running, imageId, DockerImage.EMPTY, Map.of(), 42, 43, name + ".example.com", ContainerResources.UNLIMITED, - List.of(), true); + List.of(), true, List.of()); } private static class Tester { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index fb132c9b717..6e980b26cdf 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -358,6 +358,15 @@ public class NodeAgentImplTest { verify(orchestrator, times(1)).resume(eq(hostName)); verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() .withRebootGeneration(wantedRebootGeneration))); + + // Re-create if new container config needs to be applied + when(containerOperations.shouldRecreate(eq(context), any(), any())).thenReturn(true); + nodeAgent.doConverge(context); + verify(containerOperations, times(2)).removeContainer(eq(context), any()); + verify(containerOperations, times(2)).createContainer(eq(context), any()); + verify(orchestrator, times(2)).resume(eq(hostName)); + verify(nodeRepository, times(2)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(wantedRebootGeneration))); } @Test @@ -815,7 +824,8 @@ public class NodeAgentImplTest { hostName, containerResources, List.of(), - true)) : + true, + List.of())) : Optional.empty(); }).when(containerOperations).getContainer(any()); } diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index b50f14558d6..cf7898f6951 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -84,15 +84,11 @@ <version>${vespaversion}</version> <scope>test</scope> <exclusions> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> @@ -115,10 +111,6 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-exec</artifactId> </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> </exclusions> </dependency> <dependency> diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index 3e5e7ca8025..230ddc5c7d3 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -46,12 +46,6 @@ </exclusions> </dependency> <dependency> - <!-- Added explicitly to get commons-collections, which is excluded from container-dev --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>document</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>documentapi</artifactId> <version>${project.version}</version> diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 4c57b615c16..d903fb5ec0d 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -36,10 +36,6 @@ <artifactId>aircompressor</artifactId> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </dependency> - <dependency> <groupId>net.openhft</groupId> <artifactId>zero-allocation-hashing</artifactId> <exclusions> |