diff options
6 files changed, 81 insertions, 26 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index a7f3a6224f2..f7007fec181 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -7,8 +7,12 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; +import java.nio.file.Path; +import java.util.Collections; import java.util.Optional; +import java.util.Set; /** * Container cluster for cluster-controller containers. @@ -18,6 +22,8 @@ import java.util.Optional; */ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { + private static final Set<Path> UNNECESSARY_BUNDLES = Collections.unmodifiableSet(PlatformBundles.VESPA_SECURITY_BUNDLES); + private final ReindexingContext reindexingContext; public ClusterControllerContainerCluster( @@ -29,6 +35,9 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC } @Override + protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; } + + @Override protected void doPrepare(DeployState deployState) { } @Override protected boolean messageBusEnabled() { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 680a4b97f86..928630214f4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; @@ -74,6 +75,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private static final String METRICS_PROXY_NAME = "metrics-proxy"; static final Path METRICS_PROXY_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(METRICS_PROXY_NAME); static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; + private static final Set<Path> UNNECESSARY_BUNDLES = Collections.unmodifiableSet(PlatformBundles.VESPA_SECURITY_BUNDLES); static final class AppDimensionNames { static final String SYSTEM = "system"; @@ -98,6 +100,9 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addClusterComponents(); } + @Override + protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; } + private void addClusterComponents() { addMetricsProxyComponent(ApplicationDimensions.class); addMetricsProxyComponent(ConfigSentinelClient.class); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index baeac48e4e8..32bdf6e182e 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -194,6 +194,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent(com.yahoo.jdisc.http.server.jetty.Janitor.class); } + protected abstract boolean messageBusEnabled(); + public ClusterSpec.Id id() { return ClusterSpec.Id.from(getName()); } public void setZone(Zone zone) { @@ -455,18 +457,20 @@ public abstract class ContainerCluster<CONTAINER extends Container> } /** - * Adds the Vespa bundles that are necessary for all container types. + * Adds the Vespa bundles that are necessary for most container types. + * Note that some of these can be removed later by the individual cluster types. */ public void addCommonVespaBundles() { PlatformBundles.COMMON_VESPA_BUNDLES.forEach(this::addPlatformBundle); + PlatformBundles.VESPA_SECURITY_BUNDLES.forEach(this::addPlatformBundle); } - /* - Add all search/docproc/feed related platform bundles. - This is only required for application configured containers as the platform bundle set is not allowed to change - between config generations. For standalone container platform bundles can be added on features enabled as an - update of application package requires restart. - */ + /** + * Add all search/docproc/feed related platform bundles. + * These are only required for application configured containers as the platform bundle set is not allowed to change + * between config generations. For standalone container platform bundles can be added on features enabled as an + * update of application package requires restart. + */ public void addAllPlatformBundles() { ContainerDocumentApi.addVespaClientContainerBundle(this); addSearchAndDocprocBundles(); @@ -481,9 +485,19 @@ public abstract class ContainerCluster<CONTAINER extends Container> * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar' */ public final void addPlatformBundle(Path bundlePath) { - platformBundles.add(bundlePath); + if (! unnecessaryPlatformBundles().contains(bundlePath)) { + platformBundles.add(bundlePath); + } else { + log.fine(() -> "Not installing bundle " + bundlePath + " for cluster " + getName()); + } } + /** + * Implement in subclasses to avoid installing unnecessary bundles, see {@link PlatformBundles} + * Should only return constant values, as there is no guarantee for when this is called. + */ + protected Set<Path> unnecessaryPlatformBundles() { return Set.of(); } + @Override public void getConfig(PlatformBundlesConfig.Builder builder) { platformBundles.stream() @@ -645,8 +659,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> return "container cluster '" + getName() + "'"; } - protected abstract boolean messageBusEnabled(); - /** * Mark whether the config emitted by this cluster currently should be applied by clients already running with * a previous generation of it only by restarting the consuming processes. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index d7a4af7145b..a93fe6967b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -33,18 +33,28 @@ public class PlatformBundles { public static final String SEARCH_AND_DOCPROC_BUNDLE = BundleInstantiationSpecification.CONTAINER_SEARCH_AND_DOCPROC; // Bundles that must be loaded for all container types. - public static final Set<Path> COMMON_VESPA_BUNDLES = Stream.of( + public static final Set<Path> COMMON_VESPA_BUNDLES = toBundlePaths( "container-spifly.jar", // Aries SPIFly repackaged "zkfacade", "zookeeper-server" // TODO: not necessary in metrics-proxy. - ).map(PlatformBundles::absoluteBundlePath).collect(Collectors.toSet()); + ); + + public static final Set<Path> VESPA_SECURITY_BUNDLES = toBundlePaths( + "jdisc-security-filters", + "vespa-athenz"); - public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = Stream.of( + public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "container-search-gui", "docprocs", "linguistics-components" - ).map(PlatformBundles::absoluteBundlePath).collect(Collectors.toSet()); + ); + + private static Set<Path> toBundlePaths(String... bundleNames) { + return Stream.of(bundleNames) + .map(PlatformBundles::absoluteBundlePath) + .collect(Collectors.toSet()); + } public static Path absoluteBundlePath(String fileName) { return absoluteBundlePath(fileName, JarSuffix.JAR_WITH_DEPS); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 8012a00076b..793064af2dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -20,6 +20,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; @@ -29,6 +30,7 @@ import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; @@ -38,11 +40,14 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.StringReader; +import java.nio.file.Path; import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; +import static java.util.stream.Collectors.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -110,8 +115,16 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals(4000, cfg.storage_transition_time()); assertEquals(3600000, cfg.stable_state_time_period()); } + + assertOnlyNecessaryBundles(model); } + private void assertOnlyNecessaryBundles(VespaModel model) { + PlatformBundlesConfig config = model.getConfig(PlatformBundlesConfig.class, "admin/cluster-controllers"); + Set<String> unnecessaryBundles = PlatformBundles.VESPA_SECURITY_BUNDLES.stream().map(Path::toString).collect(toSet()); + assertTrue(config.bundlePaths().stream() + .noneMatch(unnecessaryBundles::contains)); + } @Test(expected = IllegalArgumentException.class) public void testSeparateHostsRequired() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index b3860459323..1b36ff0afe6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -15,12 +15,14 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import org.junit.Test; +import java.nio.file.Path; import java.util.Collection; -import java.util.stream.Collectors; +import java.util.Set; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_FILE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; @@ -35,8 +37,8 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.g import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -47,21 +49,25 @@ public class MetricsProxyContainerClusterTest { @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var builder = new PlatformBundlesConfig.Builder(); - model.getConfig(builder, CLUSTER_CONFIG_ID); - PlatformBundlesConfig config = builder.build(); - assertFalse(config.bundlePaths().stream() - .filter(p -> p.endsWith(METRICS_PROXY_BUNDLE_FILE.toString())) - .collect(Collectors.toList()) - .isEmpty()); + PlatformBundlesConfig config = model.getConfig(PlatformBundlesConfig.class, CLUSTER_CONFIG_ID); + assertTrue(config.bundlePaths().stream() + .anyMatch(p -> p.equals(METRICS_PROXY_BUNDLE_FILE.toString()))); + } + + @Test + public void unnecessary_bundles_are_not_installed() { + VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); + PlatformBundlesConfig config = model.getConfig(PlatformBundlesConfig.class, CLUSTER_CONFIG_ID); + + Set<String> unnecessaryBundles = PlatformBundles.VESPA_SECURITY_BUNDLES.stream().map(Path::toString).collect(toSet()); + assertTrue(config.bundlePaths().stream() + .noneMatch(unnecessaryBundles::contains)); } @Test public void cluster_is_prepared_so_that_application_metadata_config_is_produced() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var builder = new ApplicationMetadataConfig.Builder(); - model.getConfig(builder, CLUSTER_CONFIG_ID); - ApplicationMetadataConfig config = builder.build(); + ApplicationMetadataConfig config = model.getConfig(ApplicationMetadataConfig.class, CLUSTER_CONFIG_ID); assertEquals(MockApplicationPackage.APPLICATION_GENERATION, config.generation()); assertEquals(MockApplicationPackage.APPLICATION_NAME, config.name()); } |