diff options
author | Harald Musum <musum@yahooinc.com> | 2023-05-08 10:35:07 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-08 10:35:07 +0200 |
commit | 5685f5e49b719526c930d35ffaf2d7f945b552a0 (patch) | |
tree | cce00a3bebb2ae78db8ad8101513f5999f0814c9 /config-model/src/main/java/com | |
parent | 5f3d9b2a6a3b1c58a8a295189208c4be262e60d2 (diff) |
Revert "Remove access logging for internal container clusters"
Diffstat (limited to 'config-model/src/main/java/com')
8 files changed, 41 insertions, 58 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 4b1467158ef..33915d48bd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -23,7 +24,11 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain addLogHandler(); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); if (isHostedVespa()) - addAccessLog(getName()); + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of(getName()), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 14b18cd8711..7b465e3eb48 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import java.nio.file.Path; import java.util.Collections; @@ -33,7 +34,11 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC this.reindexingContext = createReindexingContext(deployState); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); if (isHostedVespa()) - addAccessLog("controller"); + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of("controller"), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 58e624960f9..78374cd4030 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -86,10 +87,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC static final String LEGACY_APPLICATION = "app"; // app.instance } + private final TreeConfigProducer<?> parent; private final ApplicationId applicationId; public MetricsProxyContainerCluster(TreeConfigProducer<?> parent, String name, DeployState deployState) { super(parent, name, name, deployState, true); + this.parent = parent; applicationId = deployState.getProperties().applicationId(); setRpcServerEnabled(true); @@ -98,7 +101,11 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); addClusterComponents(); if (isHostedVespa()) - addAccessLog("metrics-proxy"); + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + "zstd", + Optional.of("metrics-proxy"), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 6977a5ca465..d101cea428e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -336,12 +336,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat protected boolean messageBusEnabled() { return messageBusEnabled; } - public void addAccessLog() { - // In hosted there is one application container per node, so we do not use the container name to distinguish log files - Optional<String> clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); - addAccessLog(clusterName); - } - public void addMbusServer(ComponentId chainId) { ComponentId serviceId = chainId.nestInNamespace(ComponentId.fromString("MbusServer")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6bbc24e8739..45060f2f27d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -26,7 +26,6 @@ import com.yahoo.container.jdisc.state.StateHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; @@ -74,7 +73,6 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; /** @@ -150,7 +148,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final Set<Path> platformBundles = new TreeSet<>(); // Ensure stable ordering private final ComponentGroup<Component<?, ?>> componentGroup; - protected final boolean isHostedVespa; + private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; private final String compressionType; @@ -171,7 +169,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> public ContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity) { this(parent, configSubId, clusterId, deployState, zooKeeperLocalhostAffinity, 1); } - public ContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity, int defaultPoolNumThreads) { super(parent, configSubId); this.name = clusterId; @@ -183,7 +180,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> componentGroup = new ComponentGroup<>(this, "component"); addCommonVespaBundles(); - addSimpleComponent(VoidRequestLog.class); + addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); addComponent(defaultHandlerThreadpool); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); @@ -290,11 +287,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> return componentGroup.removeComponent(componentId); } - public void removeSimpleComponent(Class<?> clazz) { - removeComponent(new SimpleComponent(clazz.getName()).getComponentId()); - } - - public void addSimpleComponent(Class<?> clazz) { + private void addSimpleComponent(Class<?> clazz) { addSimpleComponent(clazz.getName()); } @@ -600,23 +593,12 @@ public abstract class ContainerCluster<CONTAINER extends Container> if (containerSearch != null) containerSearch.connectSearchClusters(clusterMap); } - public void addAccessLog(String clusterName) { - addAccessLog(Optional.ofNullable(clusterName)); + public void addDefaultSearchAccessLog() { + // In hosted Vespa with one application container per node we do not use the container name to distinguish log files + Optional<String> clusterName = isHostedVespa ? Optional.empty() : Optional.of(getName()); + addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, clusterName, isHostedVespa)); } - public void addAccessLog(String fileNamePattern, String symlinkName) { - removeSimpleComponent(VoidRequestLog.class); - addSimpleComponent(AccessLog.class); - addComponent(new AccessLogComponent(jsonAccessLog, compressionType, fileNamePattern, null, true, true, symlinkName, 1024, 256 * 1024)); - } - - protected void addAccessLog(Optional<String> clusterName) { - removeSimpleComponent(VoidRequestLog.class); - addSimpleComponent(AccessLog.class); - addComponent(new AccessLogComponent(this, jsonAccessLog, compressionType, clusterName, isHostedVespa)); - } - - @Override public void getConfig(IlscriptsConfig.Builder builder) { for (SearchCluster searchCluster : Content.getSearchClusters(getRoot().configModelRepo())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 883dbebd34d..49361b887c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.component.AccessLogComponent; @@ -14,8 +12,6 @@ import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; - /** * Builds the config model for the standalone config server. * @@ -49,7 +45,10 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { @Override protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { if (isHosted()){ - cluster.addAccessLog("logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", "access-json.log"); + cluster.addComponent( + new AccessLogComponent( + AccessLogComponent.AccessLogType.jsonAccessLog, "zstd", + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { super.addAccessLogs(deployState, cluster, spec); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3d1c8ca1d76..a1e88092195 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -36,10 +36,8 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.io.IOUtils; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.path.Path; import com.yahoo.schema.OnnxModel; @@ -419,26 +417,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); - if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) { - cluster.addAccessLog(); + if (cluster.isHostedVespa() && !accessLogElements.isEmpty()) { + accessLogElements.clear(); + log.logApplicationPackage( + Level.WARNING, "Applications are not allowed to override the 'accesslog' element"); } else { - if (cluster.isHostedVespa()) { - log.logApplicationPackage(WARNING, "Applications are not allowed to override the 'accesslog' element"); - } else { - List<AccessLogComponent> components = new ArrayList<>(); - for (Element accessLog : accessLogElements) { - AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(accessLogComponent -> { - components.add(accessLogComponent); - cluster.addComponent(accessLogComponent); - }); - } - if (components.size() > 0) { - cluster.removeSimpleComponent(VoidRequestLog.class); - cluster.addSimpleComponent(AccessLog.class); - } + for (Element accessLog : accessLogElements) { + AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } } + if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) + cluster.addDefaultSearchAccessLog(); + // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "access")); @@ -448,6 +439,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return XML.getChildren(spec, "accesslog"); } + protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element httpElement = XML.getChild(spec, "http"); if (httpElement != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index e044b97546c..519fc967511 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -311,7 +311,6 @@ public class Content extends ConfigModel { indexingCluster.addDefaultHandlersWithVip(); indexingCluster.addAllPlatformBundles(); - indexingCluster.addAccessLog(); addDocproc(indexingCluster); List<ApplicationContainer> nodes = new ArrayList<>(); |