Reapply "Turn off logging for containers for some classes"

This reverts commit 8a5f333b590f5632bc1d7aafdcea12ba9d6cafa4.

10 files changed, 93 insertions, 45 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 317c3646dde..f7bc04b6da1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AnyConfigProducer;
 import com.yahoo.config.model.producer.TreeConfigProducer;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.container.logging.LevelsModSpec;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.ConfigSentinel;
@@ -24,12 +25,12 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty;
 
@@ -41,8 +42,6 @@ import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty;
  */
 public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Serializable {
 
-    private static final long serialVersionUID = 1L;
-
     private final boolean isHostedVespa;
     private final Monitoring monitoring;
     private final List<Configserver> configservers = new ArrayList<>();
@@ -68,12 +67,6 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
     }
 
     private final List<LogctlSpec> logctlSpecs = new ArrayList<>();
-    public List<LogctlSpec> getLogctlSpecs() {
-        return logctlSpecs;
-    }
-    public void addLogctlCommand(String componentSpec, String levelsModSpec) {
-        logctlSpecs.add(new LogctlSpec(componentSpec,  levelsModSpec));
-    }
 
     /**
      * The single cluster controller cluster shared by all content clusters by default when not multitenant.
@@ -101,6 +94,7 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
         this.multitenant = multitenant;
         this.fileDistribution = new FileDistributionConfigProducer(parent);
         this.applicationType = applicationType;
+        this.logctlSpecs.addAll(defaultLogctlSpecs());
     }
 
     public Configserver getConfigserver() { return defaultConfigserver; }
@@ -331,4 +325,25 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
 
     public ApplicationType getApplicationType() { return applicationType; }
 
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+    public void addLogctlCommand(String componentSpec, String levelsModSpec) {
+        logctlSpecs.add(new LogctlSpec(componentSpec,  levelsModSpec));
+    }
+
+    private static Set<LogctlSpec> defaultLogctlSpecs() {
+        // Turn off info logging for all containers for some classes (unimportant log messages that create noise in vespa log)
+        return Set.of(new LogctlSpec("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator", getLevelModSpec("-info")),
+                      new LogctlSpec("org.eclipse.jetty.server.Server", getLevelModSpec("-info")),
+                      new LogctlSpec("org.eclipse.jetty.server.handler.ContextHandler", getLevelModSpec("-info")),
+                      new LogctlSpec("org.eclipse.jetty.server.AbstractConnector", getLevelModSpec("-info")));
+    }
+
+    static String getLevelModSpec(String levels) {
+        var levelSpec = new LevelsModSpec();
+        levelSpec.setLevels(levels);
+        return levelSpec.toLogctlModSpec();
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
index e50c010c476..fef7d534c30 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
@@ -25,9 +25,6 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain
     }
 
     @Override
-    protected void doPrepare(DeployState deployState) { }
-
-    @Override
     public void getConfig(QrStartConfig.Builder builder) {
         super.getConfig(builder);
         builder.jvm.heapsize(128);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
index 25d5620eff9..c65abfb0189 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
@@ -37,9 +37,6 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC
     @Override
     protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; }
 
-    @Override
-    protected void doPrepare(DeployState deployState) { }
-
     @Override protected boolean messageBusEnabled() { return false; }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index ec543dba6fb..ada647b535d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -137,9 +137,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
     }
 
     @Override
-    protected void doPrepare(DeployState deployState) { }
-
-    @Override
     public void getConfig(MetricsNodesConfig.Builder builder) {
         builder.node.addAll(MetricsNodesConfigGenerator.generate(getContainers()));
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index 57113ff794f..a29c0e69329 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -21,11 +21,8 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
 import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets;
 import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder;
 import org.w3c.dom.Element;
-
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 
 /**
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index bc410670d5e..f901bf3c826 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -9,11 +9,8 @@ import com.yahoo.config.model.producer.TreeConfigProducer;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.search.config.QrStartConfig;
-import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
-
 import java.time.Duration;
-import java.util.List;
 import java.util.Optional;
 
 /**
@@ -43,15 +40,6 @@ public final class ApplicationContainer extends Container implements
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider"));
     }
 
-    private List<LogctlSpec> logctlSpecs = List.of();
-    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
-        this.logctlSpecs = logctlSpecs;
-    }
-    @Override
-    public List<LogctlSpec> getLogctlSpecs() {
-        return logctlSpecs;
-    }
-
     @Override
     public void getConfig(QrStartConfig.Builder builder) {
         if (getHostResource() != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 004ff5516ba..d101cea428e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -124,19 +124,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                 : defaultHeapSizePercentageOfTotalNodeMemory;
     }
 
-    private void wireLogctlSpecs() {
-        getAdmin().ifPresent(admin -> {
-            for (var c : getContainers()) {
-                c.setLogctlSpecs(admin.getLogctlSpecs());
-            }});
-    }
-
     @Override
     protected void doPrepare(DeployState deployState) {
+        super.doPrepare(deployState);
         addAndSendApplicationBundles(deployState);
         sendUserConfiguredFiles(deployState);
         createEndpointList(deployState);
-        wireLogctlSpecs();
     }
 
     private void addAndSendApplicationBundles(DeployState deployState) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index be9f3fa894f..2ca6d5d7155 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -16,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.PortAllocBridge;
 import com.yahoo.vespa.model.application.validation.RestartConfigs;
 import com.yahoo.vespa.model.container.component.Component;
@@ -34,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -65,6 +67,8 @@ public abstract class Container extends AbstractService implements
     /** The cluster this container belongs to, or null if it is not added to any cluster */
     private ContainerCluster<?> owner = null;
 
+    private List<LogctlSpec> logctlSpecs = List.of();
+
     protected final TreeConfigProducer<?> parent;
     private final String name;
     private boolean requireSpecificPorts = true;
@@ -415,4 +419,11 @@ public abstract class Container extends AbstractService implements
         return producer instanceof ContainerCluster<?> ? (ContainerCluster<?>) producer : null;
     }
 
+    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
+        this.logctlSpecs = logctlSpecs;
+    }
+
+    @Override
+    public List<LogctlSpec> getLogctlSpecs() { return logctlSpecs; }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index d786135d3ee..45060f2f27d 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -61,7 +61,6 @@ import com.yahoo.vespa.model.container.search.ContainerSearch;
 import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
 import com.yahoo.vespa.model.content.Content;
 import com.yahoo.vespa.model.search.SearchCluster;
-
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -301,7 +300,16 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         doPrepare(deployState);
     }
 
-    protected abstract void doPrepare(DeployState deployState);
+    protected void doPrepare(DeployState deployState) {
+        wireLogctlSpecs();
+    }
+
+    private void wireLogctlSpecs() {
+        getAdmin().ifPresent(admin -> {
+            for (var c : getContainers()) {
+                c.setLogctlSpecs(admin.getLogctlSpecs());
+            }});
+    }
 
     public String getName() {
         return name;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
index d060fdfd3d2..657d685684d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
@@ -17,17 +17,22 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.config.core.StateserverConfig;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.Service;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
 import org.junit.jupiter.api.Test;
-
+import java.util.List;
 import java.util.Set;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 public class AdminTestCase {
@@ -243,4 +248,44 @@ public class AdminTestCase {
         assertTrue(configIds.contains("hosts/myhost0/logforwarder"), configIds.toString());
     }
 
+    @Test
+    void testDefaultLogCtlSpecs() {
+        String hosts = "<hosts>"
+                + "  <host name=\"myhost0\">"
+                + "    <alias>node0</alias>"
+                + "  </host>"
+                + "</hosts>";
+
+        String services = "<services>" +
+                "  <admin version='2.0'>" +
+                "    <adminserver hostalias='node0' />" +
+                "  </admin>" +
+                "  <container version=\"1.0\">" +
+                "    <nodes>" +
+                "      <node hostalias=\"node0\" />" +
+                "    </nodes>" +
+                "    <search/>" +
+                "    <document-api/>" +
+                "  </container>" +
+                "</services>";
+
+        VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create();
+        List<LogctlSpec> logctlSpecs = vespaModel.getAdmin().getLogctlSpecs();
+        assertEquals(4, logctlSpecs.size());  // Default logctl specs
+        assertEquals(1, logctlSpecs
+                .stream()
+                .filter(l -> (l.componentSpec.equals("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator")
+                        && l.levelsModSpec.equals("fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=off,spam=off"))).count());
+
+        String localhostConfigId = "hosts/myhost0";
+        SentinelConfig sentinelConfig = vespaModel.getConfig(SentinelConfig.class, localhostConfigId);
+        System.out.println(sentinelConfig);
+        assertEquals(4, getConfigForService("container", sentinelConfig).logctl().size());
+        assertEquals(4, getConfigForService("metricsproxy-container", sentinelConfig).logctl().size());
+    }
+
+    private SentinelConfig.Service getConfigForService(String serviceName, SentinelConfig config) {
+        return config.service().stream().filter(service -> service.name().equals(serviceName)).findFirst().get();
+    }
+
 }