Merge pull request #26464 from vespa-engine/arnej/turn-off-some-logging-take-3

Arnej/turn off some logging take 3

13 files changed, 130 insertions, 68 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
index da4cabcf9f5..35d7b2d225e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
@@ -95,7 +95,8 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr
             serviceBuilder.environ(b -> b.varname(entry.getKey()).varvalue(entry.getValue().toString()));
         }
         for (var entry : s.getLogctlSpecs()) {
-            serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec).levelsModSpec(entry.levelsModSpec));
+            serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec())
+                                        .levelsModSpec(entry.levelsModSpec().toLogctlModSpec()));
         }
         setPreShutdownCommand(serviceBuilder, s);
         return serviceBuilder;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java
index fd959943eab..d927c38082e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java
@@ -1,11 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model;
 
-public class LogctlSpec {
-    public final String componentSpec;
-    public final String levelsModSpec;
-    public LogctlSpec(String componentSpec, String levelsModSpec) {
-        this.componentSpec = componentSpec;
-        this.levelsModSpec = levelsModSpec;
-    }
+import com.yahoo.container.logging.LevelsModSpec;
+
+public record LogctlSpec(String componentSpec, LevelsModSpec levelsModSpec) {
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 317c3646dde..62cbf61b336 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AnyConfigProducer;
 import com.yahoo.config.model.producer.TreeConfigProducer;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.container.logging.LevelsModSpec;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.ConfigSentinel;
@@ -24,7 +25,6 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -41,8 +41,6 @@ import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty;
  */
 public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Serializable {
 
-    private static final long serialVersionUID = 1L;
-
     private final boolean isHostedVespa;
     private final Monitoring monitoring;
     private final List<Configserver> configservers = new ArrayList<>();
@@ -68,12 +66,6 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
     }
 
     private final List<LogctlSpec> logctlSpecs = new ArrayList<>();
-    public List<LogctlSpec> getLogctlSpecs() {
-        return logctlSpecs;
-    }
-    public void addLogctlCommand(String componentSpec, String levelsModSpec) {
-        logctlSpecs.add(new LogctlSpec(componentSpec,  levelsModSpec));
-    }
 
     /**
      * The single cluster controller cluster shared by all content clusters by default when not multitenant.
@@ -101,6 +93,7 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
         this.multitenant = multitenant;
         this.fileDistribution = new FileDistributionConfigProducer(parent);
         this.applicationType = applicationType;
+        this.logctlSpecs.addAll(defaultLogctlSpecs());
     }
 
     public Configserver getConfigserver() { return defaultConfigserver; }
@@ -331,4 +324,25 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri
 
     public ApplicationType getApplicationType() { return applicationType; }
 
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+    public void addLogctlCommand(String componentSpec, LevelsModSpec levelsModSpec) {
+        logctlSpecs.add(new LogctlSpec(componentSpec, levelsModSpec));
+    }
+
+    private static List<LogctlSpec> defaultLogctlSpecs() {
+        // Turn off info logging for all containers for some classes (unimportant log messages that create noise in vespa log)
+        return List.of(new LogctlSpec("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator", getLevelModSpec("-info")),
+                       new LogctlSpec("org.eclipse.jetty.server.Server", getLevelModSpec("-info")),
+                       new LogctlSpec("org.eclipse.jetty.server.handler.ContextHandler", getLevelModSpec("-info")),
+                       new LogctlSpec("org.eclipse.jetty.server.AbstractConnector", getLevelModSpec("-info")));
+    }
+
+    static LevelsModSpec getLevelModSpec(String levels) {
+        var levelSpec = new LevelsModSpec();
+        levelSpec.setLevels(levels);
+        return levelSpec;
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
index e50c010c476..fef7d534c30 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
@@ -25,9 +25,6 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain
     }
 
     @Override
-    protected void doPrepare(DeployState deployState) { }
-
-    @Override
     public void getConfig(QrStartConfig.Builder builder) {
         super.getConfig(builder);
         builder.jvm.heapsize(128);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
index 25d5620eff9..c65abfb0189 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
@@ -37,9 +37,6 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC
     @Override
     protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; }
 
-    @Override
-    protected void doPrepare(DeployState deployState) { }
-
     @Override protected boolean messageBusEnabled() { return false; }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index ec543dba6fb..ada647b535d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -137,9 +137,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
     }
 
     @Override
-    protected void doPrepare(DeployState deployState) { }
-
-    @Override
     public void getConfig(MetricsNodesConfig.Builder builder) {
         builder.node.addAll(MetricsNodesConfigGenerator.generate(getContainers()));
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index 57113ff794f..9280f0ceb9a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -21,11 +21,8 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
 import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets;
 import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder;
 import org.w3c.dom.Element;
-
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -120,7 +117,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
         String levels = loggingSpec.requiredStringAttribute("levels");
         var levelSpec = new LevelsModSpec();
         levelSpec.setLevels(levels);
-        admin.addLogctlCommand(componentSpec, levelSpec.toLogctlModSpec());
+        admin.addLogctlCommand(componentSpec, levelSpec);
     }
 
     void addLoggingSpecs(ModelElement loggingElement, Admin admin) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index bc410670d5e..f901bf3c826 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -9,11 +9,8 @@ import com.yahoo.config.model.producer.TreeConfigProducer;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.search.config.QrStartConfig;
-import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
-
 import java.time.Duration;
-import java.util.List;
 import java.util.Optional;
 
 /**
@@ -43,15 +40,6 @@ public final class ApplicationContainer extends Container implements
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider"));
     }
 
-    private List<LogctlSpec> logctlSpecs = List.of();
-    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
-        this.logctlSpecs = logctlSpecs;
-    }
-    @Override
-    public List<LogctlSpec> getLogctlSpecs() {
-        return logctlSpecs;
-    }
-
     @Override
     public void getConfig(QrStartConfig.Builder builder) {
         if (getHostResource() != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 004ff5516ba..d101cea428e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -124,19 +124,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                 : defaultHeapSizePercentageOfTotalNodeMemory;
     }
 
-    private void wireLogctlSpecs() {
-        getAdmin().ifPresent(admin -> {
-            for (var c : getContainers()) {
-                c.setLogctlSpecs(admin.getLogctlSpecs());
-            }});
-    }
-
     @Override
     protected void doPrepare(DeployState deployState) {
+        super.doPrepare(deployState);
         addAndSendApplicationBundles(deployState);
         sendUserConfiguredFiles(deployState);
         createEndpointList(deployState);
-        wireLogctlSpecs();
     }
 
     private void addAndSendApplicationBundles(DeployState deployState) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index be9f3fa894f..2ca6d5d7155 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -16,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.PortAllocBridge;
 import com.yahoo.vespa.model.application.validation.RestartConfigs;
 import com.yahoo.vespa.model.container.component.Component;
@@ -34,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -65,6 +67,8 @@ public abstract class Container extends AbstractService implements
     /** The cluster this container belongs to, or null if it is not added to any cluster */
     private ContainerCluster<?> owner = null;
 
+    private List<LogctlSpec> logctlSpecs = List.of();
+
     protected final TreeConfigProducer<?> parent;
     private final String name;
     private boolean requireSpecificPorts = true;
@@ -415,4 +419,11 @@ public abstract class Container extends AbstractService implements
         return producer instanceof ContainerCluster<?> ? (ContainerCluster<?>) producer : null;
     }
 
+    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
+        this.logctlSpecs = logctlSpecs;
+    }
+
+    @Override
+    public List<LogctlSpec> getLogctlSpecs() { return logctlSpecs; }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index d786135d3ee..45060f2f27d 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -61,7 +61,6 @@ import com.yahoo.vespa.model.container.search.ContainerSearch;
 import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
 import com.yahoo.vespa.model.content.Content;
 import com.yahoo.vespa.model.search.SearchCluster;
-
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -301,7 +300,16 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         doPrepare(deployState);
     }
 
-    protected abstract void doPrepare(DeployState deployState);
+    protected void doPrepare(DeployState deployState) {
+        wireLogctlSpecs();
+    }
+
+    private void wireLogctlSpecs() {
+        getAdmin().ifPresent(admin -> {
+            for (var c : getContainers()) {
+                c.setLogctlSpecs(admin.getLogctlSpecs());
+            }});
+    }
 
     public String getName() {
         return name;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
index d060fdfd3d2..5559eb633ef 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
@@ -17,17 +17,22 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.config.core.StateserverConfig;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.Service;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
 import org.junit.jupiter.api.Test;
-
+import java.util.List;
 import java.util.Set;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 public class AdminTestCase {
@@ -243,4 +248,49 @@ public class AdminTestCase {
         assertTrue(configIds.contains("hosts/myhost0/logforwarder"), configIds.toString());
     }
 
+    @Test
+    void testDefaultLogCtlSpecs() {
+        String hosts = "<hosts>"
+                + "  <host name=\"myhost0\">"
+                + "    <alias>node0</alias>"
+                + "  </host>"
+                + "</hosts>";
+
+        String services = "<services>" +
+                "  <admin version='2.0'>" +
+                "    <adminserver hostalias='node0' />" +
+                "  </admin>" +
+                "  <container version=\"1.0\">" +
+                "    <nodes>" +
+                "      <node hostalias=\"node0\" />" +
+                "    </nodes>" +
+                "    <search/>" +
+                "    <document-api/>" +
+                "  </container>" +
+                "</services>";
+
+        VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create();
+        List<LogctlSpec> logctlSpecs = vespaModel.getAdmin().getLogctlSpecs();
+        assertEquals(4, logctlSpecs.size());  // Default logctl specs
+        assertEquals(1, logctlSpecs
+                .stream()
+                     .filter(l -> (l.componentSpec()
+                                   .equals("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator")
+                                   &&
+                                   l.levelsModSpec()
+                                   .toLogctlModSpec()
+                                   .equals("fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=off,spam=off")))
+                     .count());
+
+        String localhostConfigId = "hosts/myhost0";
+        SentinelConfig sentinelConfig = vespaModel.getConfig(SentinelConfig.class, localhostConfigId);
+        System.out.println(sentinelConfig);
+        assertEquals(4, getConfigForService("container", sentinelConfig).logctl().size());
+        assertEquals(4, getConfigForService("metricsproxy-container", sentinelConfig).logctl().size());
+    }
+
+    private SentinelConfig.Service getConfigForService(String serviceName, SentinelConfig config) {
+        return config.service().stream().filter(service -> service.name().equals(serviceName)).findFirst().get();
+    }
+
 }
diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp
index 057178cec2e..59c02b57344 100644
--- a/configd/src/apps/sentinel/logctl.cpp
+++ b/configd/src/apps/sentinel/logctl.cpp
@@ -5,6 +5,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
+#include <fcntl.h>
 #include <cstring>
 
 #include <vespa/log/log.h>
@@ -18,28 +19,40 @@ void justRunLogctl(const char *cspec, const char *lspec)
     pid_t pid = fork();
     if (pid == 0) {
         LOG(debug, "running '%s' '%s' '%s'", progName, cspec, lspec);
+        int devnull = open("/dev/null", O_WRONLY);
+        if (devnull > 1) {
+            dup2(devnull, 1);
+            close(devnull);
+        }
         int rv = execlp(progName, progName, "-c", cspec, lspec, nullptr);
         if (rv != 0) {
             LOG(warning, "execlp of '%s' failed: %s", progName, strerror(errno));
         }
     } else if (pid > 0) {
-        int wstatus = 0;
-        pid_t got = waitpid(pid, &wstatus, 0);
-        if (got == pid) {
-            if (WIFEXITED(wstatus)) {
-                int exitCode = WEXITSTATUS(wstatus);
-                if (exitCode != 0) {
-                    LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode);
+        bool again;
+        do {
+            again = false;
+            int wstatus = 0;
+            pid_t got = waitpid(pid, &wstatus, 0);
+            if (got == pid) {
+                again = false;
+                if (WIFEXITED(wstatus)) {
+                    int exitCode = WEXITSTATUS(wstatus);
+                    if (exitCode != 0) {
+                        LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode);
+                    }
+                } else if (WIFSIGNALED(wstatus)) {
+                    int termSig = WTERMSIG(wstatus);
+                    LOG(warning, "running '%s' failed (got signal %d)", progName, termSig);
+                } else {
+                    LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus);
                 }
-            } else if (WIFSIGNALED(wstatus)) {
-                int termSig = WTERMSIG(wstatus);
-                LOG(warning, "running '%s' failed (got signal %d)", progName, termSig);
+            } else if (errno == EINTR) {
+                again = true;
             } else {
-                LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus);
+                LOG(error, "waitpid() failed: %s", strerror(errno));
             }
-        } else {
-            LOG(error, "waitpid() failed: %s", strerror(errno));
-        }
+        } while (again);
     } else {
         LOG(error, "fork() failed: %s", strerror(errno));
     }