Merge pull request #25150 from vespa-engine/arnej/add-tuning-of-logging

Arnej/add tuning of logging

19 files changed, 321 insertions, 10 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
index 43b59abd5b5..dfb5b1d3e22 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
@@ -100,6 +100,9 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr
         for (var entry : s.getEnvVars().entrySet()) {
             serviceBuilder.environ(b -> b.varname(entry.getKey()).varvalue(entry.getValue().toString()));
         }
+        for (var entry : s.getLogctlSpecs()) {
+            serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec).levelsModSpec(entry.levelsModSpec));
+        }
         setPreShutdownCommand(serviceBuilder, s);
         return serviceBuilder;
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java
new file mode 100644
index 00000000000..fd959943eab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java
@@ -0,0 +1,11 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+public class LogctlSpec {
+    public final String componentSpec;
+    public final String levelsModSpec;
+    public LogctlSpec(String componentSpec, String levelsModSpec) {
+        this.componentSpec = componentSpec;
+        this.levelsModSpec = levelsModSpec;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
index f529b22cc45..2daa6ff66ba 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model;
 import com.yahoo.config.model.api.ServiceInfo;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -24,6 +25,8 @@ public interface Service extends ConfigProducer, NetworkPortRequestor {
     // environment variables specific for this service:
     Map<String, Object> getEnvVars();
 
+    default List<LogctlSpec> getLogctlSpecs() { return List.of(); }
+
     /**
      * Services that wish that a command should be run before shutdown
      * should return the command here. The command will be executed
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 79f65264249..e29615b7fb3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.ConfigSentinel;
 import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.Logd;
 import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer;
 import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster;
@@ -68,6 +69,14 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
         this.logForwarderIncludeAdmin = includeAdmin;
     }
 
+    private final List<LogctlSpec> logctlSpecs = new ArrayList<>();
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+    public void addLogctlCommand(String componentSpec, String levelsModSpec) {
+        logctlSpecs.add(new LogctlSpec(componentSpec,  levelsModSpec));
+    }
+
     /**
      * The single cluster controller cluster shared by all content clusters by default when not multitenant.
      * If multitenant, this is null.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index 93b67407933..ab748dc1fd1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -223,16 +223,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
                 .orElse(Collections.emptyMap());
     }
 
-    private Optional<Admin> getAdmin() {
-        if (parent != null) {
-            AbstractConfigProducerRoot r = parent.getRoot();
-            if (r instanceof VespaModel model) {
-                return Optional.ofNullable(model.getAdmin());
-            }
-        }
-        return Optional.empty();
-    }
-
     private Optional<String> getSystemName() {
         Monitoring monitoring = getMonitoringService();
         return monitoring != null && ! monitoring.getClustername().equals("") ?
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index 06453bffaaf..ba4a915e255 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -5,6 +5,7 @@ import com.yahoo.config.model.ConfigModelContext.ApplicationType;
 import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.log.internal.LevelsModSpec;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.model.Host;
 import com.yahoo.vespa.model.HostResource;
@@ -21,7 +22,9 @@ import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder;
 import org.w3c.dom.Element;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -110,4 +113,23 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
         }
     }
 
+    private void addLoggingSpec(ModelElement loggingSpec, Admin admin) {
+        if (loggingSpec == null) return;
+        String componentSpec = loggingSpec.requiredStringAttribute("name");
+        String levels = loggingSpec.requiredStringAttribute("levels");
+        var levelSpec = new LevelsModSpec();
+        levelSpec.setLevels(levels);
+        admin.addLogctlCommand(componentSpec, levelSpec.toLogctlModSpec());
+    }
+
+    void addLoggingSpecs(ModelElement loggingElement, Admin admin) {
+        if (loggingElement == null) return;
+        for (ModelElement e : loggingElement.children("class")) {
+            addLoggingSpec(e, admin);
+        }
+        for (ModelElement e : loggingElement.children("package")) {
+            addLoggingSpec(e, admin);
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
index ac5633a1461..10a9688d52d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
@@ -47,6 +47,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
             admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState);
 
         addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin);
+        addLoggingSpecs(new ModelElement(adminE).child("logging"), admin);
     }
 
     private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index d27f78f6a8a..567ccbfa88b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -56,6 +56,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
         assignLogserver(deployState, requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin);
 
         addLogForwarders(adminElement.child("logforwarding"), admin);
+        addLoggingSpecs(adminElement.child("logging"), admin);
     }
 
     private void assignSlobroks(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index 1971ccc3035..53858f8cc0e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -9,8 +9,10 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
@@ -42,6 +44,15 @@ public final class ApplicationContainer extends Container implements
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider"));
     }
 
+    private List<LogctlSpec> logctlSpecs = List.of();
+    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
+        this.logctlSpecs = logctlSpecs;
+    }
+    @Override
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+
     @Override
     public void getConfig(QrStartConfig.Builder builder) {
         if (getHostResource() != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index c097f856da2..ec1776730b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -31,6 +31,7 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
 import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
 import com.yahoo.vespa.model.container.component.BindingPattern;
 import com.yahoo.vespa.model.container.component.Component;
@@ -125,11 +126,19 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                 : defaultHeapSizePercentageOfTotalNodeMemory;
     }
 
+    private void wireLogctlSpecs() {
+        getAdmin().ifPresent(admin -> {
+            for (var c : getContainers()) {
+                c.setLogctlSpecs(admin.getLogctlSpecs());
+            }});
+    }
+
     @Override
     protected void doPrepare(DeployState deployState) {
         addAndSendApplicationBundles(deployState);
         sendUserConfiguredFiles(deployState);
         createEndpointList(deployState);
+        wireLogctlSpecs();
     }
 
     private void addAndSendApplicationBundles(DeployState deployState) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index f1b3c74a55d..f94c3be25bb 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -36,6 +36,8 @@ import com.yahoo.search.query.profile.config.QueryProfilesConfig;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.model.PortsMeta;
 import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.Admin;
 import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.clients.ContainerDocumentApi;
 import com.yahoo.vespa.model.container.component.AccessLogComponent;
@@ -208,6 +210,17 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         return zone;
     }
 
+    protected Optional<Admin> getAdmin() {
+        var parent = getParent();
+        if (parent != null) {
+            var r = parent.getRoot();
+            if (r instanceof VespaModel model) {
+                return Optional.ofNullable(model.getAdmin());
+            }
+        }
+        return Optional.empty();
+    }
+
     public void addDefaultHandlersWithVip() {
         addDefaultHandlersExceptStatus();
         addVipHandler();
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index 6f4c90159d8..392572e1f12 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -12,6 +12,7 @@ AdminV2 =
    AdminMonitoring? &
    Metrics? &
    ClusterControllers? &
+   LoggingSpecs? &
    LogForwarding?
  }
 
@@ -30,6 +31,7 @@ AdminV4 =
     GenericConfig* &
     AdminMonitoring? &
     Metrics? &
+    LoggingSpecs? &
     LogForwarding?
   }
 
@@ -113,3 +115,17 @@ LogForwarding = element logforwarding {
         attribute phone-home-interval { xsd:positiveInteger }?
     }
 }
+
+LoggingSpecs = element logging {
+    (
+    element class {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    |
+    element package {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    )*
+}
diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt
index 79d4af7b3a4..fde3b6c8e67 100644
--- a/configd/src/apps/sentinel/CMakeLists.txt
+++ b/configd/src/apps/sentinel/CMakeLists.txt
@@ -7,6 +7,7 @@ vespa_add_executable(configd_config-sentinel_app
     connectivity.cpp
     env.cpp
     line-splitter.cpp
+    logctl.cpp
     manager.cpp
     metrics.cpp
     model-owner.cpp
diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp
new file mode 100644
index 00000000000..94759d4f102
--- /dev/null
+++ b/configd/src/apps/sentinel/logctl.cpp
@@ -0,0 +1,48 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "logctl.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".sentinel.logctl");
+
+namespace config::sentinel {
+
+void justRunLogctl(const char *cspec, const char *lspec)
+{
+    const char *progName = "vespa-logctl";
+    pid_t pid = fork();
+    if (pid == 0) {
+        LOG(debug, "running '%s' '%s' '%s'", progName, cspec, lspec);
+        int rv = execlp(progName, progName, "-c", cspec, lspec, nullptr);
+        if (rv != 0) {
+            LOG(warning, "execlp of '%s' failed: %s", progName, strerror(errno));
+        }
+    } else if (pid > 0) {
+        int wstatus = 0;
+        pid_t got = waitpid(pid, &wstatus, 0);
+        if (got == pid) {
+            if (WIFEXITED(wstatus)) {
+                int exitCode = WEXITSTATUS(wstatus);
+                if (exitCode != 0) {
+                    LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode);
+                }
+            } else if (WIFSIGNALED(wstatus)) {
+                int termSig = WTERMSIG(wstatus);
+                LOG(warning, "running '%s' failed (got signal %d)", progName, termSig);
+            } else {
+                LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus);
+            }
+        } else {
+            LOG(error, "waitpid() failed: %s", strerror(errno));
+        }
+    } else {
+        LOG(error, "fork() failed: %s", strerror(errno));
+    }
+}
+
+}
diff --git a/configd/src/apps/sentinel/logctl.h b/configd/src/apps/sentinel/logctl.h
new file mode 100644
index 00000000000..7eb88bcd19c
--- /dev/null
+++ b/configd/src/apps/sentinel/logctl.h
@@ -0,0 +1,8 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+namespace config::sentinel {
+
+void justRunLogctl(const char *cspec, const char *lspec);
+
+}
diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
index e9225325e27..cb2c935956e 100644
--- a/configd/src/apps/sentinel/service.cpp
+++ b/configd/src/apps/sentinel/service.cpp
@@ -2,6 +2,7 @@
 
 #include "service.h"
 #include "output-connection.h"
+#include "logctl.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/signalhandler.h>
 
@@ -56,6 +57,21 @@ Service::Service(const SentinelConfig::Service& service, const SentinelConfig::A
     start();
 }
 
+void applyLogctl(const cloud::config::SentinelConfig::Service &config) {
+    for (const auto &logctl : config.logctl) {
+        const auto cspec = config.name + ":" + logctl.componentSpec;
+        const auto lspec = logctl.levelsModSpec;
+        justRunLogctl(cspec.c_str(), lspec.c_str());
+    }
+}
+
+void unApplyLogctl(const cloud::config::SentinelConfig::Service &config) {
+    for (const auto &logctl : config.logctl) {
+        const auto cspec = config.name + ":" + logctl.componentSpec;
+        justRunLogctl(cspec.c_str(), "all=on,debug=off,spam=off");
+    }
+}
+
 void
 Service::reconfigure(const SentinelConfig::Service& config)
 {
@@ -70,8 +86,10 @@ Service::reconfigure(const SentinelConfig::Service& config)
         terminate();
     }
 
+    unApplyLogctl(*_config);
     delete _config;
     _config = new SentinelConfig::Service(config);
+    applyLogctl(*_config);
 
     if ((_state == READY) || (_state == FINISHED) || (_state == RESTARTING)) {
         if (_isAutomatic) {
@@ -329,6 +347,7 @@ Service::runChild()
     for (const auto &envvar : _config->environ) {
         setenv(envvar.varname.c_str(), envvar.varvalue.c_str(), 1);
     }
+    applyLogctl(*_config);
 
     // Set up environment
     setenv("VESPA_SERVICE_NAME", _config->name.c_str(), 1);
diff --git a/configdefinitions/src/vespa/sentinel.def b/configdefinitions/src/vespa/sentinel.def
index f28c43c77d4..1a1184707f8 100644
--- a/configdefinitions/src/vespa/sentinel.def
+++ b/configdefinitions/src/vespa/sentinel.def
@@ -43,6 +43,10 @@ service[].command string
 service[].environ[].varname string
 service[].environ[].varvalue string
 
+## Tune log-level settings for specific components
+service[].logctl[].componentSpec string
+service[].logctl[].levelsModSpec string
+
 ## The command to run before stopping service. The same properties as for
 ## startup command holds.
 service[].preShutdownCommand string default=""
diff --git a/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java b/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java
new file mode 100644
index 00000000000..4e45b2a91c5
--- /dev/null
+++ b/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java
@@ -0,0 +1,78 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log.internal;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class LevelsModSpec {
+    private static final String ON = "on";
+    private static final String OFF = "off";
+
+    private static Map<String, String> defaultLogLevels() {
+        var m = new LinkedHashMap<String,String>();
+        m.put("fatal", ON);
+        m.put("error", ON);
+        m.put("warning", ON);
+        m.put("info", ON);
+        m.put("event", ON);
+        m.put("config", ON);
+        m.put("debug", OFF);
+        m.put("spam", OFF);
+        return m;
+    }
+    private Map<String, String> levelMods = defaultLogLevels();
+
+    private void setAll(String value) {
+        for (String k : levelMods.keySet()) {
+            levelMods.put(k, value);
+        }
+    }
+    private void setAll() {
+        setAll(ON);
+    }
+    private void clearAll() {
+        setAll(OFF);
+    }
+
+    public LevelsModSpec addModifications(String mods) {
+        for (String s : mods.split("[+ ,]")) {
+            String offOn = ON;
+            if (s.startsWith("-")) {
+                offOn = OFF;
+                s = s.substring(1);
+            }
+            if (s.isEmpty()) continue;
+            if (s.equals("all")) {
+                setAll(offOn);
+            } else if (levelMods.containsKey(s)) {
+                levelMods.put(s, offOn);
+            } else {
+                throw new IllegalArgumentException("Unknown log level: "+s);
+            }
+        }
+        return this;
+    }
+
+    public LevelsModSpec setLevels(String levels) {
+        if (! (levels.startsWith("+") || levels.startsWith("-"))) {
+            clearAll();
+        }
+        return addModifications(levels);
+    }
+
+    public String toLogctlModSpec() {
+        var spec = new StringBuilder();
+        boolean comma = false;
+        for (var entry : levelMods.entrySet()) {
+            if (comma) {
+                spec.append(",");
+            }
+            spec.append(entry.getKey());
+            spec.append("=");
+            spec.append(entry.getValue());
+            comma = true;
+        }
+        return spec.toString();
+    }
+
+}
diff --git a/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java b/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java
new file mode 100644
index 00000000000..0c4f5033133
--- /dev/null
+++ b/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java
@@ -0,0 +1,64 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log.internal;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author arnej
+ */
+public class LevelsModSpecTest {
+
+    @Test
+    public void hasCorrectDefault() {
+        String wanted = "fatal=on,error=on,warning=on,info=on,event=on,config=on,debug=off,spam=off";
+        var l = new LevelsModSpec();
+        assertEquals(wanted, l.toLogctlModSpec());
+    }
+
+    @Test
+    public void canTurnInfoOff() {
+        String wanted = "fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=off,spam=off";
+        var l = new LevelsModSpec();
+        l.setLevels("-info");
+        assertEquals(wanted, l.toLogctlModSpec());
+    }
+
+    @Test
+    public void canTurnDebugOn() {
+        String wanted = "fatal=on,error=on,warning=on,info=on,event=on,config=on,debug=on,spam=off";
+        var l = new LevelsModSpec();
+        l.setLevels("+debug");
+        assertEquals(wanted, l.toLogctlModSpec());
+    }
+
+    @Test
+    public void canSpeficyOnlySome() {
+        String wanted = "fatal=off,error=off,warning=on,info=off,event=off,config=off,debug=on,spam=off";
+        var l = new LevelsModSpec();
+        l.setLevels("warning debug");
+        assertEquals(wanted, l.toLogctlModSpec());
+        l = new LevelsModSpec();
+        l.setLevels("warning,debug");
+        assertEquals(wanted, l.toLogctlModSpec());
+        l = new LevelsModSpec();
+        l.setLevels("warning, debug");
+        assertEquals(wanted, l.toLogctlModSpec());
+    }
+
+    @Test
+    public void canSpeficyAllMinusSome() {
+        String wanted ="fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=on,spam=off";
+        var l = new LevelsModSpec();
+        l.setLevels("all -info -spam");
+        assertEquals(wanted, l.toLogctlModSpec());
+        l = new LevelsModSpec();
+        l.setLevels("all,-info,-spam");
+        assertEquals(wanted, l.toLogctlModSpec());
+        l = new LevelsModSpec();
+        l.setLevels("all, -info, -spam");
+        assertEquals(wanted, l.toLogctlModSpec());
+    }
+
+}