diff options
19 files changed, 321 insertions, 10 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index 43b59abd5b5..dfb5b1d3e22 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -100,6 +100,9 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr for (var entry : s.getEnvVars().entrySet()) { serviceBuilder.environ(b -> b.varname(entry.getKey()).varvalue(entry.getValue().toString())); } + for (var entry : s.getLogctlSpecs()) { + serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec).levelsModSpec(entry.levelsModSpec)); + } setPreShutdownCommand(serviceBuilder, s); return serviceBuilder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java new file mode 100644 index 00000000000..fd959943eab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +public class LogctlSpec { + public final String componentSpec; + public final String levelsModSpec; + public LogctlSpec(String componentSpec, String levelsModSpec) { + this.componentSpec = componentSpec; + this.levelsModSpec = levelsModSpec; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java index f529b22cc45..2daa6ff66ba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.model.api.ServiceInfo; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -24,6 +25,8 @@ public interface Service extends ConfigProducer, NetworkPortRequestor { // environment variables specific for this service: Map<String, Object> getEnvVars(); + default List<LogctlSpec> getLogctlSpecs() { return List.of(); } + /** * Services that wish that a command should be run before shutdown * should return the command here. The command will be executed diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 79f65264249..e29615b7fb3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.ConfigSentinel; import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.Logd; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; @@ -68,6 +69,14 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable this.logForwarderIncludeAdmin = includeAdmin; } + private final List<LogctlSpec> logctlSpecs = new ArrayList<>(); + public List<LogctlSpec> getLogctlSpecs() { + return logctlSpecs; + } + public void addLogctlCommand(String componentSpec, String levelsModSpec) { + logctlSpecs.add(new LogctlSpec(componentSpec, levelsModSpec)); + } + /** * The single cluster controller cluster shared by all content clusters by default when not multitenant. * If multitenant, this is null. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 93b67407933..ab748dc1fd1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -223,16 +223,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC .orElse(Collections.emptyMap()); } - private Optional<Admin> getAdmin() { - if (parent != null) { - AbstractConfigProducerRoot r = parent.getRoot(); - if (r instanceof VespaModel model) { - return Optional.ofNullable(model.getAdmin()); - } - } - return Optional.empty(); - } - private Optional<String> getSystemName() { Monitoring monitoring = getMonitoringService(); return monitoring != null && ! monitoring.getClustername().equals("") ? diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 06453bffaaf..ba4a915e255 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.log.internal.LevelsModSpec; import com.yahoo.text.XML; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; @@ -21,7 +22,9 @@ import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; import org.w3c.dom.Element; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -110,4 +113,23 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu } } + private void addLoggingSpec(ModelElement loggingSpec, Admin admin) { + if (loggingSpec == null) return; + String componentSpec = loggingSpec.requiredStringAttribute("name"); + String levels = loggingSpec.requiredStringAttribute("levels"); + var levelSpec = new LevelsModSpec(); + levelSpec.setLevels(levels); + admin.addLogctlCommand(componentSpec, levelSpec.toLogctlModSpec()); + } + + void addLoggingSpecs(ModelElement loggingElement, Admin admin) { + if (loggingElement == null) return; + for (ModelElement e : loggingElement.children("class")) { + addLoggingSpec(e, admin); + } + for (ModelElement e : loggingElement.children("package")) { + addLoggingSpec(e, admin); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index ac5633a1461..10a9688d52d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -47,6 +47,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState); addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin); + addLoggingSpecs(new ModelElement(adminE).child("logging"), admin); } private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index d27f78f6a8a..567ccbfa88b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -56,6 +56,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { assignLogserver(deployState, requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin); addLogForwarders(adminElement.child("logforwarding"), admin); + addLoggingSpecs(adminElement.child("logging"), admin); } private void assignSlobroks(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 1971ccc3035..53858f8cc0e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -9,8 +9,10 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -42,6 +44,15 @@ public final class ApplicationContainer extends Container implements addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider")); } + private List<LogctlSpec> logctlSpecs = List.of(); + void setLogctlSpecs(List<LogctlSpec> logctlSpecs) { + this.logctlSpecs = logctlSpecs; + } + @Override + public List<LogctlSpec> getLogctlSpecs() { + return logctlSpecs; + } + @Override public void getConfig(QrStartConfig.Builder builder) { if (getHostResource() != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c097f856da2..ec1776730b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; @@ -125,11 +126,19 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat : defaultHeapSizePercentageOfTotalNodeMemory; } + private void wireLogctlSpecs() { + getAdmin().ifPresent(admin -> { + for (var c : getContainers()) { + c.setLogctlSpecs(admin.getLogctlSpecs()); + }}); + } + @Override protected void doPrepare(DeployState deployState) { addAndSendApplicationBundles(deployState); sendUserConfiguredFiles(deployState); createEndpointList(deployState); + wireLogctlSpecs(); } private void addAndSendApplicationBundles(DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index f1b3c74a55d..f94c3be25bb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -36,6 +36,8 @@ import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.PortsMeta; import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.component.AccessLogComponent; @@ -208,6 +210,17 @@ public abstract class ContainerCluster<CONTAINER extends Container> return zone; } + protected Optional<Admin> getAdmin() { + var parent = getParent(); + if (parent != null) { + var r = parent.getRoot(); + if (r instanceof VespaModel model) { + return Optional.ofNullable(model.getAdmin()); + } + } + return Optional.empty(); + } + public void addDefaultHandlersWithVip() { addDefaultHandlersExceptStatus(); addVipHandler(); diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 6f4c90159d8..392572e1f12 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -12,6 +12,7 @@ AdminV2 = AdminMonitoring? & Metrics? & ClusterControllers? & + LoggingSpecs? & LogForwarding? } @@ -30,6 +31,7 @@ AdminV4 = GenericConfig* & AdminMonitoring? & Metrics? & + LoggingSpecs? & LogForwarding? } @@ -113,3 +115,17 @@ LogForwarding = element logforwarding { attribute phone-home-interval { xsd:positiveInteger }? } } + +LoggingSpecs = element logging { + ( + element class { + attribute name { xsd:Name } & + attribute levels { xsd:string } + } + | + element package { + attribute name { xsd:Name } & + attribute levels { xsd:string } + } + )* +} diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index 79d4af7b3a4..fde3b6c8e67 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_executable(configd_config-sentinel_app connectivity.cpp env.cpp line-splitter.cpp + logctl.cpp manager.cpp metrics.cpp model-owner.cpp diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp new file mode 100644 index 00000000000..94759d4f102 --- /dev/null +++ b/configd/src/apps/sentinel/logctl.cpp @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "logctl.h" + +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#include <stdio.h> + +#include <vespa/log/log.h> +LOG_SETUP(".sentinel.logctl"); + +namespace config::sentinel { + +void justRunLogctl(const char *cspec, const char *lspec) +{ + const char *progName = "vespa-logctl"; + pid_t pid = fork(); + if (pid == 0) { + LOG(debug, "running '%s' '%s' '%s'", progName, cspec, lspec); + int rv = execlp(progName, progName, "-c", cspec, lspec, nullptr); + if (rv != 0) { + LOG(warning, "execlp of '%s' failed: %s", progName, strerror(errno)); + } + } else if (pid > 0) { + int wstatus = 0; + pid_t got = waitpid(pid, &wstatus, 0); + if (got == pid) { + if (WIFEXITED(wstatus)) { + int exitCode = WEXITSTATUS(wstatus); + if (exitCode != 0) { + LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode); + } + } else if (WIFSIGNALED(wstatus)) { + int termSig = WTERMSIG(wstatus); + LOG(warning, "running '%s' failed (got signal %d)", progName, termSig); + } else { + LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus); + } + } else { + LOG(error, "waitpid() failed: %s", strerror(errno)); + } + } else { + LOG(error, "fork() failed: %s", strerror(errno)); + } +} + +} diff --git a/configd/src/apps/sentinel/logctl.h b/configd/src/apps/sentinel/logctl.h new file mode 100644 index 00000000000..7eb88bcd19c --- /dev/null +++ b/configd/src/apps/sentinel/logctl.h @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace config::sentinel { + +void justRunLogctl(const char *cspec, const char *lspec); + +} diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp index e9225325e27..cb2c935956e 100644 --- a/configd/src/apps/sentinel/service.cpp +++ b/configd/src/apps/sentinel/service.cpp @@ -2,6 +2,7 @@ #include "service.h" #include "output-connection.h" +#include "logctl.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/signalhandler.h> @@ -56,6 +57,21 @@ Service::Service(const SentinelConfig::Service& service, const SentinelConfig::A start(); } +void applyLogctl(const cloud::config::SentinelConfig::Service &config) { + for (const auto &logctl : config.logctl) { + const auto cspec = config.name + ":" + logctl.componentSpec; + const auto lspec = logctl.levelsModSpec; + justRunLogctl(cspec.c_str(), lspec.c_str()); + } +} + +void unApplyLogctl(const cloud::config::SentinelConfig::Service &config) { + for (const auto &logctl : config.logctl) { + const auto cspec = config.name + ":" + logctl.componentSpec; + justRunLogctl(cspec.c_str(), "all=on,debug=off,spam=off"); + } +} + void Service::reconfigure(const SentinelConfig::Service& config) { @@ -70,8 +86,10 @@ Service::reconfigure(const SentinelConfig::Service& config) terminate(); } + unApplyLogctl(*_config); delete _config; _config = new SentinelConfig::Service(config); + applyLogctl(*_config); if ((_state == READY) || (_state == FINISHED) || (_state == RESTARTING)) { if (_isAutomatic) { @@ -329,6 +347,7 @@ Service::runChild() for (const auto &envvar : _config->environ) { setenv(envvar.varname.c_str(), envvar.varvalue.c_str(), 1); } + applyLogctl(*_config); // Set up environment setenv("VESPA_SERVICE_NAME", _config->name.c_str(), 1); diff --git a/configdefinitions/src/vespa/sentinel.def b/configdefinitions/src/vespa/sentinel.def index f28c43c77d4..1a1184707f8 100644 --- a/configdefinitions/src/vespa/sentinel.def +++ b/configdefinitions/src/vespa/sentinel.def @@ -43,6 +43,10 @@ service[].command string service[].environ[].varname string service[].environ[].varvalue string +## Tune log-level settings for specific components +service[].logctl[].componentSpec string +service[].logctl[].levelsModSpec string + ## The command to run before stopping service. The same properties as for ## startup command holds. service[].preShutdownCommand string default="" diff --git a/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java b/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java new file mode 100644 index 00000000000..4e45b2a91c5 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/internal/LevelsModSpec.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.internal; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class LevelsModSpec { + private static final String ON = "on"; + private static final String OFF = "off"; + + private static Map<String, String> defaultLogLevels() { + var m = new LinkedHashMap<String,String>(); + m.put("fatal", ON); + m.put("error", ON); + m.put("warning", ON); + m.put("info", ON); + m.put("event", ON); + m.put("config", ON); + m.put("debug", OFF); + m.put("spam", OFF); + return m; + } + private Map<String, String> levelMods = defaultLogLevels(); + + private void setAll(String value) { + for (String k : levelMods.keySet()) { + levelMods.put(k, value); + } + } + private void setAll() { + setAll(ON); + } + private void clearAll() { + setAll(OFF); + } + + public LevelsModSpec addModifications(String mods) { + for (String s : mods.split("[+ ,]")) { + String offOn = ON; + if (s.startsWith("-")) { + offOn = OFF; + s = s.substring(1); + } + if (s.isEmpty()) continue; + if (s.equals("all")) { + setAll(offOn); + } else if (levelMods.containsKey(s)) { + levelMods.put(s, offOn); + } else { + throw new IllegalArgumentException("Unknown log level: "+s); + } + } + return this; + } + + public LevelsModSpec setLevels(String levels) { + if (! (levels.startsWith("+") || levels.startsWith("-"))) { + clearAll(); + } + return addModifications(levels); + } + + public String toLogctlModSpec() { + var spec = new StringBuilder(); + boolean comma = false; + for (var entry : levelMods.entrySet()) { + if (comma) { + spec.append(","); + } + spec.append(entry.getKey()); + spec.append("="); + spec.append(entry.getValue()); + comma = true; + } + return spec.toString(); + } + +} diff --git a/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java b/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java new file mode 100644 index 00000000000..0c4f5033133 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/internal/LevelsModSpecTest.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.internal; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author arnej + */ +public class LevelsModSpecTest { + + @Test + public void hasCorrectDefault() { + String wanted = "fatal=on,error=on,warning=on,info=on,event=on,config=on,debug=off,spam=off"; + var l = new LevelsModSpec(); + assertEquals(wanted, l.toLogctlModSpec()); + } + + @Test + public void canTurnInfoOff() { + String wanted = "fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=off,spam=off"; + var l = new LevelsModSpec(); + l.setLevels("-info"); + assertEquals(wanted, l.toLogctlModSpec()); + } + + @Test + public void canTurnDebugOn() { + String wanted = "fatal=on,error=on,warning=on,info=on,event=on,config=on,debug=on,spam=off"; + var l = new LevelsModSpec(); + l.setLevels("+debug"); + assertEquals(wanted, l.toLogctlModSpec()); + } + + @Test + public void canSpeficyOnlySome() { + String wanted = "fatal=off,error=off,warning=on,info=off,event=off,config=off,debug=on,spam=off"; + var l = new LevelsModSpec(); + l.setLevels("warning debug"); + assertEquals(wanted, l.toLogctlModSpec()); + l = new LevelsModSpec(); + l.setLevels("warning,debug"); + assertEquals(wanted, l.toLogctlModSpec()); + l = new LevelsModSpec(); + l.setLevels("warning, debug"); + assertEquals(wanted, l.toLogctlModSpec()); + } + + @Test + public void canSpeficyAllMinusSome() { + String wanted ="fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=on,spam=off"; + var l = new LevelsModSpec(); + l.setLevels("all -info -spam"); + assertEquals(wanted, l.toLogctlModSpec()); + l = new LevelsModSpec(); + l.setLevels("all,-info,-spam"); + assertEquals(wanted, l.toLogctlModSpec()); + l = new LevelsModSpec(); + l.setLevels("all, -info, -spam"); + assertEquals(wanted, l.toLogctlModSpec()); + } + +} |