diff options
4 files changed, 72 insertions, 19 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index 3da6b34542c..38123b88a53 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -3,12 +3,24 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.log.LogLevel; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.util.List; import java.util.Objects; import java.util.logging.Level; +import java.util.stream.Collectors; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; -/** Immutable, simple log entries. */ +/** + * Immutable, simple log entries. + * + * @author jonmv + */ public class LogEntry { private final long id; @@ -42,6 +54,25 @@ public class LogEntry { return message; } + public static List<LogEntry> parseVespaLog(InputStream log) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(log, UTF_8))) { + return reader.lines() + .map(line -> line.split("\t")) + .filter(parts -> parts.length == 7) + .map(parts -> new LogEntry(0, + (long) (Double.parseDouble(parts[0]) * 1000), + typeOf(LogLevel.parse(parts[5])), + parts[1] + '\t' + parts[3] + '\t' + parts[4] + '\n' + + parts[6].replaceAll("\\\\n", "\n") + .replaceAll("\\\\t", "\t"))) + .collect(Collectors.toUnmodifiableList()); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override public String toString() { return "LogEntry{" + @@ -75,6 +106,7 @@ public class LogEntry { : Type.error; } + /** The type of entry, used for rendering. */ public enum Type { debug, info, warning, error, html; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 88ce8c41561..bc202368aaf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -42,6 +42,13 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); + /** + * Gets the Vespa logs of the given deployment. + * + * If the "from" and/or "to" query parameters are present, they are read as millis since EPOCH, and used + * to limit the time window for which log entries are gathered. + * If the "hostname" query parameter is present, it limits the entries to be from that host. + */ InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters); List<ClusterMetrics> getMetrics(DeploymentId deployment); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 03a1ee6e7b3..6a1eed0132b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; @@ -503,25 +502,10 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { - ZoneId zone = id.type().zone(controller.system()); if (deployment(id.application(), id.type()).isPresent()) try { - logger.log("Copying Vespa log from nodes of " + id.application() + " in " + zone + " ..."); - List<LogEntry> entries = new ArrayList<>(); - String logs = IOUtils.readAll(controller.serviceRegistry().configServer().getLogs(new DeploymentId(id.application(), zone), - Collections.emptyMap()), // Get all logs. - UTF_8); - for (String line : logs.split("\n")) { - String[] parts = line.split("\t"); - if (parts.length != 7) continue; - entries.add(new LogEntry(0, - (long) (Double.parseDouble(parts[0]) * 1000), - LogEntry.typeOf(LogLevel.parse(parts[5])), - parts[1] + '\t' + parts[3] + '\t' + parts[4] + '\n' + - parts[6].replaceAll("\\\\n", "\n") - .replaceAll("\\\\t", "\t"))); - } - controller.jobController().log(id, Step.copyVespaLogs, entries); + logger.log("Copying Vespa log from nodes of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().updateVespaLog(id); } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 2f8c08c15be..5b83e0bd2c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.io.IOUtils; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -25,9 +26,15 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.net.URI; import java.security.cert.X509Certificate; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -46,8 +53,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.logging.Level.INFO; import static java.util.stream.Collectors.toList; /** @@ -136,6 +147,25 @@ public class JobController { log(id, step, level, Collections.singletonList(message)); } + /** Fetches any new Vespa log entries, and records the timestamp of the last of these, for continuation. */ + public void updateVespaLog(RunId id) { + locked(id, run -> { + ZoneId zone = id.type().zone(controller.system()); + if ( ! controller.applications().require(id.application()) + .deployments().containsKey(zone)) + return run; + + List<LogEntry> log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.application(), zone), + Map.of("from", Long.toString(run.lastVespaLogTimestamp().toEpochMilli())))); + if (log.isEmpty()) + return run; + + logs.append(id.application(), id.type(), Step.copyVespaLogs, log); + return run.with(Instant.ofEpochMilli(log.get(log.size() - 1).at())); + }); + } + /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ public void updateTestLog(RunId id) { locked(id, run -> { |