diff options
Diffstat (limited to 'container-core/src')
10 files changed, 181 insertions, 251 deletions
diff --git a/container-core/src/main/java/ai/vespa/cloud/Environment.java b/container-core/src/main/java/ai/vespa/cloud/Environment.java deleted file mode 100644 index 8f1d9fc962a..00000000000 --- a/container-core/src/main/java/ai/vespa/cloud/Environment.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.cloud; - -/** - * The environments of a Vespa cloud instance - * - * @author bratseth - */ -public enum Environment { - - dev, perf, test, staging, prod - -} diff --git a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java b/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java deleted file mode 100644 index 0524ae072cd..00000000000 --- a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.cloud; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; - -/** - * Provides information about the system in which this container is running. - * This is available and can be injected when running in a cloud environment. - * - * @author bratseth - */ -public class SystemInfo { - - private final Zone zone; - - /** Do not use */ - @Inject - public SystemInfo(ConfigserverConfig config) { - this.zone = new Zone(Environment.valueOf(config.environment()), config.region()); - } - - /** Create an instance for testing */ - public SystemInfo(Zone zone) { - this.zone = zone; - } - - /** Returns the zone this is running in */ - public Zone zone() { return zone; } - -} diff --git a/container-core/src/main/java/ai/vespa/cloud/Zone.java b/container-core/src/main/java/ai/vespa/cloud/Zone.java deleted file mode 100644 index 48293aa7908..00000000000 --- a/container-core/src/main/java/ai/vespa/cloud/Zone.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.cloud; - -import java.util.Objects; - -/** - * The zone in which a cloud deployment may be running. - * A zone is a combination of an environment and a region. - * - * @author bratseth - */ -public class Zone { - - private final Environment environment; - - private final String region; - - public Zone(Environment environment, String region) { - this.environment = environment; - this.region = region; - } - - public Environment environment() { return environment; } - public String region() { return region; } - - /** Returns the string environment.region */ - @Override - public String toString() { return environment + "." + region; } - - @Override - public int hashCode() { return Objects.hash(environment, region); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof Zone)) return false; - Zone other = (Zone)o; - return this.environment.equals(other.environment) && this.region.equals(other.region); - } - - /** - * Creates a zone from a string on the form environment.region - * - * @throws IllegalArgumentException if the given string is not a valid zone - */ - public static Zone from(String zoneString) { - String[] parts = zoneString.split("\\."); - if (parts.length != 2) - throw new IllegalArgumentException("A zone string must be on the form [environment].[region], but was '" + zoneString + "'"); - - Environment environment; - try { - environment = Environment.valueOf(parts[0]); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid zone '" + zoneString + "': No environment named '" + parts[0] + "'"); - } - return new Zone(environment, parts[1]); - } - -} diff --git a/container-core/src/main/java/ai/vespa/cloud/package-info.java b/container-core/src/main/java/ai/vespa/cloud/package-info.java deleted file mode 100644 index 259a2bda258..00000000000 --- a/container-core/src/main/java/ai/vespa/cloud/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Public API to the Vespa cloud, available when this container runs in a cloud. - */ -@ExportPackage -@PublicApi -package ai.vespa.cloud; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 0b42b3a481b..1d6e1a0893d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -35,9 +35,6 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } @Override public void render(OutputStream outputStream) { logReader.writeLogs(outputStream, from, to); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index e3fef4e0e44..3cf849a6835 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.google.common.collect.Iterators; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -19,18 +19,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Comparator.comparing; /** * @author olaaun @@ -39,6 +42,9 @@ import static java.util.Comparator.comparing; */ class LogReader { + static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(.gz)?"); + static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:.gz)?)?"); + private final Path logDirectory; private final Pattern logFilePattern; @@ -51,61 +57,110 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream outputStream, Instant from, Instant to) { + void writeLogs(OutputStream out, Instant from, Instant to) { + double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; + double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); try { - List<Path> logs = getMatchingFiles(from, to); - for (int i = 0; i < logs.size(); i++) { - Path log = logs.get(i); - boolean zipped = log.toString().endsWith(".gz"); - try (InputStream in = Files.newInputStream(log)) { - InputStream inProxy; - - // If the log needs filtering, possibly unzip (and rezip) it, and filter its lines on timestamp. - if (i == 0 || i == logs.size() - 1) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zipped ? new GZIPOutputStream(buffer) : buffer, UTF_8))) { - for (String line; (line = reader.readLine()) != null; ) { - String[] parts = line.split("\t"); - if (parts.length != 7) - continue; - - Instant at = Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS); - if (at.isAfter(from) && ! at.isAfter(to)) { - writer.write(line); - writer.newLine(); - } - } - } - inProxy = new ByteArrayInputStream(buffer.toByteArray()); + for (List<Path> logs : getMatchingFiles(from, to)) { + List<LogLineIterator> logLineIterators = new ArrayList<>(); + try { + // Logs in each sub-list contain entries covering the same time interval, so do a merge sort while reading + for (Path log : logs) + logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds)); + + Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, + Comparator.comparingDouble(LineWithTimestamp::timestamp)); + while (lines.hasNext()) { + writer.write(lines.next().line()); + writer.newLine(); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + finally { + for (LogLineIterator ll : logLineIterators) { + try { ll.close(); } catch (IOException ignored) { } } - else - inProxy = in; - - // At the point when logs switch to un-zipped, replace the output stream with a zipping proxy. - if ( ! zipped && ! (outputStream instanceof GZIPOutputStream)) - outputStream = new GZIPOutputStream(outputStream); - - inProxy.transferTo(outputStream); } } } - catch (IOException e) { - throw new UncheckedIOException(e); - } finally { + Exceptions.uncheck(writer::flush); + } + } + + private static class LogLineIterator implements Iterator<LineWithTimestamp>, AutoCloseable { + + private final BufferedReader reader; + private final double from; + private final double to; + private LineWithTimestamp next; + + private LogLineIterator(Path log, double from, double to) throws IOException { + boolean zipped = log.toString().endsWith(".gz"); + InputStream in = Files.newInputStream(log); + this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); + this.from = from; + this.to = to; + this.next = readNext(); + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public LineWithTimestamp next() { + LineWithTimestamp current = next; + next = readNext(); + return current; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private LineWithTimestamp readNext() { try { - outputStream.close(); + for (String line; (line = reader.readLine()) != null; ) { + String[] parts = line.split("\t"); + if (parts.length != 7) + continue; + + double timestamp = Double.parseDouble(parts[0]); + if (timestamp > to) + return null; + + if (timestamp >= from) + return new LineWithTimestamp(line, timestamp); + } + return null; } catch (IOException e) { throw new UncheckedIOException(e); } } + } - /** Returns log files which may have relevant entries, sorted by modification time — the first and last must be filtered. */ - private List<Path> getMatchingFiles(Instant from, Instant to) { - Map<Path, Instant> paths = new HashMap<>(); + private static class LineWithTimestamp { + final String line; + final double timestamp; + LineWithTimestamp(String line, double timestamp) { + this.line = line; + this.timestamp = timestamp; + } + String line() { return line; } + double timestamp() { return timestamp; } + } + + /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ + private List<List<Path>> getMatchingFiles(Instant from, Instant to) { + List<Path> paths = new ArrayList<>(); try { Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() { @@ -117,7 +172,7 @@ class LogReader { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (logFilePattern.matcher(file.getFileName().toString()).matches()) - paths.put(file, attrs.lastModifiedTime().toInstant()); + paths.add(file); return FileVisitResult.CONTINUE; } @@ -132,15 +187,54 @@ class LogReader { throw new UncheckedIOException(e); } - List<Path> sorted = new ArrayList<>(); - for (var entries = paths.entrySet().stream().sorted(comparing(Map.Entry::getValue)).iterator(); entries.hasNext(); ) { - var entry = entries.next(); - if (entry.getValue().isAfter(from)) - sorted.add(entry.getKey()); - if (entry.getValue().isAfter(to)) + var logsByTimestamp = paths.stream() + .collect(Collectors.groupingBy(this::extractTimestamp, + TreeMap::new, + Collectors.toList())); + + List<List<Path>> sorted = new ArrayList<>(); + for (var entry : logsByTimestamp.entrySet()) { + if (entry.getKey().isAfter(from)) + sorted.add(entry.getValue()); + if (entry.getKey().isAfter(to)) break; } return sorted; } + /** Extracts a timestamp after all entries in the log file with the given path. */ + Instant extractTimestamp(Path path) { + String relativePath = logDirectory.relativize(path).toString(); + Matcher matcher = logArchivePathPattern.matcher(relativePath); + if (matcher.matches()) { + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + 0, + 0, + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofHours(1)); + } + matcher = vespaLogPathPattern.matcher(relativePath); + if (matcher.matches()) { + if (matcher.group(1) == null) + return Instant.MAX; + + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + Integer.parseInt(matcher.group(5)), + Integer.parseInt(matcher.group(6)), + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofSeconds(1)); + } + throw new IllegalArgumentException("Unrecognized file pattern for file at '" + path + "'"); + } + } diff --git a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java index 38f5b72336b..16cf741813c 100644 --- a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java +++ b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java @@ -5,7 +5,7 @@ import com.yahoo.protect.Process; /** * An injectable terminator of the Java vm. - * Components that encounters conditions where the vm should be terminator should + * Components that encounters conditions where the vm should be terminated should * request an instance of this injected. That makes termination testable * as tests can create subclasses of this which register the termination request * rather than terminating. diff --git a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java deleted file mode 100644 index 6bc8b395e00..00000000000 --- a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.cloud; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class SystemInfoTest { - - @Test - public void testSystemInfo() { - Zone zone = new Zone(Environment.dev, "us-west-1"); - SystemInfo info = new SystemInfo(zone); - assertEquals(zone, info.zone()); - } - - @Test - public void testZone() { - Zone zone = Zone.from("dev.us-west-1"); - zone = Zone.from(zone.toString()); - assertEquals(Environment.dev, zone.environment()); - assertEquals("us-west-1", zone.region()); - Zone sameZone = Zone.from("dev.us-west-1"); - assertEquals(sameZone.hashCode(), zone.hashCode()); - assertEquals(sameZone, zone); - - try { - Zone.from("invalid"); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("A zone string must be on the form [environment].[region], but was 'invalid'", - e.getMessage()); - } - - try { - Zone.from("invalid.us-west-1"); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Invalid zone 'invalid.us-west-1': No environment named 'invalid'", e.getMessage()); - } - } - -} diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 01dcb885a97..ab0d0d54675 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -47,12 +47,12 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream outputStream, Instant from, Instant to) { + protected void writeLogs(OutputStream out, Instant from, Instant to) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { - outputStream.write("newer log".getBytes()); + out.write("newer log".getBytes()); } else { - outputStream.write("older log".getBytes()); + out.write("older log".getBytes()); } } catch (Exception e) {} } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index c68facf4f01..3f7a78e13be 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -12,7 +12,7 @@ import java.io.OutputStream; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.time.Instant; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; @@ -26,47 +26,56 @@ public class LogReaderTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Path logDirectory = fileSystem.getPath("/opt/vespa/logs"); - private static final String log1 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; - private static final String log2 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; + private static final String logv11 = "3600.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; + private static final String logv = "90000.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; + private static final String log100 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tsecond\n"; + private static final String log101 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; + private static final String log110 = "3600.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tthird\n"; + private static final String log200 = "86400.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; @Before public void setup() throws IOException { - Files.createDirectories(logDirectory.resolve("subfolder")); - - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("log1.log.gz"), compress(log1)), - FileTime.from(Instant.ofEpochMilli(123))); - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("subfolder/log2.log"), log2.getBytes(UTF_8)), - FileTime.from(Instant.ofEpochMilli(234))); - + // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. + // Multiple entries may exist for each hour. + Files.createDirectories(logDirectory.resolve("1970/01/01")); + Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress(log100)); + Files.write(logDirectory.resolve("1970/01/01/00-1"), log101.getBytes(UTF_8)); + Files.write(logDirectory.resolve("1970/01/01/01-0.gz"), compress(log110)); + + Files.createDirectories(logDirectory.resolve("1970/01/02")); + Files.write(logDirectory.resolve("1970/01/02/00-0"), log200.getBytes(UTF_8)); + + // Vespa log file names are the second-truncated timestamp of the last entry. + // The current log file has no timestamp suffix. + Files.write(logDirectory.resolve("vespa.log-1970-01-01.01-00-00"), logv11.getBytes(UTF_8)); + Files.write(logDirectory.resolve("vespa.log"), logv.getBytes(UTF_8)); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(160)); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050)); - assertEquals("", decompress(baos.toByteArray())); + assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - logReader.writeLogs(baos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log2, decompress(baos.toByteArray())); + assertEquals(log101 + logv11, baos.toString(UTF_8)); } @Test public void testZippedStreaming() throws IOException { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log1 + log2, decompress(zippedBaos.toByteArray())); + assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } private byte[] compress(String input) throws IOException { @@ -77,10 +86,4 @@ public class LogReaderTest { return baos.toByteArray(); } - private String decompress(byte[] input) throws IOException { - if (input.length == 0) return ""; - byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); - return new String(decompressed); - } - } |