diff options
6 files changed, 33 insertions, 29 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 0baf1bba4fa..3a9ebde9da1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -83,6 +83,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta private final List<ClusterStateBundle> convergedStates = new ArrayList<>(); private final Queue<RemoteClusterControllerTask> remoteTasks = new LinkedList<>(); private final MetricUpdater metricUpdater; + private final LegacyIndexPageRequestHandler indexPageRequestHandler; private boolean isMaster = false; private boolean inMasterMoratorium = false; @@ -141,9 +142,10 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta this.statusRequestRouter.addHandler( "^/clusterstate", new ClusterStateRequestHandler(stateVersionTracker)); + this.indexPageRequestHandler = new LegacyIndexPageRequestHandler(timer, cluster, masterElectionHandler, stateVersionTracker, eventLog, options); this.statusRequestRouter.addHandler( "^/$", - new LegacyIndexPageRequestHandler(timer, cluster, masterElectionHandler, stateVersionTracker, eventLog, options)); + indexPageRequestHandler); propagateOptions(); } @@ -488,6 +490,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio()); communicator.propagateOptions(options); + indexPageRequestHandler.propagateOptions(options); if (nodeLookup instanceof SlobrokClient) { ((SlobrokClient) nodeLookup).setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()); @@ -677,7 +680,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } boolean sentAny = false; // Give nodes a fair chance to respond first time to state gathering requests, so we don't - // disturb system when we take over. Allow anyways if we have states from all nodes. + // disturb system when we take over. Allow anyway if we have states from all nodes. long currentTime = timer.getCurrentTimeInMillis(); if ((currentTime >= firstAllowedStateBroadcast || cluster.allStatesReported()) && currentTime >= nextStateSendTime) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 33613911c85..235b9671ef9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -42,7 +42,8 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa private final StateVersionTracker stateVersionTracker; private final EventLog eventLog; private final long startedTime; - private final FleetControllerOptions options; + + private FleetControllerOptions options; public LegacyIndexPageRequestHandler(Timer timer, ContentCluster cluster, @@ -59,6 +60,10 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa this.options = options; } + public void propagateOptions(FleetControllerOptions options) { + this.options = options; + } + @Override public StatusPageResponse handle(StatusPageServer.HttpRequest request) { TimeZone tz = TimeZone.getTimeZone("UTC"); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index d151eb6862d..72a399744f3 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -4,21 +4,15 @@ package com.yahoo.container.handler; import com.yahoo.component.annotation.Inject; import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.AsyncHttpResponse; -import com.yahoo.container.jdisc.ContentChannelOutputStream; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import java.io.IOException; -import java.io.InterruptedIOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; public class LogHandler extends ThreadedHttpRequestHandler { @@ -42,6 +36,8 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant to = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); + long maxLines = Optional.ofNullable(request.getProperty("maxLines")) + .map(Long::valueOf).orElse(100_000L); Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); return new AsyncHttpResponse(200) { @@ -50,7 +46,7 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) { try (output) { - logReader.writeLogs(output, from, to, hostname); + logReader.writeLogs(output, from, to, maxLines, hostname); } catch (Throwable t) { log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); @@ -62,6 +58,4 @@ public class LogHandler extends ThreadedHttpRequestHandler { }; } - - } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 93881b52eb6..2890cbfb5ab 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -59,9 +59,10 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { + void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) { double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + long linesWritten = 0; BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); for (List<Path> logs : getMatchingFiles(from, to)) { List<LogLineIterator> logLineIterators = new ArrayList<>(); @@ -73,6 +74,7 @@ class LogReader { Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, Comparator.comparingDouble(LineWithTimestamp::timestamp)); while (lines.hasNext()) { + if (linesWritten++ >= maxLines) return; String line = lines.next().line(); writer.write(line); writer.newLine(); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 3d9a2360e77..8b698b25ab2 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -51,7 +51,7 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { + protected void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { out.write("newer log".getBytes()); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 106b8cef35f..e98f8cac276 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -37,7 +37,7 @@ public class LogReaderTest { @BeforeEach public void setup() throws IOException { - logDirectory = newFolder(folder, "opt/vespa/logs").toPath(); + logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs")); // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. // Multiple entries may exist for each hour. Files.createDirectories(logDirectory.resolve("1970/01/01")); @@ -59,7 +59,7 @@ public class LogReaderTest { void testThatLogsOutsideRangeAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty()); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty()); assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @@ -68,7 +68,7 @@ public class LogReaderTest { void testThatLogsNotMatchingRegexAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); assertEquals(log101 + logv11, baos.toString(UTF_8)); } @@ -78,7 +78,7 @@ public class LogReaderTest { void testZippedStreaming() { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } @@ -87,11 +87,20 @@ public class LogReaderTest { void logsForSingeNodeIsRetrieved() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com")); assertEquals(log101 + log100 + log200, baos.toString(UTF_8)); } + @Test + void logsLimitedToMaxLines() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com")); + + assertEquals(log101 + log100, baos.toString(UTF_8)); + } + private byte[] compress1(String input) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream zip = new GZIPOutputStream(baos); @@ -100,18 +109,9 @@ public class LogReaderTest { return baos.toByteArray(); } - private byte[] compress2(String input) throws IOException { + private byte[] compress2(String input) { byte[] data = input.getBytes(); return new ZstdCompressor().compress(data, 0, data.length); } - private static File newFolder(File root, String... subDirs) throws IOException { - String subFolder = String.join("/", subDirs); - File result = new File(root, subFolder); - if (!result.mkdirs()) { - throw new IOException("Couldn't create folders " + root); - } - return result; - } - } |