1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.handler;
import com.yahoo.compress.ZstdCompressor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIf;
import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class LogReaderTest {
@TempDir
public File folder;
private Path logDirectory;
private static final String logv11 = "3600.2\tnode1.com\t5480\tcontainer\tstdout\tinfo\tfourth\n";
private static final String logv = "90000.1\tnode1.com\t5480\tcontainer\tstdout\tinfo\tlast\n";
private static final String log100a = "0.2\tnode2.com\t5480\tcontainer\tstdout\tinfo\tsecond\n";
private static final String log100b = "0.15\tnode2.com\t5480\tcontainer\tstdout\tinfo\tfirst\n";
private static final String log101 = "0.1\tnode2.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n";
private static final String log110 = "3600.1\tnode1.com\t5480\tcontainer\tstderr\twarning\tthird\n";
private static final String log200 = "86400.1\tnode2.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n";
@BeforeEach
public void setup() throws IOException {
logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs"));
// Log archive paths and file names indicate what hour they contain logs for, with the start of that hour.
// Multiple entries may exist for each hour.
Files.createDirectories(logDirectory.resolve("1970/01/01"));
// Files may contain out-of-order entries.
Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress1(log100a + log100b));
Files.writeString(logDirectory.resolve("1970/01/01/00-1"), log101);
Files.write(logDirectory.resolve("1970/01/01/01-0.zst"), compress2(log110));
Files.createDirectories(logDirectory.resolve("1970/01/02"));
Files.writeString(logDirectory.resolve("1970/01/02/00-0"), log200);
// Vespa log file names are the second-truncated timestamp of the last entry.
// The current log file has no timestamp suffix.
Files.writeString(logDirectory.resolve("vespa.log-1970-01-01.01-00-00"), logv11);
Files.writeString(logDirectory.resolve("vespa.log"), logv);
}
private static boolean hasZstdcat() {
try {
return new ProcessBuilder("zstdcat", "--version").start().waitFor() == 0;
}
catch (Exception e) {
return false;
}
}
@EnabledIf("hasZstdcat")
@Test
void testThatLogsOutsideRangeAreExcluded() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty());
assertEquals(log100b + log100a + logv11 + log110, baos.toString(UTF_8));
}
@Test
void testThatLogsNotMatchingRegexAreExcluded() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
assertEquals(log101 + logv11, baos.toString(UTF_8));
}
@EnabledIf("hasZstdcat")
@Test
void testZippedStreaming() {
ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
assertEquals(log101 + log100b + log100a + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8));
}
@Test
void logsForSingeNodeIsRetrieved() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com"));
assertEquals(log101 + log100b + log100a + log200, baos.toString(UTF_8));
}
@Test
void logsLimitedToMaxLines() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com"));
assertEquals(log101 + log100b, baos.toString(UTF_8));
}
private byte[] compress1(String input) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream zip = new GZIPOutputStream(baos);
zip.write(input.getBytes());
zip.close();
return baos.toByteArray();
}
private byte[] compress2(String input) {
byte[] data = input.getBytes();
return new ZstdCompressor().compress(data, 0, data.length);
}
}
|
