diff options
Diffstat (limited to 'jdisc_http_service/src/main/java/com')
11 files changed, 179 insertions, 346 deletions
diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java index cdb4febb775..5c1a549070c 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -9,14 +9,13 @@ import com.yahoo.component.provider.ComponentRegistry; * Logs to all the configured access logs. * * @author Tony Vaagenes - * @author bjorncs */ -public class AccessLog implements RequestLog { +public class AccessLog { - private final ComponentRegistry<RequestLogHandler> implementers; + private ComponentRegistry<AccessLogInterface> implementers; @Inject - public AccessLog(ComponentRegistry<RequestLogHandler> implementers) { + public AccessLog(ComponentRegistry<AccessLogInterface> implementers) { this.implementers = implementers; } @@ -24,10 +23,9 @@ public class AccessLog implements RequestLog { return new AccessLog(new ComponentRegistry<>()); } - @Override - public void log(RequestLogEntry entry) { - for (RequestLogHandler handler: implementers.allComponents()) { - handler.log(entry); + public void log(AccessLogEntry accessLogEntry) { + for (AccessLogInterface log: implementers.allComponents()) { + log.log(accessLogEntry); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogHandler.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogInterface.java index 85df08e4abb..2523174abef 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogInterface.java @@ -4,6 +4,6 @@ package com.yahoo.container.logging; /** * @author Tony Vaagenes */ -public interface RequestLogHandler { - void log(RequestLogEntry entry); +public interface AccessLogInterface { + void log(AccessLogEntry accessLogEntry); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogSampler.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogSampler.java new file mode 100644 index 00000000000..12d29c2f333 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogSampler.java @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Samples entries from access log. It first samples every query until it have some data, and then sub-samples + * much less frequently to reduce CPU usage and latency impact. It only samples successful requests and requests + * that starts with /search. + * + * @author dybis + */ +public class AccessLogSampler implements AccessLogInterface { + + private final AtomicLong accessLineCounter = new AtomicLong(0); + private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper; + + public AccessLogSampler(CircularArrayAccessLogKeeper circularArrayAccessLogKeeper) { + this.circularArrayAccessLogKeeper = circularArrayAccessLogKeeper; + } + + @Override + public void log(AccessLogEntry accessLogEntry) { + if (accessLogEntry.getStatusCode() != 200) { + return; + } + String uriString = accessLogEntry.getRawPath(); + if (! uriString.startsWith("/search")) { + return; + } + final long count = accessLineCounter.incrementAndGet(); + if (count >= CircularArrayAccessLogKeeper.SIZE && count % CircularArrayAccessLogKeeper.SIZE != 0) { + return; + } + circularArrayAccessLogKeeper.addUri(uriString); + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONAccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONAccessLog.java index aa2df959a4c..5a188d68e89 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONAccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONAccessLog.java @@ -11,7 +11,7 @@ import java.util.logging.Level; * @author frodelu * @author Tony Vaagenes */ -public final class JSONAccessLog implements RequestLogHandler { +public final class JSONAccessLog implements AccessLogInterface { private final AccessLogHandler logHandler; private final JSONFormatter formatter; @@ -22,8 +22,8 @@ public final class JSONAccessLog implements RequestLogHandler { } @Override - public void log(RequestLogEntry entry) { - logHandler.access.log(Level.INFO, formatter.format(entry) + '\n'); + public void log(AccessLogEntry logEntry) { + logHandler.access.log(Level.INFO, formatter.format(logEntry) + '\n'); } // TODO: This is never called. We should have a DI provider and call this method from its deconstruct. diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java index dc0acc4fbac..b6ceaa08007 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -12,7 +12,8 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; import java.security.Principal; -import java.util.Collection; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,76 +42,76 @@ public class JSONFormatter { } /** - * The main method for formatting the associated {@link RequestLogEntry} as a Vespa JSON access log string + * The main method for formatting the associated {@link AccessLogEntry} as a Vespa JSON access log string * * @return The Vespa JSON access log string without trailing newline */ - public String format(RequestLogEntry entry) { + public String format(AccessLogEntry accessLogEntry) { ByteArrayOutputStream logLine = new ByteArrayOutputStream(); try { JsonGenerator generator = generatorFactory.createGenerator(logLine, JsonEncoding.UTF8); generator.writeStartObject(); - String peerAddress = entry.peerAddress().get(); - generator.writeStringField("ip", peerAddress); - long time = entry.timestamp().get().toEpochMilli(); - generator.writeNumberField("time", toTimestampInSeconds(time)); - generator.writeNumberField("duration", durationAsSeconds(entry.duration().get().toMillis())); - generator.writeNumberField("responsesize", entry.contentSize().orElse(0)); - generator.writeNumberField("code", entry.statusCode().orElse(0)); - generator.writeStringField("method", entry.httpMethod().orElse("")); - generator.writeStringField("uri", getNormalizedURI(entry.rawPath().get(), entry.rawQuery().orElse(null))); - generator.writeStringField("version", entry.httpVersion().orElse("")); - generator.writeStringField("agent", entry.userAgent().orElse("")); - generator.writeStringField("host", entry.hostString().orElse("")); - generator.writeStringField("scheme", entry.scheme().orElse(null)); - generator.writeNumberField("localport", entry.localPort().getAsInt()); - - String connectionId = entry.connectionId().orElse(null); + generator.writeStringField("ip", accessLogEntry.getIpV4Address()); + generator.writeNumberField("time", toTimestampInSeconds(accessLogEntry.getTimeStampMillis())); + generator.writeNumberField("duration", durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); + generator.writeNumberField("responsesize", accessLogEntry.getReturnedContentSize()); + generator.writeNumberField("code", accessLogEntry.getStatusCode()); + generator.writeStringField("method", accessLogEntry.getHttpMethod()); + generator.writeStringField("uri", getNormalizedURI(accessLogEntry.getRawPath(), accessLogEntry.getRawQuery().orElse(null))); + generator.writeStringField("version", accessLogEntry.getHttpVersion()); + generator.writeStringField("agent", accessLogEntry.getUserAgent()); + generator.writeStringField("host", accessLogEntry.getHostString()); + generator.writeStringField("scheme", accessLogEntry.getScheme()); + generator.writeNumberField("localport", accessLogEntry.getLocalPort()); + + String connectionId = accessLogEntry.getConnectionId().orElse(null); if (connectionId != null) { generator.writeStringField("connection", connectionId); } - Principal principal = entry.userPrincipal().orElse(null); + Principal principal = accessLogEntry.getUserPrincipal(); if (principal != null) { generator.writeStringField("user-principal", principal.getName()); } - String remoteAddress = entry.remoteAddress().orElse(null); - int remotePort = entry.remotePort().orElse(0); + Principal sslPrincipal = accessLogEntry.getSslPrincipal(); + if (sslPrincipal != null) { + generator.writeStringField("ssl-principal", sslPrincipal.getName()); + } + // Only add remote address/port fields if relevant - if (remoteAddressDiffers(peerAddress, remoteAddress)) { - generator.writeStringField("remoteaddr", remoteAddress); - if (remotePort > 0) { - generator.writeNumberField("remoteport", remotePort); + if (remoteAddressDiffers(accessLogEntry.getIpV4Address(), accessLogEntry.getRemoteAddress())) { + generator.writeStringField("remoteaddr", accessLogEntry.getRemoteAddress()); + if (accessLogEntry.getRemotePort() > 0) { + generator.writeNumberField("remoteport", accessLogEntry.getRemotePort()); } } // Only add peer address/port fields if relevant - if (peerAddress != null) { - generator.writeStringField("peeraddr", peerAddress); + if (accessLogEntry.getPeerAddress() != null) { + generator.writeStringField("peeraddr", accessLogEntry.getPeerAddress()); - int peerPort = entry.peerPort().getAsInt(); - if (peerPort > 0 && peerPort != remotePort) { + int peerPort = accessLogEntry.getPeerPort(); + if (peerPort > 0 && peerPort != accessLogEntry.getRemotePort()) { generator.writeNumberField("peerport", peerPort); } } - TraceNode trace = entry.traceNode().orElse(null); + TraceNode trace = accessLogEntry.getTrace(); if (trace != null) { long timestamp = trace.timestamp(); if (timestamp == 0L) { - timestamp = time; + timestamp = accessLogEntry.getTimeStampMillis(); } trace.accept(new TraceRenderer(generator, timestamp)); } // Only add search sub block of this is a search request - if (isSearchRequest(entry)) { - HitCounts hitCounts = entry.hitCounts().get(); + if (isSearchRequest(accessLogEntry)) { generator.writeObjectFieldStart("search"); - generator.writeNumberField("totalhits", getTotalHitCount(hitCounts)); - generator.writeNumberField("hits", getRetrievedHitCount(hitCounts)); - Coverage c = hitCounts.getCoverage(); + generator.writeNumberField("totalhits", getTotalHitCount(accessLogEntry.getHitCounts())); + generator.writeNumberField("hits", getRetrievedHitCount(accessLogEntry.getHitCounts())); + Coverage c = accessLogEntry.getHitCounts().getCoverage(); if (c != null) { generator.writeObjectFieldStart(COVERAGE); generator.writeNumberField(COVERAGE_COVERAGE, c.getResultPercentage()); @@ -134,17 +135,16 @@ public class JSONFormatter { // Add key/value access log entries. Keys with single values are written as single // string value fields while keys with multiple values are written as string arrays - Collection<String> keys = entry.extraAttributeKeys(); - if (!keys.isEmpty()) { + Map<String,List<String>> keyValues = accessLogEntry.getKeyValues(); + if (keyValues != null && !keyValues.isEmpty()) { generator.writeObjectFieldStart("attributes"); - for (String key : keys) { - Collection<String> values = entry.extraAttributeValues(key); - if (values.size() == 1) { - generator.writeStringField(key, values.iterator().next()); + for (Map.Entry<String,List<String>> entry : keyValues.entrySet()) { + if (entry.getValue().size() == 1) { + generator.writeStringField(entry.getKey(), entry.getValue().get(0)); } else { - generator.writeFieldName(key); + generator.writeFieldName(entry.getKey()); generator.writeStartArray(); - for (String s : values) { + for (String s : entry.getValue()) { generator.writeString(s); } generator.writeEndArray(); @@ -168,8 +168,8 @@ public class JSONFormatter { return remoteAddress != null && !Objects.equals(ipV4Address, remoteAddress); } - private boolean isSearchRequest(RequestLogEntry entry) { - return entry != null && entry.hitCounts().isPresent(); + private boolean isSearchRequest(AccessLogEntry logEntry) { + return logEntry != null && (logEntry.getHitCounts() != null); } private long getTotalHitCount(HitCounts counts) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLog.java deleted file mode 100644 index 2090ba1b9f1..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLog.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.logging; - -/** - * Access logging for requests - * - * @author bjorncs - */ -public interface RequestLog { - - void log(RequestLogEntry entry); - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java deleted file mode 100644 index b771ea11ed0..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.logging; - -import com.yahoo.yolean.trace.TraceNode; - -import java.security.Principal; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static java.util.Objects.requireNonNull; - -/** - * A immutable request log entry - * - * @author bjorncs - */ -public class RequestLogEntry { - - private final String connectionId; - private final Instant timestamp; - private final Duration duration; - private final int localPort; - private final String peerAddress; - private final int peerPort; - private final String remoteAddress; - private final int remotePort; - private final String userAgent; - private final String referer; - private final String httpMethod; - private final String httpVersion; - private final String hostString; - private final int statusCode; - private final long contentSize; - private final String scheme; - private final String rawPath; - private final String rawQuery; - private final Principal userPrincipal; - private final HitCounts hitCounts; - private final TraceNode traceNode; - private final Map<String, Collection<String>> extraAttributes; - - private RequestLogEntry(Builder builder) { - this.connectionId = builder.connectionId; - this.timestamp = builder.timestamp; - this.duration = builder.duration; - this.localPort = builder.localPort; - this.peerAddress = builder.peerAddress; - this.peerPort = builder.peerPort; - this.remoteAddress = builder.remoteAddress; - this.remotePort = builder.remotePort; - this.userAgent = builder.userAgent; - this.referer = builder.referer; - this.httpMethod = builder.httpMethod; - this.httpVersion = builder.httpVersion; - this.hostString = builder.hostString; - this.statusCode = builder.statusCode; - this.contentSize = builder.contentSize; - this.scheme = builder.scheme; - this.rawPath = builder.rawPath; - this.rawQuery = builder.rawQuery; - this.userPrincipal = builder.userPrincipal; - this.hitCounts = builder.hitCounts; - this.traceNode = builder.traceNode; - this.extraAttributes = copyExtraAttributes(builder.extraAttributes); - } - - public Optional<String> connectionId() { return Optional.ofNullable(connectionId); } - public Optional<Instant> timestamp() { return Optional.ofNullable(timestamp); } - public Optional<Duration> duration() { return Optional.ofNullable(duration); } - public OptionalInt localPort() { return optionalInt(localPort); } - public Optional<String> peerAddress() { return Optional.ofNullable(peerAddress); } - public OptionalInt peerPort() { return optionalInt(peerPort); } - public Optional<String> remoteAddress() { return Optional.ofNullable(remoteAddress); } - public OptionalInt remotePort() { return optionalInt(remotePort); } - public Optional<String> userAgent() { return Optional.ofNullable(userAgent); } - public Optional<String> referer() { return Optional.ofNullable(referer); } - public Optional<String> httpMethod() { return Optional.ofNullable(httpMethod); } - public Optional<String> httpVersion() { return Optional.ofNullable(httpVersion); } - public Optional<String> hostString() { return Optional.ofNullable(hostString); } - public OptionalInt statusCode() { return optionalInt(statusCode); } - public OptionalLong contentSize() { return optionalLong(contentSize); } - public Optional<String> scheme() { return Optional.ofNullable(scheme); } - public Optional<String> rawPath() { return Optional.ofNullable(rawPath); } - public Optional<String> rawQuery() { return Optional.ofNullable(rawQuery); } - public Optional<Principal> userPrincipal() { return Optional.ofNullable(userPrincipal); } - public Optional<HitCounts> hitCounts() { return Optional.ofNullable(hitCounts); } - public Optional<TraceNode> traceNode() { return Optional.ofNullable(traceNode); } - public Collection<String> extraAttributeKeys() { return Collections.unmodifiableCollection(extraAttributes.keySet()); } - public Collection<String> extraAttributeValues(String key) { return Collections.unmodifiableCollection(extraAttributes.get(key)); } - - private static OptionalInt optionalInt(int value) { - if (value == -1) return OptionalInt.empty(); - return OptionalInt.of(value); - } - - private static OptionalLong optionalLong(long value) { - if (value == -1) return OptionalLong.empty(); - return OptionalLong.of(value); - } - - private static Map<String, Collection<String>> copyExtraAttributes(Map<String, Collection<String>> extraAttributes) { - Map<String, Collection<String>> copy = new HashMap<>(); - extraAttributes.forEach((key, value) -> copy.put(key, new ArrayList<>(value))); - return copy; - } - - public static class Builder { - - private String connectionId; - private Instant timestamp; - private Duration duration; - private int localPort = -1; - private String peerAddress; - private int peerPort = -1; - private String remoteAddress; - private int remotePort = -1; - private String userAgent; - private String referer; - private String httpMethod; - private String httpVersion; - private String hostString; - private int statusCode = -1; - private long contentSize = -1; - private String scheme; - private String rawPath; - private String rawQuery; - private Principal userPrincipal; - private HitCounts hitCounts; - private TraceNode traceNode; - private final Map<String, Collection<String>> extraAttributes = new HashMap<>(); - - public Builder connectionId(String connectionId) { this.connectionId = requireNonNull(connectionId); return this; } - public Builder timestamp(Instant timestamp) { this.timestamp = requireNonNull(timestamp); return this; } - public Builder duration(Duration duration) { this.duration = requireNonNull(duration); return this; } - public Builder localPort(int localPort) { this.localPort = requireNonNegative(localPort); return this; } - public Builder peerAddress(String peerAddress) { this.peerAddress = requireNonNull(peerAddress); return this; } - public Builder peerPort(int peerPort) { this.peerPort = requireNonNegative(peerPort); return this; } - public Builder remoteAddress(String remoteAddress) { this.remoteAddress = requireNonNull(remoteAddress); return this; } - public Builder remotePort(int remotePort) { this.remotePort = requireNonNegative(remotePort); return this; } - public Builder userAgent(String userAgent) { this.userAgent = requireNonNull(userAgent); return this; } - public Builder referer(String referer) { this.referer = requireNonNull(referer); return this; } - public Builder httpMethod(String httpMethod) { this.httpMethod = requireNonNull(httpMethod); return this; } - public Builder httpVersion(String httpVersion) { this.httpVersion = requireNonNull(httpVersion); return this; } - public Builder hostString(String hostString) { this.hostString = requireNonNull(hostString); return this; } - public Builder statusCode(int statusCode) { this.statusCode = requireNonNegative(statusCode); return this; } - public Builder contentSize(long contentSize) { this.contentSize = requireNonNegative(contentSize); return this; } - public Builder scheme(String scheme) { this.scheme = requireNonNull(scheme); return this; } - public Builder rawPath(String rawPath) { this.rawPath = requireNonNull(rawPath); return this; } - public Builder rawQuery(String rawQuery) { this.rawQuery = requireNonNull(rawQuery); return this; } - public Builder userPrincipal(Principal userPrincipal) { this.userPrincipal = requireNonNull(userPrincipal); return this; } - public Builder hitCounts(HitCounts hitCounts) { this.hitCounts = requireNonNull(hitCounts); return this; } - public Builder traceNode(TraceNode traceNode) { this.traceNode = requireNonNull(traceNode); return this; } - public Builder addExtraAttribute(String key, String value) { - this.extraAttributes.computeIfAbsent(requireNonNull(key), __ -> new ArrayList<>()).add(requireNonNull(value)); - return this; - } - public Builder addExtraAttributes(String key, Collection<String> values) { - this.extraAttributes.computeIfAbsent(requireNonNull(key), __ -> new ArrayList<>()).addAll(requireNonNull(values)); - return this; - } - public RequestLogEntry build() { return new RequestLogEntry(this); } - - private static int requireNonNegative(int value) { - if (value < 0) throw new IllegalArgumentException("Value must be non-negative: " + value); - return value; - } - - private static long requireNonNegative(long value) { - if (value < 0) throw new IllegalArgumentException("Value must be non-negative: " + value); - return value; - } - } -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/VespaAccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/VespaAccessLog.java index 1040846dda3..054fc0fcbf7 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/VespaAccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/VespaAccessLog.java @@ -12,7 +12,7 @@ import java.util.logging.Level; * @author Bjorn Borud * @author Oyvind Bakksjo */ -public final class VespaAccessLog implements RequestLogHandler { +public final class VespaAccessLog implements AccessLogInterface { private static final ThreadLocal<SimpleDateFormat> dateFormat = ThreadLocal.withInitial(VespaAccessLog::createDateFormat); @@ -92,20 +92,20 @@ public final class VespaAccessLog implements RequestLogHandler { } @Override - public void log(RequestLogEntry entry) { + public void log(final AccessLogEntry accessLogEntry) { writeLog( - entry.peerAddress().get(), - null, + accessLogEntry.getIpV4Address(), + accessLogEntry.getUser(), getRequest( - entry.httpMethod().orElse(null), - entry.rawPath().orElse(null), - entry.rawQuery().orElse(null), - entry.httpVersion().orElse(null)), - entry.referer().orElse(null), - entry.userAgent().orElse(null), - entry.duration().get().toMillis(), - entry.contentSize().orElse(0L), - entry.hitCounts().orElse(null), - entry.statusCode().orElse(0)); + accessLogEntry.getHttpMethod(), + accessLogEntry.getRawPath(), + accessLogEntry.getRawQuery().orElse(null), + accessLogEntry.getHttpVersion()), + accessLogEntry.getReferer(), + accessLogEntry.getUserAgent(), + accessLogEntry.getDurationBetweenRequestResponseMillis(), + accessLogEntry.getReturnedContentSize(), + accessLogEntry.getHitCounts(), + accessLogEntry.getStatusCode()); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index e82373fdaae..32790534f86 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -4,22 +4,20 @@ package com.yahoo.jdisc.http.server.jetty; import com.google.common.base.Objects; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.AccessLogEntry; -import com.yahoo.container.logging.RequestLog; -import com.yahoo.container.logging.RequestLogEntry; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.servlet.ServletRequest; import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.RequestLog; import org.eclipse.jetty.server.Response; import org.eclipse.jetty.util.component.AbstractLifeCycle; import javax.servlet.http.HttpServletRequest; import java.security.Principal; -import java.time.Duration; -import java.time.Instant; +import java.security.cert.X509Certificate; import java.util.List; +import java.util.Optional; import java.util.OptionalInt; import java.util.UUID; -import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,19 +30,19 @@ import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnectorLoca * @author Oyvind Bakksjo * @author bjorncs */ -class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty.server.RequestLog { +class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog { private static final Logger logger = Logger.getLogger(AccessLogRequestLog.class.getName()); // HTTP headers that are logged as extra key-value-pairs in access log entries private static final List<String> LOGGED_REQUEST_HEADERS = List.of("Vespa-Client-Version"); - private final RequestLog requestLog; + private final AccessLog accessLog; private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; - AccessLogRequestLog(RequestLog requestLog, ServerConfig.AccessLog config) { - this.requestLog = requestLog; + AccessLogRequestLog(AccessLog accessLog, ServerConfig.AccessLog config) { + this.accessLog = accessLog; this.remoteAddressHeaders = config.remoteAddressHeaders(); this.remotePortHeaders = config.remotePortHeaders(); } @@ -52,67 +50,83 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty @Override public void log(Request request, Response response) { try { - RequestLogEntry.Builder builder = new RequestLogEntry.Builder(); - - String peerAddress = request.getRemoteAddr(); - int peerPort = request.getRemotePort(); - long startTime = request.getTimeStamp(); - long endTime = System.currentTimeMillis(); - builder.peerAddress(peerAddress) - .peerPort(peerPort) - .localPort(getConnectorLocalPort(request)) - .timestamp(Instant.ofEpochMilli(startTime)) - .duration(Duration.ofMillis(endTime - startTime)) - .contentSize(response.getHttpChannel().getBytesWritten()) - .statusCode(response.getCommittedMetaData().getStatus()); - - addNonNullValue(builder, request.getMethod(), RequestLogEntry.Builder::httpMethod); - addNonNullValue(builder, request.getRequestURI(), RequestLogEntry.Builder::rawPath); - addNonNullValue(builder, request.getProtocol(), RequestLogEntry.Builder::httpVersion); - addNonNullValue(builder, request.getScheme(), RequestLogEntry.Builder::scheme); - addNonNullValue(builder, request.getHeader("User-Agent"), RequestLogEntry.Builder::userAgent); - addNonNullValue(builder, request.getHeader("Host"), RequestLogEntry.Builder::hostString); - addNonNullValue(builder, request.getHeader("Referer"), RequestLogEntry.Builder::referer); - addNonNullValue(builder, request.getQueryString(), RequestLogEntry.Builder::rawQuery); - - Principal principal = (Principal) request.getAttribute(ServletRequest.JDISC_REQUEST_PRINCIPAL); - addNonNullValue(builder, principal, RequestLogEntry.Builder::userPrincipal); - - String requestFilterId = (String) request.getAttribute(ServletRequest.JDISC_REQUEST_CHAIN); - addNonNullValue(builder, requestFilterId, (b, chain) -> b.addExtraAttribute("request-chain", chain)); - - String responseFilterId = (String) request.getAttribute(ServletRequest.JDISC_RESPONSE_CHAIN); - addNonNullValue(builder, responseFilterId, (b, chain) -> b.addExtraAttribute("response-chain", chain)); + AccessLogEntry accessLogEntry = Optional.ofNullable(request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY)) + .map(AccessLogEntry.class::cast) + .orElseGet(AccessLogEntry::new); + + accessLogEntry.setRawPath(request.getRequestURI()); + String queryString = request.getQueryString(); + if (queryString != null) { + accessLogEntry.setRawQuery(queryString); + } - UUID connectionId = (UUID) request.getAttribute(JettyConnectionLogger.CONNECTION_ID_REQUEST_ATTRIBUTE); - addNonNullValue(builder, connectionId, (b, uuid) -> b.connectionId(uuid.toString())); + accessLogEntry.setUserAgent(request.getHeader("User-Agent")); + accessLogEntry.setHttpMethod(request.getMethod()); + accessLogEntry.setHostString(request.getHeader("Host")); + accessLogEntry.setReferer(request.getHeader("Referer")); + String peerAddress = request.getRemoteAddr(); + accessLogEntry.setIpV4Address(peerAddress); + accessLogEntry.setPeerAddress(peerAddress); String remoteAddress = getRemoteAddress(request); if (!Objects.equal(remoteAddress, peerAddress)) { - builder.remoteAddress(remoteAddress); + accessLogEntry.setRemoteAddress(remoteAddress); } + + int peerPort = request.getRemotePort(); + accessLogEntry.setPeerPort(peerPort); int remotePort = getRemotePort(request); if (remotePort != peerPort) { - builder.remotePort(remotePort); + accessLogEntry.setRemotePort(remotePort); + } + accessLogEntry.setHttpVersion(request.getProtocol()); + accessLogEntry.setScheme(request.getScheme()); + accessLogEntry.setLocalPort(getConnectorLocalPort(request)); + Principal principal = (Principal) request.getAttribute(ServletRequest.JDISC_REQUEST_PRINCIPAL); + if (principal != null) { + accessLogEntry.setUserPrincipal(principal); + } + X509Certificate[] clientCert = (X509Certificate[]) request.getAttribute(ServletRequest.SERVLET_REQUEST_X509CERT); + if (clientCert != null && clientCert.length > 0) { + accessLogEntry.setSslPrincipal(clientCert[0].getSubjectX500Principal()); + } + String sslSessionId = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_SSL_SESSION_ID); + if (sslSessionId != null) { + accessLogEntry.addKeyValue("ssl-session-id", sslSessionId); } + String cipherSuite = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_CIPHER_SUITE); + if (cipherSuite != null) { + accessLogEntry.addKeyValue("cipher-suite", cipherSuite); + } + String requestFilterId = (String) request.getAttribute(ServletRequest.JDISC_REQUEST_CHAIN); + if (requestFilterId != null) { + accessLogEntry.addKeyValue("request-chain", requestFilterId); + } + String responseFilterId = (String) request.getAttribute(ServletRequest.JDISC_RESPONSE_CHAIN); + if (responseFilterId != null) { + accessLogEntry.addKeyValue("response-chain", responseFilterId); + } + + long startTime = request.getTimeStamp(); + long endTime = System.currentTimeMillis(); + accessLogEntry.setTimeStamp(startTime); + accessLogEntry.setDurationBetweenRequestResponse(endTime - startTime); + accessLogEntry.setReturnedContentSize(response.getHttpChannel().getBytesWritten()); + accessLogEntry.setStatusCode(response.getCommittedMetaData().getStatus()); + LOGGED_REQUEST_HEADERS.forEach(header -> { String value = request.getHeader(header); if (value != null) { - builder.addExtraAttribute(header, value); + accessLogEntry.addKeyValue(header, value); } }); - AccessLogEntry accessLogEntry = (AccessLogEntry) request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY); - if (accessLogEntry != null) { - var extraAttributes = accessLogEntry.getKeyValues(); - if (extraAttributes != null) { - extraAttributes.forEach(builder::addExtraAttributes); - } - addNonNullValue(builder, accessLogEntry.getHitCounts(), RequestLogEntry.Builder::hitCounts); - addNonNullValue(builder, accessLogEntry.getTrace(), RequestLogEntry.Builder::traceNode); + UUID connectionId = (UUID) request.getAttribute(JettyConnectionLogger.CONNECTION_ID_REQUEST_ATTRIBUTE); + if (connectionId != null) { + accessLogEntry.setConnectionId(connectionId.toString()); } - requestLog.log(builder.build()); + accessLog.log(accessLogEntry); } catch (Exception e) { // Catching any exceptions here as it is unclear how Jetty handles exceptions from a RequestLog. logger.log(Level.SEVERE, "Failed to log access log entry: " + e.getMessage(), e); @@ -146,11 +160,4 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty } } - private static <T> void addNonNullValue( - RequestLogEntry.Builder builder, T value, BiConsumer<RequestLogEntry.Builder, T> setter) { - if (value != null) { - setter.accept(builder, value); - } - } - } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 510c561c10f..1e077c32ea1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -7,7 +7,6 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.ConnectionLog; -import com.yahoo.container.logging.RequestLog; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ServerConfig; @@ -74,7 +73,7 @@ public class JettyHttpServer extends AbstractServerProvider { ComponentRegistry<ConnectorFactory> connectorFactories, ComponentRegistry<ServletHolder> servletHolders, FilterInvoker filterInvoker, - RequestLog requestLog, + AccessLog accessLog, ConnectionLog connectionLog) { super(container); if (connectorFactories.allComponents().isEmpty()) @@ -84,7 +83,7 @@ public class JettyHttpServer extends AbstractServerProvider { server = new Server(); server.setStopTimeout((long)(serverConfig.stopTimeout() * 1000.0)); - server.setRequestLog(new AccessLogRequestLog(requestLog, serverConfig.accessLog())); + server.setRequestLog(new AccessLogRequestLog(accessLog, serverConfig.accessLog())); setupJmx(server, serverConfig); configureJettyThreadpool(server, serverConfig); JettyConnectionLogger connectionLogger = new JettyConnectionLogger(serverConfig.connectionLog(), connectionLog); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/VoidRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/VoidRequestLog.java deleted file mode 100644 index 9db5ba99115..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/VoidRequestLog.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.server.jetty; - -import com.yahoo.container.logging.RequestLog; -import com.yahoo.container.logging.RequestLogEntry; - -/** - * @author bjorncs - */ -public class VoidRequestLog implements RequestLog { - - @Override public void log(RequestLogEntry entry) {} - -} |