diff options
Diffstat (limited to 'container-accesslogging/src/main/java/com/yahoo/container/logging')
13 files changed, 2142 insertions, 0 deletions
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java new file mode 100644 index 00000000000..86cdd712031 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +import java.net.InetSocketAddress; +import java.net.URI; + +/** + * Logs to all the configured access logs. + * @author tonytv + */ +public class AccessLog { + + private ComponentRegistry<AccessLogInterface> implementers; + + @Inject + public AccessLog(ComponentRegistry<AccessLogInterface> implementers) { + this.implementers = implementers; + } + + public static AccessLog voidAccessLog() { + return new AccessLog(new ComponentRegistry<>()); + } + + public void log(final AccessLogEntry accessLogEntry) { + for (AccessLogInterface log: implementers.allComponents()) { + log.log(accessLogEntry); + } + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java new file mode 100644 index 00000000000..2e31898fe03 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -0,0 +1,711 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.collections.ListMap; +import org.apache.commons.lang.builder.ReflectionToStringBuilder; + +import static java.util.stream.Collectors.toMap; + +/** + * <p>Information to be logged in the access log.</p> + * + * <p>This class contains the union of all information that can be + * logged with all the supported access log formats.</p> + * + * <p>The add methods can be called multiple times, + * but the parameters should be different for each + * invocation of the same method.</p> + * + * This class is thread-safe, but the inner class {@link AdInfo} is not. + * + * @author tonytv + * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a> + */ +public class AccessLogEntry { + public enum CookieType { + b, + l, + n, + geocookie, + I, + R, + Y, + M; + } + + // Sadly, there's no way to do compile-time validation of these field references. + private static final String[] FIELDS_EXCLUDED_FROM_TOSTRING = new String[] { + "monitor" + }; + + private final Object monitor = new Object(); + + private List<AdInfo> adInfos; + private String spaceID; + + private String ipV4AddressInDotDecimalNotation; + private long timeStampMillis; + private long durationBetweenRequestResponseMillis; + private long numBytesReturned; + private URI uri; + + + private String remoteAddress; + private int remotePort; + private String peerAddress; + private int peerPort; + + private CookieType cookieType; + private String cookie; + private String weekOfRegistration; + private String profile; + private String internationalInfo; + private String contentAttribute; + private String webfactsDigitalSignature; + private String errorMessage; + private String fileName; + private String userAgent; + private String referer; + private String user; + private HitCounts hitCounts; + private String requestExtra; + private String responseExtra; + private Boolean resultFromCache; + private String httpMethod; + private String httpVersion; + private String partner; + private String adRationale; + private String incrementSlotByOneRequest; + private String zDataIncrementSlotByOneRequest; + private String hostString; + private int statusCode; + + private ListMap<String,String> keyValues=null; + + public void setCookie( CookieType type, String cookie) { + synchronized (monitor) { + requireNull(this.cookieType); + requireNull(this.cookie); + this.cookieType = type; + this.cookie = cookie; + } + } + + public CookieType getCookieType() { + synchronized (monitor) { + return cookieType; + } + } + + public String getCookie() { + synchronized (monitor) { + return cookie; + } + } + + public void setWeekOfRegistration( String weekOfRegistration ) { + synchronized (monitor) { + requireNull(this.weekOfRegistration); + this.weekOfRegistration = weekOfRegistration; + } + } + + public String getWeekOfRegistration() { + synchronized (monitor) { + return weekOfRegistration; + } + } + + public void setProfile( String profile ) { + synchronized (monitor) { + requireNull(this.profile); + this.profile = profile; + } + } + + public String getProfile() { + synchronized (monitor) { + return profile; + } + } + + public void setInternationalInfo( String intl ) { + synchronized (monitor) { + requireNull(this.internationalInfo); + this.internationalInfo = intl; + } + } + + public String getInternationalInfo() { + synchronized (monitor) { + return internationalInfo; + } + } + + public void setContentAttribute( String contentAttribute ) { + synchronized (monitor) { + requireNull(this.contentAttribute); + this.contentAttribute = contentAttribute; + } + } + + public String getContentAttribute() { + synchronized (monitor) { + return contentAttribute; + } + } + + public void setAdSpaceID(String spaceID) { + synchronized (monitor) { + requireNull(this.spaceID); + this.spaceID = spaceID; + } + } + + public String getAdSpaceID() { + synchronized (monitor) { + return spaceID; + } + } + + public void addAdInfo(AdInfo adInfo) { + synchronized (monitor) { + if (adInfos == null) { + adInfos = new ArrayList<>(); + } + adInfos.add( adInfo ); + } + } + + public List<AdInfo> getAdInfos() { + synchronized (monitor) { + if (adInfos == null) { + return Collections.emptyList(); + } + // TODO: The returned list is unmodifiable, but its elements are not. But we're all friendly here, right? + return Collections.unmodifiableList(adInfos); + } + } + + /** + * This class is NOT thread-safe. It is assumed that a single instance is created/written by a single thread, + * and all reads happen-after creation/population, i.e. no mutation after the instance is shared between threads. + */ + public static class AdInfo { + + private String adServerString; + private String adId; + private String matchId; + private String position; + private String property; + private String cpc; + private String adClientVersion; + private String linkId; + private String bidPosition; + + public void setAdID(String id) { + this.adId = id; + } + + public String getAdID() { + return adId; + } + + public void setMatchID(String id) { + this.matchId = id; + } + + public String getMatchID() { + return matchId; + } + + public void setPosition(String position) { + this.position = position; + } + + public String getPosition() { + return position; + } + + public void setProperty(String property) { + this.property = property; + } + + public String getProperty() { + return property; + } + + public void setCPC(String cpc) { + this.cpc = cpc; + } + + public String getCPC() { + return cpc; + } + + public void setAdClientVersion(String adClientVersion) { + this.adClientVersion = adClientVersion; + } + + public String getAdClientVersion() { + return adClientVersion; + } + + public void setLinkID(String id) { + this.linkId = id; + } + + public String getLinkID() { + return linkId; + } + + public void setBidPosition(String bidPosition) { + this.bidPosition = bidPosition; + } + + public String getBidPosition() { + return bidPosition; + } + + public AdInfo() {} + + AdInfo(String adServerString) { + this.adServerString = adServerString; + } + + String getAdServerString() { + return adServerString; + } + } + + public void setWebfactsDigitalSignature(String signature) { + synchronized (monitor) { + requireNull(this.webfactsDigitalSignature); + this.webfactsDigitalSignature = signature; + } + } + + public String getWebfactsDigitalSignature() { + synchronized (monitor) { + return webfactsDigitalSignature; + } + } + + public void setErrorMessage(String errorMessage) { + synchronized (monitor) { + requireNull(this.errorMessage); + this.errorMessage = errorMessage; + } + } + + public String getErrorMessage() { + synchronized (monitor) { + return errorMessage; + } + } + + public void setFileName(String fileName) { + synchronized (monitor) { + requireNull(this.fileName); + this.fileName = fileName; + } + } + + public String getFileName() { + synchronized (monitor) { + return fileName; + } + } + + public void setUserAgent(String userAgent) { + synchronized (monitor) { + requireNull(this.userAgent); + this.userAgent = userAgent; + } + } + + public String getUserAgent() { + synchronized (monitor) { + return userAgent; + } + } + + public void setReferer(String referer) { + synchronized (monitor) { + requireNull(this.referer); + this.referer = referer; + } + } + + public String getReferer() { + synchronized (monitor) { + return referer; + } + } + + public void setUser(final String user) { + synchronized (monitor) { + requireNull(this.user); + this.user = user; + } + } + + public String getUser() { + synchronized (monitor) { + return user; + } + } + + public void setHitCounts(final HitCounts hitCounts) { + synchronized (monitor) { + requireNull(this.hitCounts); + this.hitCounts = hitCounts; + } + } + + public HitCounts getHitCounts() { + synchronized (monitor) { + return hitCounts; + } + } + + public String getRequestExtra() { + synchronized (monitor) { + return requestExtra; + } + } + + public String getResponseExtra() { + synchronized (monitor) { + return responseExtra; + } + } + + public void addKeyValue(String key,String value) { + synchronized (monitor) { + if (keyValues == null) { + keyValues = new ListMap<>(); + } + keyValues.put(key,value); + } + } + + public Map<String, List<String>> getKeyValues() { + synchronized (monitor) { + if (keyValues == null) { + return null; + } + + final Map<String, List<String>> newMapWithImmutableValues = mapValues( + keyValues.entrySet(), + valueList -> Collections.unmodifiableList(new ArrayList<>(valueList))); + return Collections.unmodifiableMap(newMapWithImmutableValues); + } + } + + private static <K, V1, V2> Map<K, V2> mapValues( + final Set<Map.Entry<K, V1>> entrySet, + final Function<V1, V2> valueConverter) { + return entrySet.stream() + .collect(toMap( + entry -> entry.getKey(), + entry -> valueConverter.apply(entry.getValue()))); + } + + public void setResultFromCache(boolean fromCache) { + synchronized (monitor) { + requireNull(this.resultFromCache); + this.resultFromCache = fromCache; + } + } + + public Boolean getResultFromCache() { + synchronized (monitor) { + return resultFromCache; + } + } + + public enum HttpMethod { + GET, POST; + } + + public void setHttpMethod(HttpMethod method) { + setHttpMethod(method.toString()); + } + + public void setHttpMethod(String method) { + synchronized (monitor) { + requireNull(this.httpMethod); + this.httpMethod = method; + } + } + + public String getHttpMethod() { + synchronized (monitor) { + return httpMethod; + } + } + + public void setHttpVersion(final String httpVersion) { + synchronized (monitor) { + requireNull(this.httpVersion); + this.httpVersion = httpVersion; + } + } + + public String getHttpVersion() { + synchronized (monitor) { + return httpVersion; + } + } + + public void setPartner(String partner) { + synchronized (monitor) { + requireNull(this.partner); + this.partner = partner; + } + } + + public String getPartner() { + synchronized (monitor) { + return partner; + } + } + + public void setAdRationale(String adRationale) { + synchronized (monitor) { + requireNull(this.adRationale); + this.adRationale = adRationale; + } + } + + public String getAdRationale() { + synchronized (monitor) { + return adRationale; + } + } + + public void setIncrementSlotByOneRequest(String slotName) { + synchronized (monitor) { + requireNull(this.incrementSlotByOneRequest); + this.incrementSlotByOneRequest = slotName; + } + } + + public String getIncrementSlotByOneRequest() { + synchronized (monitor) { + return incrementSlotByOneRequest; + } + } + + public void setZDataIncrementSlotByOneRequest(String slotName) { + synchronized (monitor) { + requireNull(this.zDataIncrementSlotByOneRequest); + this.zDataIncrementSlotByOneRequest = slotName; + } + } + + public String getZDataIncrementSlotByOneRequest() { + synchronized (monitor) { + return zDataIncrementSlotByOneRequest; + } + } + + public void setHostString(String hostString) { + synchronized (monitor) { + requireNull(this.hostString); + this.hostString = hostString; + } + } + + public String getHostString() { + synchronized (monitor) { + return hostString; + } + } + + public void setIpV4Address(String ipV4AddressInDotDecimalNotation) { + synchronized (monitor) { + requireNull(this.ipV4AddressInDotDecimalNotation); + this.ipV4AddressInDotDecimalNotation = ipV4AddressInDotDecimalNotation; + } + } + + public String getIpV4Address() { + synchronized (monitor) { + return ipV4AddressInDotDecimalNotation; + } + } + + public void setTimeStamp(long numMillisSince1Jan1970AtMidnightUTC) { + synchronized (monitor) { + requireZero(this.timeStampMillis); + timeStampMillis = numMillisSince1Jan1970AtMidnightUTC; + } + } + + public long getTimeStampMillis() { + synchronized (monitor) { + return timeStampMillis; + } + } + + public void setDurationBetweenRequestResponse(long timeInMillis) { + synchronized (monitor) { + requireZero(this.durationBetweenRequestResponseMillis); + durationBetweenRequestResponseMillis = timeInMillis; + } + } + + public long getDurationBetweenRequestResponseMillis() { + synchronized (monitor) { + return durationBetweenRequestResponseMillis; + } + } + + public void setReturnedContentSize(int byteCount) { + setReturnedContentSize((long) byteCount); + } + + public void setReturnedContentSize(long byteCount) { + synchronized (monitor) { + requireZero(this.numBytesReturned); + numBytesReturned = byteCount; + } + } + + public long getReturnedContentSize() { + synchronized (monitor) { + return numBytesReturned; + } + } + + public void setURI(final URI uri) { + synchronized (monitor) { + requireNull(this.uri); + this.uri = uri; + } + } + + public URI getURI() { + synchronized (monitor) { + return uri; + } + } + + public void setRemoteAddress(String remoteAddress) { + synchronized (monitor) { + requireNull(this.remoteAddress); + this.remoteAddress = remoteAddress; + } + } + + public void setRemoteAddress(final InetSocketAddress remoteAddress) { + setRemoteAddress(getIpAddressAsString(remoteAddress)); + } + + private static String getIpAddressAsString(final InetSocketAddress remoteAddress) { + final InetAddress inetAddress = remoteAddress.getAddress(); + if (inetAddress == null) { + return null; + } + return inetAddress.getHostAddress(); + } + + public String getRemoteAddress() { + synchronized (monitor) { + return remoteAddress; + } + } + + public void setRemotePort(int remotePort) { + synchronized (monitor) { + requireZero(this.remotePort); + this.remotePort = remotePort; + } + } + + public int getRemotePort() { + synchronized (monitor) { + return remotePort; + } + } + + public void setPeerAddress(final String peerAddress) { + synchronized (monitor) { + requireNull(this.peerAddress); + this.peerAddress = peerAddress; + } + } + + public void setPeerPort(int peerPort) { + synchronized (monitor) { + requireZero(this.peerPort); + this.peerPort = peerPort; + } + } + + public int getPeerPort() { + synchronized (monitor) { + return peerPort; + } + } + + public String getPeerAddress() { + synchronized (monitor) { + return peerAddress; + } + } + + public void setStatusCode(int statusCode) { + synchronized (monitor) { + requireZero(this.statusCode); + this.statusCode = statusCode; + } + } + + public int getStatusCode() { + synchronized (monitor) { + return statusCode; + } + } + + @Override + public String toString() { + synchronized (monitor) { + return new ReflectionToStringBuilder(this) + .setExcludeFieldNames(FIELDS_EXCLUDED_FROM_TOSTRING) + .toString(); + } + } + + private static void requireNull(final Object value) { + if (value != null) { + throw new IllegalStateException("Attempt to overwrite field that has been assigned. Value: " + value); + } + } + + private static void requireZero(final long value) { + if (value != 0) { + throw new IllegalStateException("Attempt to overwrite field that has been assigned. Value: " + value); + } + } + + private static void requireZero(final int value) { + if (value != 0) { + throw new IllegalStateException("Attempt to overwrite field that has been assigned. Value: " + value); + } + } +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java new file mode 100644 index 00000000000..27a5cf9e827 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import com.yahoo.container.core.AccessLogConfig; + +import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme.DATE; + +import java.util.logging.Logger; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +class AccessLogHandler { + + public Logger access = Logger.getAnonymousLogger(); + private LogFileHandler logFileHandler; + + public AccessLogHandler(AccessLogConfig.FileHandler config) { + access.setUseParentHandlers(false); + + logFileHandler = new LogFileHandler(config.rotateScheme()); + + logFileHandler.setFilePattern(config.pattern()); + logFileHandler.setRotationTimes(config.rotation()); + + if (config.rotateScheme() == DATE) + createSymlink(config, logFileHandler); + + + LogFormatter lf = new LogFormatter(); + lf.messageOnly(true); + this.logFileHandler.setFormatter(lf); + access.addHandler(this.logFileHandler); + } + + private void createSymlink(AccessLogConfig.FileHandler config, LogFileHandler handler) { + if (!config.symlink().isEmpty()) + handler.setSymlinkName(config.symlink()); + } + + public void shutdown() { + logFileHandler.close(); + access.removeHandler(logFileHandler); + + if (logFileHandler!=null) + logFileHandler.shutdown(); + } + + void rotateNow() { + logFileHandler.rotateNow(); + } +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogInterface.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogInterface.java new file mode 100644 index 00000000000..674f5883c03 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogInterface.java @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +/** + * @author tonytv + */ +public interface AccessLogInterface { + void log(AccessLogEntry accessLogEntry); +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java new file mode 100644 index 00000000000..987035bb0e9 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Samples entries from access log. It first samples every query until it have some data, and then sub-samples + * much less frequently to reduce CPU usage and latency impact. It only samples successful requests and requests + * that starts with /search. + * + * @author dybdahl + */ +public class AccessLogSampler implements AccessLogInterface { + + private final AtomicLong accessLineCounter = new AtomicLong(0); + private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper; + + public AccessLogSampler(CircularArrayAccessLogKeeper circularArrayAccessLogKeeper) { + this.circularArrayAccessLogKeeper = circularArrayAccessLogKeeper; + } + + @Override + public void log(AccessLogEntry accessLogEntry) { + if (accessLogEntry.getStatusCode() != 200) { + return; + } + String uriString = accessLogEntry.getURI().toString(); + if (! uriString.startsWith("/search")) { + return; + } + final long count = accessLineCounter.incrementAndGet(); + if (count >= CircularArrayAccessLogKeeper.SIZE && count % CircularArrayAccessLogKeeper.SIZE != 0) { + return; + } + circularArrayAccessLogKeeper.addUri(uriString); + } +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java new file mode 100644 index 00000000000..7a60eb2098c --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; + +/** + * This class keeps some information from the access log from the requests in memory. It is thread-safe. + * + * @author dybdahl + */ +public class CircularArrayAccessLogKeeper { + public static final int SIZE = 1000; + private final Deque<String> uris = new ArrayDeque<>(SIZE); + private final Object monitor = new Object(); + + /** + * This class is intended to be used with injection so it can be shared between other classes. + */ + public CircularArrayAccessLogKeeper() {} + + /** + * Creates a list of Uris. + * @return URIs as string + */ + public List<String> getUris() { + final List<String> uriList = new ArrayList<>(); + synchronized (monitor) { + uris.iterator().forEachRemaining(uri -> uriList.add(uri)); + } + return uriList; + } + + /** + * Add a new URI. It might remove an old entry to make space for new entry. + * @param uri uri as string + */ + public void addUri(String uri) { + synchronized (monitor) { + if (uris.size() == SIZE) { + uris.pop(); + } + uris.add(uri); + } + } +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java new file mode 100644 index 00000000000..f403308a404 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +/** + * A wrapper for hit counts, modelled after a search system. + * Advanced database searches and similar could use these + * structures as well. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class HitCounts { + + // see the javadoc for the accessors for short comments on each field + private final int retrievedHits; + private final int summaryCount; + private final long totalHitCount; + private final int requestedHits; + private final int requestedOffset; + + public HitCounts( + int retrievedHits, + int summaryCount, + long totalHitCount, + int requestedHits, + int requestedOffset) { + + this.retrievedHits = retrievedHits; + this.summaryCount = summaryCount; + this.totalHitCount = totalHitCount; + this.requestedHits = requestedHits; + this.requestedOffset = requestedOffset; + } + + /** + * The number of hits returned by the server. + * Compare to getRequestedHits(). + */ + public int getRetrievedHitCount() { + return retrievedHits; + } + + /** + * The number of hit summaries ("document contents") fetched. + */ + public int getSummaryCount() { + return summaryCount; + } + + /** + * The total number of matching hits + * for the request. + */ + public long getTotalHitCount() { + return totalHitCount; + } + + /** + * The number of hits requested by the user. + * Compare to getRetrievedHitCount(). + */ + public int getRequestedHits() { + return requestedHits; + } + + /** + * The user requested offset into the linear mapping of the result space. + */ + public int getRequestedOffset() { + return requestedOffset; + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java new file mode 100644 index 00000000000..5e86c55c294 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -0,0 +1,363 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import com.yahoo.container.core.AccessLogConfig; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.StreamHandler; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * <p>Implements log file naming/rotating logic for container logs.</p> + * + * <p>Overridden methods: publish</p> + * + * <p>Added methods: setFilePattern, setRotationTimes, rotateNow (+ few others)</p> + * + * @author Bob Travis + */ +public class LogFileHandler extends StreamHandler { + + /** True to use the sequence file name scheme, false (default) to use the date scheme */ + private final boolean useSequenceNameScheme; + private long[] rotationTimes = {0}; //default to one log per day, at midnight + private String filePattern = "./log.%T"; // default to current directory, ms time stamp + private long lastRotationTime = -1; // absolute time (millis since epoch) of current file start + private int numberOfRecords = -1; + private long nextRotationTime = 0; + private OutputStream currentOutputStream = null; + private String fileName; + private String symlinkName = null; + private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(1000); + LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow"); + + static private class LogThread extends Thread { + LogFileHandler logFileHandler; + public LogThread(LogFileHandler logFile) { + super("Logger"); + logFileHandler = logFile; + } + @Override + public void run() { + try { + storeLogRecords(); + } catch (InterruptedException e) { + } catch (Exception e) { + com.yahoo.protect.Process.logAndDie("Failed storing log records", e); + } + + logFileHandler.flush(); + } + + private void storeLogRecords() throws InterruptedException { + while (!isInterrupted()) { + LogRecord r = logFileHandler.logQueue.take(); + if (r == logFileHandler.rotateCmd) { + logFileHandler.internalRotateNow(); + } else { + logFileHandler.internalPublish(r); + } + } + } + } + LogThread logThread = null; + + public LogFileHandler() { + this(AccessLogConfig.FileHandler.RotateScheme.Enum.DATE); + } + + public LogFileHandler(AccessLogConfig.FileHandler.RotateScheme.Enum rotateScheme) { + super(); + this.useSequenceNameScheme = rotateScheme == AccessLogConfig.FileHandler.RotateScheme.Enum.SEQUENCE; + init(); + } + + /** + * Constructs a log handler + * + * @param useSequenceNameScheme True to use the sequence file name scheme, false (default) to use the date scheme + */ + public LogFileHandler(OutputStream out, Formatter formatter,boolean useSequenceNameScheme) { + super(out,formatter); + this.useSequenceNameScheme=useSequenceNameScheme; + init(); + } + + private void init() { + logThread = new LogThread(this); + logThread.start(); + } + + /** + * Sends logrecord to file, first rotating file if needed. + * + * @param r logrecord to publish + */ + public void publish(LogRecord r) { + try { + logQueue.put(r); + } catch (InterruptedException e) { + } + } + + private void internalPublish(LogRecord r) throws InterruptedException { + // first check to see if new file needed. + // if so, use this.internalRotateNow() to do it + + long now = System.currentTimeMillis(); + if (nextRotationTime <= 0) { + nextRotationTime = getNextRotationTime(now); // lazy initialization + } + if (now > nextRotationTime || currentOutputStream == null) { + internalRotateNow(); + } + // count records, and publish + numberOfRecords++; + super.publish(r); + flush(); + } + + /** + * Assign pattern for generating (rotating) file names. + * + * @param pattern See LogFormatter for definition + */ + public void setFilePattern ( String pattern ) { + filePattern = pattern; + } + + /** + * Assign times for rotating output files. + * + * @param timesOfDay in millis, from midnight + * + */ + public void setRotationTimes ( long[] timesOfDay ) { + rotationTimes = timesOfDay; + } + + /** Assign time for rotating output files + * + * @param prescription string form of times, in minutes + */ + public void setRotationTimes ( String prescription ) { + setRotationTimes(calcTimesMinutes(prescription)); + } + + /** + * Find next rotation after specified time. + * + * @param now the specified time; if zero, current time is used. + * @return the next rotation time + */ + public long getNextRotationTime (long now) { + if (now <= 0) { + now = System.currentTimeMillis(); + } + long nowTod = timeOfDayMillis(now); + long next = 0; + for (int i = 0; i<rotationTimes.length; i++) { + if (nowTod < rotationTimes[i]) { + next = rotationTimes[i]-nowTod + now; + break; + } + } + if (next == 0) { // didn't find one -- use 1st time 'tomorrow' + next = rotationTimes[0]+lengthOfDayMillis-nowTod + now; + } + + return next; + } + + private void checkAndCreateDir(String pathname) throws IOException { + int lastSlash = pathname.lastIndexOf("/"); + if (lastSlash > -1) { + String pathExcludingFilename = pathname.substring(0, lastSlash); + File filepath = new File(pathExcludingFilename); + if (!filepath.exists()) { + filepath.mkdirs(); + } + } + } + + /** + * Force file rotation now, independent of schedule. + */ + public void rotateNow () { + publish(rotateCmd); + } + + // Throw InterruptedException upwards rather than relying on isInterrupted to stop the thread as + // isInterrupted() returns false after inerruption in p.waitFor + private void internalRotateNow() throws InterruptedException { + // figure out new file name, then + // use super.setOutputStream to switch to a new file + + long now = System.currentTimeMillis(); + fileName = LogFormatter.insertDate(filePattern, now); + super.flush(); + super.close(); + + if (useSequenceNameScheme) + moveCurrentFile(); + + try { + checkAndCreateDir(fileName); + FileOutputStream os = new FileOutputStream(fileName, true); // append mode, for safety + super.setOutputStream(os); + currentOutputStream = os; + } + catch (IOException e) { + throw new RuntimeException("Couldn't open log file '" + fileName + "'", e); + } + + if ( ! useSequenceNameScheme) + createSymlinkToCurrentFile(); + + numberOfRecords = 0; + lastRotationTime = now; + nextRotationTime = 0; //figure it out later (lazy evaluation) + } + + /** Name files by date - create a symlink with a constant name to the newest file */ + private void createSymlinkToCurrentFile() throws InterruptedException { + if (symlinkName == null) return; + File f = new File(fileName); + File f2 = new File(f.getParent(), symlinkName); + try { + Runtime r = Runtime.getRuntime(); + Process p = r.exec(new String[] { "/bin/ln", "-sf", f.getCanonicalPath(), f2.getPath() }); + // Detonator pattern: Think of all the fun we can have if ln isn't what we + // think it is, if it doesn't return, etc, etc + p.waitFor(); + } catch (IOException e) { + // little we can do... + } + } + + /** + * Name the current file to "name.n" where n + * 1+ the largest integer in existing file names + */ + private void moveCurrentFile() { + File file=new File(fileName); + if ( ! file.exists()) return; // no current file + File dir=file.getParentFile(); + Pattern logFilePattern=Pattern.compile(".*\\.(\\d+)"); + long largestN=0; + for (File existingFile : dir.listFiles()) { + Matcher matcher=logFilePattern.matcher(existingFile.getName()); + if (!matcher.matches()) continue; + long thisN=Long.parseLong(matcher.group(1)); + if (thisN>largestN) + largestN=thisN; + } + file.renameTo(new File(dir,file.getName() + "." + (largestN + 1))); + } + + /** + * @return last time file rotation occurred for this output file + */ + public long getLastRotationTime () { + return lastRotationTime; + } + + /** + * @return number of records written to this file since last rotation + */ + public long getNumberRecords () { + return numberOfRecords; + } + + /** + * Calculate rotation times array, given times in minutes, as "0 60 ..." + * + */ + public static long[] calcTimesMinutes(String times) { + ArrayList<Long> list = new ArrayList<>(50); + int i = 0; + boolean etc = false; + + while (i < times.length()) { + if (times.charAt(i) == ' ') { i++; continue; } // skip spaces + int j = i; // start of string + i = times.indexOf(' ', i); + if (i == -1) i = times.length(); + if (times.charAt(j) == '.' && times.substring(j,i).equals("...")) { // ... + etc = true; + break; + } + list.add(new Long(times.substring(j,i))); + } + + int size = list.size(); + long[] longtimes = new long[size]; + for (i = 0; i<size; i++) { + longtimes[i] = list.get(i).longValue() // pick up value in minutes past midnight + * 60000; // and multiply to get millis + } + + if (etc) { // fill out rest of day, same as final interval + long endOfDay = 24*60*60*1000; + long lasttime = longtimes[size-1]; + long interval = lasttime - longtimes[size-2]; + long moreneeded = (endOfDay - lasttime)/interval; + if (moreneeded > 0) { + int newsize = size + (int)moreneeded; + long[] temp = new long[newsize]; + for (i=0; i<size; i++) { + temp[i] = longtimes[i]; + } + while (size < newsize) { + lasttime += interval; + temp[size++] = lasttime; + } + longtimes = temp; + } + } + + return longtimes; + } + + // Support staff :-) + private static final long lengthOfDayMillis = 24*60*60*1000; // ? is this close enough ? + + private static final long timeOfDayMillis ( long time ) { + return time % lengthOfDayMillis; + } + + public void setSymlinkName(String symlinkName) { + this.symlinkName = symlinkName; + } + + /** + * Flushes all queued messages, interrupts the log thread in this and + * waits for it to end before returning + */ + public void shutdown() { + logThread.interrupt(); + try { + logThread.join(); + } + catch (InterruptedException e) { + } + } + + /** + * Only for unit testing. Do not use. + */ + public String getFileName() { + return fileName; + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java new file mode 100644 index 00000000000..961bfc32cc1 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java @@ -0,0 +1,190 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + + +/** + * Produces compact output format for prelude logs + * + * @author Bob Travis + */ +public class LogFormatter extends Formatter { + + /** date format objects */ + static SimpleDateFormat ddMMMyyyy; + static DateFormat dfMMM; + static SimpleDateFormat yyyyMMdd; + + static { + ddMMMyyyy = new SimpleDateFormat("[dd/MMM/yyyy:HH:mm:ss Z]"); + ddMMMyyyy.setTimeZone(TimeZone.getTimeZone("UTC")); + + dfMMM = new SimpleDateFormat("MMM"); + dfMMM.setTimeZone(TimeZone.getTimeZone("UTC")); + + yyyyMMdd = new SimpleDateFormat("[yyyy-MM-dd HH:mm:ss]"); + yyyyMMdd.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + /** Whether to strip down the message to only the message or not */ + private boolean messageOnly = false; + + /** Controls which of the available timestamp formats is used in all log records + */ + private static final int timestampFormat = 2; // 0=millis, 1=mm/dd/yyyy, 2=yyyy-mm-dd + + /** + * Standard constructor + */ + + public LogFormatter() {} + + /** + * Make it possible to log stripped messages + */ + public void messageOnly (boolean messageOnly) { + this.messageOnly = messageOnly; + } + + public String format(LogRecord record) { + + // if we don't want any other stuff we just return the message + if (messageOnly) { + return formatMessage(record); + } + + String rawMsg = record.getMessage(); + boolean isLogMsg = + rawMsg.charAt(0) == 'L' + && rawMsg.charAt(1) == 'O' + && rawMsg.charAt(2) == 'G' + && rawMsg.charAt(3) == ':'; + String nameInsert = + (!isLogMsg) + ? record.getLevel().getName() + ": " + : ""; + return (timeStamp(record) + + nameInsert + + formatMessage(record) + + "\n" + ); + } + + /** + * Public support methods + */ + + /** + * Static insertDate method will insert date fragments into a string + * based on '%x' pattern elements. Equivalents in SimpleDateFormatter patterns, + * with examples: + * <ul> + * <li>%Y YYYY 2003 + * <li>%m MM 08 + * <li>%x MMM Aug + * <li>%d dd 25 + * <li>%H HH 14 + * <li>%M mm 30 + * <li>%S ss 35 + * <li>%s SSS 123 + * <li>%Z Z -0400 + * </ul> + *Others: + * <ul> + * <li>%T Long.toString(time) + * <li>%% % + * </ul> + */ + public static final String insertDate (String pattern, long time) { + DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS Z"); + df.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = new Date(time); + String datetime = df.format(date); + StringBuilder result = new StringBuilder(); + int i=0; + while (i < pattern.length()) { + int j = pattern.indexOf('%',i); + if (j == -1 || j >= pattern.length()-1) { // done + result.append(pattern.substring(i)); // copy rest of pattern and quit + break; + } + result.append(pattern.substring(i, j)); + switch (pattern.charAt(j+1)) { + case 'Y': + result.append(datetime.substring(0,4)); // year + break; + case 'm': + result.append(datetime.substring(5,7)); // month + break; + case 'd': + result.append(datetime.substring(8,10)); // day of month + break; + case 'H': + result.append(datetime.substring(11,13)); // hour + break; + case 'M': + result.append(datetime.substring(14,16)); // minute + break; + case 'S': + result.append(datetime.substring(17,19)); // second + break; + case 's': + result.append(datetime.substring(20,23)); // thousanths + break; + case 'Z': + result.append(datetime.substring(24)); // time zone string + break; + case 'T': + result.append(Long.toString(time)); //time in Millis + break; + case 'x': + result.append(capitalize(dfMMM.format(date))); + break; + case '%': + result.append("%%"); + break; + default: + result.append("%"); // copy pattern escape and move on + j--; // only want to bump by one position.... + break; + } + i = j+2; + } + + return result.toString(); + } + + /** + * Private methods: timeStamp(LogRecord) + */ + private String timeStamp(LogRecord record) { + Date date = new Date(record.getMillis()); + String stamp; + switch (timestampFormat) { + case 0: + stamp = Long.toString(record.getMillis()); + break; + case 1: + stamp = ddMMMyyyy.format(date); + break; + case 2: + default: + stamp = yyyyMMdd.format(date); + break; + } + return stamp; + } + + /** Return the given string with the first letter in upper case */ + private static String capitalize(String string) { + if (Character.isUpperCase(string.charAt(0))) return string; + return Character.toUpperCase(string.charAt(0)) + string.substring(1); + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/VespaAccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/VespaAccessLog.java new file mode 100644 index 00000000000..566cf17f89f --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/VespaAccessLog.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.net.UriTools; + +import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; +import java.util.logging.Level; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a> + */ +public final class VespaAccessLog implements AccessLogInterface { + + private static final SimpleDateFormat dateFormat = createDateFormat(); + + private final AccessLogHandler logHandler; + + public VespaAccessLog(AccessLogConfig config) { + logHandler = new AccessLogHandler(config.fileHandler()); + } + + private static SimpleDateFormat createDateFormat() { + SimpleDateFormat format = new SimpleDateFormat("[dd/MMM/yyyy:HH:mm:ss Z]"); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + return format; + } + + private String getDate () { + Date date = new Date(); + return dateFormat.format(date); + } + + private String getRequest(final String httpMethod, final URI uri, final String httpVersion) { + final URI normalizedUri = uri.normalize(); + return httpMethod + " " + UriTools.rawRequest(normalizedUri) + " " + httpVersion; + } + + private String getUser(String user) { + return (user == null) ? "-" : user; + } + + private void writeLog(String ipAddr, String user, String request, String referer, String agent, long startTime, + long durationMillis, long byteCount, HitCounts hitcounts, int returnCode) + { + long ms = Math.max(0L, durationMillis); + StringBuilder sb = new StringBuilder() + .append(ipAddr) + .append(" - ") + .append(getUser(user)) + .append(' ') + .append(getDate()) + .append(" \"") + .append(request) + .append("\" ") + .append(returnCode) + .append(' ') + .append(byteCount) + .append(" \"") + .append(referer) + .append("\" \"") + .append(agent) + .append("\" ") + .append(ms/1000) + .append('.'); + decimalsOfSecondsFromMilliseconds(ms, sb); + sb.append(' ') + .append((hitcounts == null) ? 0 : hitcounts.getTotalHitCount()) + .append(" 0.0 ") + .append((hitcounts == null) ? 0 : hitcounts.getSummaryCount()) + .append('\n'); + logHandler.access.log(Level.INFO, sb.toString()); + } + + private void decimalsOfSecondsFromMilliseconds(long ms, StringBuilder sb) { + long dec = ms % 1000; + String numbers = String.valueOf(dec); + if (dec <= 9) { + sb.append("00"); + } else if (dec <= 99) { + sb.append('0'); + } + sb.append(numbers); + } + + /** + * TODO: This is never called. We should have a DI provider and call this method from its deconstruct. + */ + public void shutdown() { + if (logHandler!=null) + logHandler.shutdown(); + } + + @Override + public void log(final AccessLogEntry accessLogEntry) { + writeLog( + accessLogEntry.getIpV4Address(), + accessLogEntry.getUser(), + getRequest( + accessLogEntry.getHttpMethod(), + accessLogEntry.getURI(), + accessLogEntry.getHttpVersion()), + accessLogEntry.getReferer(), + accessLogEntry.getUserAgent(), + accessLogEntry.getTimeStampMillis(), + accessLogEntry.getDurationBetweenRequestResponseMillis(), + accessLogEntry.getReturnedContentSize(), + accessLogEntry.getHitCounts(), + accessLogEntry.getStatusCode()); + } +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java new file mode 100644 index 00000000000..127368049c4 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import com.yahoo.container.core.AccessLogConfig; + +import java.util.logging.Level; + +/** + * Log a message in yApache log format. + * + * @author tinyv + */ +public final class YApacheAccessLog implements AccessLogInterface { + + private final AccessLogHandler logHandler; + + public YApacheAccessLog(AccessLogConfig config) { + logHandler = new AccessLogHandler(config.fileHandler()); + } + + @Override + public void log(final AccessLogEntry logEntry) { + logHandler.access.log(Level.INFO, new YApacheFormatter(logEntry).format() + '\n'); + } + + + // TODO: This is never called. We should have a DI provider and call this method from its deconstruct. + public void shutdown() { + logHandler.shutdown(); + } + + void rotateNow() { + logHandler.rotateNow(); + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java new file mode 100644 index 00000000000..ec19e909614 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java @@ -0,0 +1,470 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; + +import java.net.URI; +import java.util.Formatter; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.container.logging.AccessLogEntry.CookieType; + +/** + * Formatting of an {@link AccessLogEntry} in the yapache access log format. + * + * @author tonytv + * @author bakksjo + */ +public class YApacheFormatter { + + private AccessLogEntry accessLogEntry; + + public YApacheFormatter(final AccessLogEntry entry) { + accessLogEntry = entry; + } + + public String format() { + // Initial 32-byte fixed block of mandatory, non-prefixed fields. + setIpV4Address(accessLogEntry.getIpV4Address()); + setTimeStampMillis(accessLogEntry.getTimeStampMillis()); + setDurationBetweenRequestResponseMillis(accessLogEntry.getDurationBetweenRequestResponseMillis()); + setReturnedContentSize(accessLogEntry.getReturnedContentSize()); + + // Optional, prefixed fields in arbitrary order. + setStatusCode(accessLogEntry.getStatusCode()); + setRemoteAddress(accessLogEntry.getRemoteAddress()); + setRemotePort(accessLogEntry.getRemotePort()); + setURI(accessLogEntry.getURI()); + setCookie(accessLogEntry.getCookieType(), accessLogEntry.getCookie()); + setWeekOfRegistration(accessLogEntry.getWeekOfRegistration()); + setProfile(accessLogEntry.getProfile()); + setInternationalInfo(accessLogEntry.getInternationalInfo()); + setContentAttribute(accessLogEntry.getContentAttribute()); + setAdSpaceID(accessLogEntry.getAdSpaceID()); + setErrorMessage(accessLogEntry.getErrorMessage()); + setFileName(accessLogEntry.getFileName()); + setUserAgent(accessLogEntry.getUserAgent()); + setWebfactsDigitalSignature(accessLogEntry.getWebfactsDigitalSignature()); + setReferer(accessLogEntry.getReferer()); + setRequestExtra(accessLogEntry.getRequestExtra()); + setResponseExtra(accessLogEntry.getResponseExtra()); + setResultFromCache(accessLogEntry.getResultFromCache()); + setHttpMethod(accessLogEntry.getHttpMethod()); + setPartner(accessLogEntry.getPartner()); + setAdRationale(accessLogEntry.getAdRationale()); + setIncrementSlotByOneRequest(accessLogEntry.getIncrementSlotByOneRequest()); + setZDataIncrementSlotByOneRequest(accessLogEntry.getZDataIncrementSlotByOneRequest()); + setHostString(accessLogEntry.getHostString()); + setPeerAddress(accessLogEntry.getPeerAddress()); + setPeerPort(accessLogEntry.getPeerPort()); + adInfos = accessLogEntry.getAdInfos(); + keyValues = accessLogEntry.getKeyValues(); + + return toYApacheAccessEntry(); + } + + private static final Map<CookieType, Character> cookieTypeFirstCharMap = new HashMap<>(); + static { + cookieTypeFirstCharMap.put(CookieType.b, '1'); + cookieTypeFirstCharMap.put(CookieType.l, '2'); + cookieTypeFirstCharMap.put(CookieType.n, '3'); + cookieTypeFirstCharMap.put(CookieType.geocookie,'5'); + cookieTypeFirstCharMap.put(CookieType.I, '7'); + cookieTypeFirstCharMap.put(CookieType.R, '9'); + cookieTypeFirstCharMap.put(CookieType.Y, 'c'); + cookieTypeFirstCharMap.put(CookieType.M, 'M'); + } + + private List<AccessLogEntry.AdInfo> adInfos; + private String spaceID; + + private String ipV4AddressInDotDecimalNotation; + private String unixTimeStamp; + private String durationBetweenRequestResponseInMS; + private String numBytesReturned; + private String uri; + + + private String remoteAddress; + private int remotePort; + private String peerAddress; + private int peerPort; + + private Map<String,List<String>> keyValues; + + private static Logger logger = Logger.getLogger(YApacheFormatter.class.getName()); + + private void setCookie(CookieType type, String cookie) { + final Character firstChar = cookieTypeFirstCharMap.get(type); + if (firstChar == null) { + return; + } + addField(firstChar, cookie); + } + + private void setWeekOfRegistration( String weekOfRegistration ) { + addField('4', weekOfRegistration); + } + + private void setProfile( String profile ) { + addField('6', profile); + } + + private void setInternationalInfo( String intl ) { + addField('8', intl); + } + + private void setContentAttribute( String contentAttribute ) { + addField('a', contentAttribute); + } + + private void setAdSpaceID(String spaceID) { + this.spaceID = spaceID; + } + + private static class AdInfo { + + private void setAdID(String id) { + add('B', id); + } + + private void setMatchID(String id) { + add('C', id); + } + + private void setPosition(String position) { + add('D', position); + } + + private void setProperty(String property) { + add('F', property); + } + + private void setCPC(String cpc) { + add('G', cpc); + } + + private void setAdClientVersion(String adClientVersion) { + add('K', adClientVersion); + } + + private void setLinkID(String id) { + add('L', id); + } + + private void setBidPosition(String bidPosition) { + add('P', bidPosition); + } + + private StringBuilder adInfo = new StringBuilder(); + + AdInfo(final AccessLogEntry.AdInfo model) { + final String modelAdServerString = model.getAdServerString(); + if (modelAdServerString != null) { + adInfo.append(modelAdServerString); + } + setAdID(model.getAdID()); + setMatchID(model.getMatchID()); + setPosition(model.getPosition()); + setProperty(model.getProperty()); + setCPC(model.getCPC()); + setAdClientVersion(model.getAdClientVersion()); + setLinkID(model.getLinkID()); + setBidPosition(model.getBidPosition()); + } + + String adInfoString() { + return adInfo.toString(); + } + + private void add(char controlChar, String value) { + if (value == null) { + return; + } + adInfo.append( controlCharacter(controlChar) ); + adInfo.append( value ); + } + } + + private void setWebfactsDigitalSignature(String signature) { + addField('d', signature); + } + + private void setErrorMessage(String errorMessage) { + addField('e', errorMessage); + } + + private void setFileName(String fileName) { + addField('f', fileName); + } + + private void setUserAgent(String userAgent) { + addField('g', userAgent); + } + + private void setReferer(String referer) { + addField('r', referer); + } + + private void setRequestExtra(String requestExtra) { + if (requestExtra == null || requestExtra.isEmpty()) { + return; + } + if (fieldPrefix != requestExtra.charAt(0)) { + yApacheEntry.append(fieldPrefix); + } + yApacheEntry.append(requestExtra); + } + + private void setResponseExtra(String responseExtra) { + if (responseExtra == null || responseExtra.isEmpty()) { + return; + } + if (fieldPrefix != responseExtra.charAt(0)) { + yApacheEntry.append(fieldPrefix); + } + yApacheEntry.append(responseExtra); + } + + private void setResultFromCache(Boolean fromCache) { + if (fromCache == null) { + return; + } + addField('h', + fromCache ? "1" : "0"); + } + + private void setHttpMethod(String method) { + addField('m', method); + } + + private void setPartner(String partner) { + addField('p', partner); + } + + private void setAdRationale(String adRationale) { + addField('R', adRationale); + } + + private void setIncrementSlotByOneRequest(String slotName) { + addField('t', slotName); + } + + private void setZDataIncrementSlotByOneRequest(String slotName) { + addField('z', slotName); + } + + private void setHostString(String hostString) { + addField('w', hostString); + } + + private StringBuilder yApacheEntry = new StringBuilder(); + + private static char fieldPrefix = controlCharacter('E'); + private static char valueSeparator = controlCharacter('A'); + + + //assumes A <= c <= Z + private static char controlCharacter(char c) { + return (char)((c - 'A') + 1); + } + + private void appendStartOfField(StringBuilder builder, char firstChar) { + builder.append(fieldPrefix); + builder.append(firstChar); + } + + private void addField(char firstChar, String field) { + if (field == null) { + return; + } + appendStartOfField(yApacheEntry, firstChar); + yApacheEntry.append(field); + } + + private void addDecimalField(char firstChar, int field) { + addField(firstChar, Integer.toString(field)); + } + + private String to8ByteHexString(final long value) { + Formatter formatter = new Formatter(); + formatter.format("%08x", value); + return formatter.toString(); + } + + private void appendAdInfo(StringBuilder buf) { + if (spaceID != null || ! adInfos.isEmpty()) { + buf.append(fieldPrefix); + buf.append('b'); + + if (spaceID != null) { + buf.append( controlCharacter('A') ); + buf.append( spaceID ); + } + + for (AccessLogEntry.AdInfo adInfo : adInfos ) { + buf.append( new AdInfo(adInfo).adInfoString() ); + } + } + } + + private void appendFirst32Bytes(StringBuilder buf) { + buf.append(ipv4AddressToHexString(ipV4AddressInDotDecimalNotation)); + buf.append(unixTimeStamp); + buf.append(durationBetweenRequestResponseInMS); + buf.append(numBytesReturned); + + assert(buf.length() == 32); + } + + private String toYApacheAccessEntry() { + StringBuilder b = new StringBuilder(); + + appendFirst32Bytes(b); + b.append(uri); + + b.append(yApacheEntry); + appendIPv6AddressInfo(b); + appendAdInfo(b); + + appendKeyValues(b); + return b.toString(); + } + + private void appendIPv6AddressInfo(StringBuilder builder) { + appendStartOfField(builder, 'A'); + final boolean remoteAddressesAreEqual = Objects.equals(ipV4AddressInDotDecimalNotation, remoteAddress); + if (!remoteAddressesAreEqual && remoteAddress != null) { + builder.append('A').append(remoteAddress).append(valueSeparator); + } + + builder.append('B').append(remotePort); + + if (peerAddress != null) { + builder.append(valueSeparator).append('C').append(peerAddress); + } + + if (peerPort > 0 && peerPort != remotePort) { + builder.append(valueSeparator).append('D').append(peerPort); + } + } + + /** + * Encodes key-values added to this entry at the "property extension" key 'X' as + * <code>^EY[key1]^F[value1.1]^B[value1.2]^A[key2]^F[value2]</code>, + */ + private void appendKeyValues(StringBuilder b) { + if (keyValues==null) return; + b.append(fieldPrefix); + b.append('X'); + for (Map.Entry<String,List<String>> entry : keyValues.entrySet()) { + b.append(entry.getKey()); + b.append(controlCharacter('F')); + for (Iterator<String> i=entry.getValue().iterator(); i.hasNext(); ) { + b.append(i.next()); + if (i.hasNext()) + b.append(controlCharacter('B')); + } + b.append(controlCharacter('A')); + } + b.deleteCharAt(b.length()-1); // Deletes the last ^A to be able to do foreach looping :-) + } + + private String ipv4AddressToHexString(String ipv4AddressInDotDecimalNotation) { + try { + String[] parts = ipv4AddressInDotDecimalNotation.split("\\."); + if (parts.length != 4) { + throw new Exception(); + } else { + Formatter byteHexFormatter = new Formatter(); + + for (String part : parts) { + int i = Integer.parseInt(part); + if ( i > 0xff || i < 0) + throw new Exception(); + byteHexFormatter.format("%02x", i); + } + + return byteHexFormatter.toString(); + } + } catch( Exception e ) { + logger.log(Level.WARNING, "IPv4 address not in dot decimal notation: " + + ipv4AddressInDotDecimalNotation); + return "00000000"; + } + } + + private void setIpV4Address(String ipV4AddressInDotDecimalNotation) { + this.ipV4AddressInDotDecimalNotation = ipV4AddressInDotDecimalNotation; + } + + private void setTimeStampMillis(long numMillisSince1Jan1970AtMidnightUTC) { + int unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000); + + if (numMillisSince1Jan1970AtMidnightUTC/1000 > 0x7fffffff) { + logger.log(Level.WARNING, "A year 2038 problem occurred."); + logger.log(Level.INFO, "numMillisSince1Jan1970AtMidnightUTC: " + + numMillisSince1Jan1970AtMidnightUTC); + unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000 % 0x7fffffff); + } + + unixTimeStamp = to8ByteHexString(unixTime); + } + + private void setDurationBetweenRequestResponseMillis(long timeInMillis) { + long timeInMicroSeconds = timeInMillis*1000; + if (timeInMicroSeconds > 0xffffffffL) { + logger.log(Level.WARNING, "Duration too long: " + timeInMillis); + timeInMicroSeconds = 0xffffffffL; + } + + durationBetweenRequestResponseInMS = to8ByteHexString(timeInMicroSeconds); + } + + private void setReturnedContentSize(long byteCount) { + numBytesReturned = to8ByteHexString(byteCount); + } + + private void setURI(final URI uri) { + setNormalizedURI(uri.normalize()); + } + + private void setNormalizedURI(final URI normalizedUri) { + String uriString = normalizedUri.getPath(); + if (normalizedUri.getRawQuery() != null) { + uriString = uriString + "?" + normalizedUri.getRawQuery(); + } + + this.uri = uriString; + } + + private void setRemoteAddress(String remoteAddress) { + this.remoteAddress = remoteAddress; + } + + private void setRemotePort(int remotePort) { + this.remotePort = remotePort; + } + + private void setPeerAddress(final String peerAddress) { + this.peerAddress = peerAddress; + } + + private void setPeerPort(int peerPort) { + this.peerPort = peerPort; + } + + /** Sets the status code, which will end up in the "s" field. If this is 200 the field is not written (by spec). */ + private void setStatusCode(int statusCode) { + if (statusCode == 0) { + return; + } + if (statusCode!=200) + addDecimalField('s', statusCode); + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/package-info.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/package-info.java new file mode 100644 index 00000000000..e2c1f7c0bb7 --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.logging; + +import com.yahoo.osgi.annotation.ExportPackage; |