summaryrefslogtreecommitdiffstats
path: root/container-accesslogging
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2019-03-11 15:28:10 +0100
committerHenning Baldersheim <balder@yahoo-inc.com>2019-03-11 15:28:10 +0100
commit79d1a0a92f4548204ade47159204bf1b3c90d2fa (patch)
tree8b7718d7cc391cbf5d2ba451198d8840180a5964 /container-accesslogging
parent48ce50681ad29a1a17446dbb1f0413615ca35725 (diff)
GC unused yapache log and derivatives.
Diffstat (limited to 'container-accesslogging')
-rw-r--r--container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java303
-rw-r--r--container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java36
-rw-r--r--container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java470
-rw-r--r--container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java289
4 files changed, 1 insertions, 1097 deletions
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
index 46e5aa95dab..72eaf129453 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
@@ -7,7 +7,6 @@ import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import javax.security.auth.x500.X500Principal;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,23 +28,13 @@ import static java.util.stream.Collectors.toMap;
* but the parameters should be different for each
* invocation of the same method.</p>
*
- * This class is thread-safe, but the inner class {@link AdInfo} is not.
+ * This class is thread-safe.
*
* @author Tony Vaagenes
* @author bakksjo
* @author bjorncs
*/
public class AccessLogEntry {
- public enum CookieType {
- b,
- l,
- n,
- geocookie,
- I,
- R,
- Y,
- M;
- }
// Sadly, there's no way to do compile-time validation of these field references.
private static final String[] FIELDS_EXCLUDED_FROM_TOSTRING = new String[] {
@@ -54,42 +43,25 @@ public class AccessLogEntry {
private final Object monitor = new Object();
- private List<AdInfo> adInfos;
- private String spaceID;
-
private String ipV4AddressInDotDecimalNotation;
private long timeStampMillis;
private long durationBetweenRequestResponseMillis;
private long numBytesReturned;
-
private String remoteAddress;
private int remotePort;
private String peerAddress;
private int peerPort;
- private CookieType cookieType;
- private String cookie;
- private String weekOfRegistration;
private String profile;
- private String internationalInfo;
- private String contentAttribute;
- private String webfactsDigitalSignature;
private String errorMessage;
private String fileName;
private String userAgent;
private String referer;
private String user;
private HitCounts hitCounts;
- private String requestExtra;
- private String responseExtra;
- private Boolean resultFromCache;
private String httpMethod;
private String httpVersion;
- private String partner;
- private String adRationale;
- private String incrementSlotByOneRequest;
- private String zDataIncrementSlotByOneRequest;
private String hostString;
private int statusCode;
private String scheme;
@@ -101,40 +73,6 @@ public class AccessLogEntry {
private ListMap<String,String> keyValues=null;
- public void setCookie( CookieType type, String cookie) {
- synchronized (monitor) {
- requireNull(this.cookieType);
- requireNull(this.cookie);
- this.cookieType = type;
- this.cookie = cookie;
- }
- }
-
- public CookieType getCookieType() {
- synchronized (monitor) {
- return cookieType;
- }
- }
-
- public String getCookie() {
- synchronized (monitor) {
- return cookie;
- }
- }
-
- public void setWeekOfRegistration( String weekOfRegistration ) {
- synchronized (monitor) {
- requireNull(this.weekOfRegistration);
- this.weekOfRegistration = weekOfRegistration;
- }
- }
-
- public String getWeekOfRegistration() {
- synchronized (monitor) {
- return weekOfRegistration;
- }
- }
-
public void setProfile( String profile ) {
synchronized (monitor) {
requireNull(this.profile);
@@ -148,168 +86,6 @@ public class AccessLogEntry {
}
}
- public void setInternationalInfo( String intl ) {
- synchronized (monitor) {
- requireNull(this.internationalInfo);
- this.internationalInfo = intl;
- }
- }
-
- public String getInternationalInfo() {
- synchronized (monitor) {
- return internationalInfo;
- }
- }
-
- public void setContentAttribute( String contentAttribute ) {
- synchronized (monitor) {
- requireNull(this.contentAttribute);
- this.contentAttribute = contentAttribute;
- }
- }
-
- public String getContentAttribute() {
- synchronized (monitor) {
- return contentAttribute;
- }
- }
-
- public void setAdSpaceID(String spaceID) {
- synchronized (monitor) {
- requireNull(this.spaceID);
- this.spaceID = spaceID;
- }
- }
-
- public String getAdSpaceID() {
- synchronized (monitor) {
- return spaceID;
- }
- }
-
- public void addAdInfo(AdInfo adInfo) {
- synchronized (monitor) {
- if (adInfos == null) {
- adInfos = new ArrayList<>();
- }
- adInfos.add( adInfo );
- }
- }
-
- public List<AdInfo> getAdInfos() {
- synchronized (monitor) {
- if (adInfos == null) {
- return Collections.emptyList();
- }
- // TODO: The returned list is unmodifiable, but its elements are not. But we're all friendly here, right?
- return Collections.unmodifiableList(adInfos);
- }
- }
-
- /**
- * This class is NOT thread-safe. It is assumed that a single instance is created/written by a single thread,
- * and all reads happen-after creation/population, i.e. no mutation after the instance is shared between threads.
- */
- public static class AdInfo {
-
- private String adServerString;
- private String adId;
- private String matchId;
- private String position;
- private String property;
- private String cpc;
- private String adClientVersion;
- private String linkId;
- private String bidPosition;
-
- public void setAdID(String id) {
- this.adId = id;
- }
-
- public String getAdID() {
- return adId;
- }
-
- public void setMatchID(String id) {
- this.matchId = id;
- }
-
- public String getMatchID() {
- return matchId;
- }
-
- public void setPosition(String position) {
- this.position = position;
- }
-
- public String getPosition() {
- return position;
- }
-
- public void setProperty(String property) {
- this.property = property;
- }
-
- public String getProperty() {
- return property;
- }
-
- public void setCPC(String cpc) {
- this.cpc = cpc;
- }
-
- public String getCPC() {
- return cpc;
- }
-
- public void setAdClientVersion(String adClientVersion) {
- this.adClientVersion = adClientVersion;
- }
-
- public String getAdClientVersion() {
- return adClientVersion;
- }
-
- public void setLinkID(String id) {
- this.linkId = id;
- }
-
- public String getLinkID() {
- return linkId;
- }
-
- public void setBidPosition(String bidPosition) {
- this.bidPosition = bidPosition;
- }
-
- public String getBidPosition() {
- return bidPosition;
- }
-
- public AdInfo() {}
-
- AdInfo(String adServerString) {
- this.adServerString = adServerString;
- }
-
- String getAdServerString() {
- return adServerString;
- }
- }
-
- public void setWebfactsDigitalSignature(String signature) {
- synchronized (monitor) {
- requireNull(this.webfactsDigitalSignature);
- this.webfactsDigitalSignature = signature;
- }
- }
-
- public String getWebfactsDigitalSignature() {
- synchronized (monitor) {
- return webfactsDigitalSignature;
- }
- }
-
public void setErrorMessage(String errorMessage) {
synchronized (monitor) {
requireNull(this.errorMessage);
@@ -388,18 +164,6 @@ public class AccessLogEntry {
}
}
- public String getRequestExtra() {
- synchronized (monitor) {
- return requestExtra;
- }
- }
-
- public String getResponseExtra() {
- synchronized (monitor) {
- return responseExtra;
- }
- }
-
public void addKeyValue(String key,String value) {
synchronized (monitor) {
if (keyValues == null) {
@@ -431,19 +195,6 @@ public class AccessLogEntry {
entry -> valueConverter.apply(entry.getValue())));
}
- public void setResultFromCache(boolean fromCache) {
- synchronized (monitor) {
- requireNull(this.resultFromCache);
- this.resultFromCache = fromCache;
- }
- }
-
- public Boolean getResultFromCache() {
- synchronized (monitor) {
- return resultFromCache;
- }
- }
-
public enum HttpMethod {
GET, POST;
}
@@ -478,58 +229,6 @@ public class AccessLogEntry {
}
}
- public void setPartner(String partner) {
- synchronized (monitor) {
- requireNull(this.partner);
- this.partner = partner;
- }
- }
-
- public String getPartner() {
- synchronized (monitor) {
- return partner;
- }
- }
-
- public void setAdRationale(String adRationale) {
- synchronized (monitor) {
- requireNull(this.adRationale);
- this.adRationale = adRationale;
- }
- }
-
- public String getAdRationale() {
- synchronized (monitor) {
- return adRationale;
- }
- }
-
- public void setIncrementSlotByOneRequest(String slotName) {
- synchronized (monitor) {
- requireNull(this.incrementSlotByOneRequest);
- this.incrementSlotByOneRequest = slotName;
- }
- }
-
- public String getIncrementSlotByOneRequest() {
- synchronized (monitor) {
- return incrementSlotByOneRequest;
- }
- }
-
- public void setZDataIncrementSlotByOneRequest(String slotName) {
- synchronized (monitor) {
- requireNull(this.zDataIncrementSlotByOneRequest);
- this.zDataIncrementSlotByOneRequest = slotName;
- }
- }
-
- public String getZDataIncrementSlotByOneRequest() {
- synchronized (monitor) {
- return zDataIncrementSlotByOneRequest;
- }
- }
-
public void setHostString(String hostString) {
synchronized (monitor) {
requireNull(this.hostString);
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java
deleted file mode 100644
index 6fb0cd8161a..00000000000
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.logging;
-
-import com.yahoo.container.core.AccessLogConfig;
-
-import java.util.logging.Level;
-
-/**
- * Log a message in yApache log format.
- *
- * @author tinyv
- */
-public final class YApacheAccessLog implements AccessLogInterface {
-
- private final AccessLogHandler logHandler;
-
- public YApacheAccessLog(AccessLogConfig config) {
- logHandler = new AccessLogHandler(config.fileHandler());
- }
-
- @Override
- public void log(final AccessLogEntry logEntry) {
- logHandler.access.log(Level.INFO, new YApacheFormatter(logEntry).format() + '\n');
- }
-
-
- // TODO: This is never called. We should have a DI provider and call this method from its deconstruct.
- public void shutdown() {
- logHandler.shutdown();
- }
-
- void rotateNow() {
- logHandler.rotateNow();
- }
-
-}
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java
deleted file mode 100644
index f63d02b1a77..00000000000
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java
+++ /dev/null
@@ -1,470 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.logging;
-
-import java.net.URI;
-import java.util.Formatter;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static com.yahoo.container.logging.AccessLogEntry.CookieType;
-
-/**
- * Formatting of an {@link AccessLogEntry} in the yapache access log format.
- *
- * @author Tony Vaagenes
- * @author bakksjo
- */
-public class YApacheFormatter {
-
- private AccessLogEntry accessLogEntry;
-
- public YApacheFormatter(final AccessLogEntry entry) {
- accessLogEntry = entry;
- }
-
- public String format() {
- // Initial 32-byte fixed block of mandatory, non-prefixed fields.
- setIpV4Address(accessLogEntry.getIpV4Address());
- setTimeStampMillis(accessLogEntry.getTimeStampMillis());
- setDurationBetweenRequestResponseMillis(accessLogEntry.getDurationBetweenRequestResponseMillis());
- setReturnedContentSize(accessLogEntry.getReturnedContentSize());
-
- // Optional, prefixed fields in arbitrary order.
- setStatusCode(accessLogEntry.getStatusCode());
- setRemoteAddress(accessLogEntry.getRemoteAddress());
- setRemotePort(accessLogEntry.getRemotePort());
- setURI(URI.create(accessLogEntry.getRawPath()));
- setCookie(accessLogEntry.getCookieType(), accessLogEntry.getCookie());
- setWeekOfRegistration(accessLogEntry.getWeekOfRegistration());
- setProfile(accessLogEntry.getProfile());
- setInternationalInfo(accessLogEntry.getInternationalInfo());
- setContentAttribute(accessLogEntry.getContentAttribute());
- setAdSpaceID(accessLogEntry.getAdSpaceID());
- setErrorMessage(accessLogEntry.getErrorMessage());
- setFileName(accessLogEntry.getFileName());
- setUserAgent(accessLogEntry.getUserAgent());
- setWebfactsDigitalSignature(accessLogEntry.getWebfactsDigitalSignature());
- setReferer(accessLogEntry.getReferer());
- setRequestExtra(accessLogEntry.getRequestExtra());
- setResponseExtra(accessLogEntry.getResponseExtra());
- setResultFromCache(accessLogEntry.getResultFromCache());
- setHttpMethod(accessLogEntry.getHttpMethod());
- setPartner(accessLogEntry.getPartner());
- setAdRationale(accessLogEntry.getAdRationale());
- setIncrementSlotByOneRequest(accessLogEntry.getIncrementSlotByOneRequest());
- setZDataIncrementSlotByOneRequest(accessLogEntry.getZDataIncrementSlotByOneRequest());
- setHostString(accessLogEntry.getHostString());
- setPeerAddress(accessLogEntry.getPeerAddress());
- setPeerPort(accessLogEntry.getPeerPort());
- adInfos = accessLogEntry.getAdInfos();
- keyValues = accessLogEntry.getKeyValues();
-
- return toYApacheAccessEntry();
- }
-
- private static final Map<CookieType, Character> cookieTypeFirstCharMap = new HashMap<>();
- static {
- cookieTypeFirstCharMap.put(CookieType.b, '1');
- cookieTypeFirstCharMap.put(CookieType.l, '2');
- cookieTypeFirstCharMap.put(CookieType.n, '3');
- cookieTypeFirstCharMap.put(CookieType.geocookie,'5');
- cookieTypeFirstCharMap.put(CookieType.I, '7');
- cookieTypeFirstCharMap.put(CookieType.R, '9');
- cookieTypeFirstCharMap.put(CookieType.Y, 'c');
- cookieTypeFirstCharMap.put(CookieType.M, 'M');
- }
-
- private List<AccessLogEntry.AdInfo> adInfos;
- private String spaceID;
-
- private String ipV4AddressInDotDecimalNotation;
- private String unixTimeStamp;
- private String durationBetweenRequestResponseInMS;
- private String numBytesReturned;
- private String uri;
-
-
- private String remoteAddress;
- private int remotePort;
- private String peerAddress;
- private int peerPort;
-
- private Map<String,List<String>> keyValues;
-
- private static Logger logger = Logger.getLogger(YApacheFormatter.class.getName());
-
- private void setCookie(CookieType type, String cookie) {
- final Character firstChar = cookieTypeFirstCharMap.get(type);
- if (firstChar == null) {
- return;
- }
- addField(firstChar, cookie);
- }
-
- private void setWeekOfRegistration( String weekOfRegistration ) {
- addField('4', weekOfRegistration);
- }
-
- private void setProfile( String profile ) {
- addField('6', profile);
- }
-
- private void setInternationalInfo( String intl ) {
- addField('8', intl);
- }
-
- private void setContentAttribute( String contentAttribute ) {
- addField('a', contentAttribute);
- }
-
- private void setAdSpaceID(String spaceID) {
- this.spaceID = spaceID;
- }
-
- private static class AdInfo {
-
- private void setAdID(String id) {
- add('B', id);
- }
-
- private void setMatchID(String id) {
- add('C', id);
- }
-
- private void setPosition(String position) {
- add('D', position);
- }
-
- private void setProperty(String property) {
- add('F', property);
- }
-
- private void setCPC(String cpc) {
- add('G', cpc);
- }
-
- private void setAdClientVersion(String adClientVersion) {
- add('K', adClientVersion);
- }
-
- private void setLinkID(String id) {
- add('L', id);
- }
-
- private void setBidPosition(String bidPosition) {
- add('P', bidPosition);
- }
-
- private StringBuilder adInfo = new StringBuilder();
-
- AdInfo(final AccessLogEntry.AdInfo model) {
- final String modelAdServerString = model.getAdServerString();
- if (modelAdServerString != null) {
- adInfo.append(modelAdServerString);
- }
- setAdID(model.getAdID());
- setMatchID(model.getMatchID());
- setPosition(model.getPosition());
- setProperty(model.getProperty());
- setCPC(model.getCPC());
- setAdClientVersion(model.getAdClientVersion());
- setLinkID(model.getLinkID());
- setBidPosition(model.getBidPosition());
- }
-
- String adInfoString() {
- return adInfo.toString();
- }
-
- private void add(char controlChar, String value) {
- if (value == null) {
- return;
- }
- adInfo.append( controlCharacter(controlChar) );
- adInfo.append( value );
- }
- }
-
- private void setWebfactsDigitalSignature(String signature) {
- addField('d', signature);
- }
-
- private void setErrorMessage(String errorMessage) {
- addField('e', errorMessage);
- }
-
- private void setFileName(String fileName) {
- addField('f', fileName);
- }
-
- private void setUserAgent(String userAgent) {
- addField('g', userAgent);
- }
-
- private void setReferer(String referer) {
- addField('r', referer);
- }
-
- private void setRequestExtra(String requestExtra) {
- if (requestExtra == null || requestExtra.isEmpty()) {
- return;
- }
- if (fieldPrefix != requestExtra.charAt(0)) {
- yApacheEntry.append(fieldPrefix);
- }
- yApacheEntry.append(requestExtra);
- }
-
- private void setResponseExtra(String responseExtra) {
- if (responseExtra == null || responseExtra.isEmpty()) {
- return;
- }
- if (fieldPrefix != responseExtra.charAt(0)) {
- yApacheEntry.append(fieldPrefix);
- }
- yApacheEntry.append(responseExtra);
- }
-
- private void setResultFromCache(Boolean fromCache) {
- if (fromCache == null) {
- return;
- }
- addField('h',
- fromCache ? "1" : "0");
- }
-
- private void setHttpMethod(String method) {
- addField('m', method);
- }
-
- private void setPartner(String partner) {
- addField('p', partner);
- }
-
- private void setAdRationale(String adRationale) {
- addField('R', adRationale);
- }
-
- private void setIncrementSlotByOneRequest(String slotName) {
- addField('t', slotName);
- }
-
- private void setZDataIncrementSlotByOneRequest(String slotName) {
- addField('z', slotName);
- }
-
- private void setHostString(String hostString) {
- addField('w', hostString);
- }
-
- private StringBuilder yApacheEntry = new StringBuilder();
-
- private static char fieldPrefix = controlCharacter('E');
- private static char valueSeparator = controlCharacter('A');
-
-
- //assumes A <= c <= Z
- private static char controlCharacter(char c) {
- return (char)((c - 'A') + 1);
- }
-
- private void appendStartOfField(StringBuilder builder, char firstChar) {
- builder.append(fieldPrefix);
- builder.append(firstChar);
- }
-
- private void addField(char firstChar, String field) {
- if (field == null) {
- return;
- }
- appendStartOfField(yApacheEntry, firstChar);
- yApacheEntry.append(field);
- }
-
- private void addDecimalField(char firstChar, int field) {
- addField(firstChar, Integer.toString(field));
- }
-
- private String to8ByteHexString(final long value) {
- Formatter formatter = new Formatter();
- formatter.format("%08x", value);
- return formatter.toString();
- }
-
- private void appendAdInfo(StringBuilder buf) {
- if (spaceID != null || ! adInfos.isEmpty()) {
- buf.append(fieldPrefix);
- buf.append('b');
-
- if (spaceID != null) {
- buf.append( controlCharacter('A') );
- buf.append( spaceID );
- }
-
- for (AccessLogEntry.AdInfo adInfo : adInfos ) {
- buf.append( new AdInfo(adInfo).adInfoString() );
- }
- }
- }
-
- private void appendFirst32Bytes(StringBuilder buf) {
- buf.append(ipv4AddressToHexString(ipV4AddressInDotDecimalNotation));
- buf.append(unixTimeStamp);
- buf.append(durationBetweenRequestResponseInMS);
- buf.append(numBytesReturned);
-
- assert(buf.length() == 32);
- }
-
- private String toYApacheAccessEntry() {
- StringBuilder b = new StringBuilder();
-
- appendFirst32Bytes(b);
- b.append(uri);
-
- b.append(yApacheEntry);
- appendIPv6AddressInfo(b);
- appendAdInfo(b);
-
- appendKeyValues(b);
- return b.toString();
- }
-
- private void appendIPv6AddressInfo(StringBuilder builder) {
- appendStartOfField(builder, 'A');
- final boolean remoteAddressesAreEqual = Objects.equals(ipV4AddressInDotDecimalNotation, remoteAddress);
- if (!remoteAddressesAreEqual && remoteAddress != null) {
- builder.append('A').append(remoteAddress).append(valueSeparator);
- }
-
- builder.append('B').append(remotePort);
-
- if (peerAddress != null) {
- builder.append(valueSeparator).append('C').append(peerAddress);
- }
-
- if (peerPort > 0 && peerPort != remotePort) {
- builder.append(valueSeparator).append('D').append(peerPort);
- }
- }
-
- /**
- * Encodes key-values added to this entry at the "property extension" key 'X' as
- * <code>^EY[key1]^F[value1.1]^B[value1.2]^A[key2]^F[value2]</code>,
- */
- private void appendKeyValues(StringBuilder b) {
- if (keyValues==null) return;
- b.append(fieldPrefix);
- b.append('X');
- for (Map.Entry<String,List<String>> entry : keyValues.entrySet()) {
- b.append(entry.getKey());
- b.append(controlCharacter('F'));
- for (Iterator<String> i=entry.getValue().iterator(); i.hasNext(); ) {
- b.append(i.next());
- if (i.hasNext())
- b.append(controlCharacter('B'));
- }
- b.append(controlCharacter('A'));
- }
- b.deleteCharAt(b.length()-1); // Deletes the last ^A to be able to do foreach looping :-)
- }
-
- private String ipv4AddressToHexString(String ipv4AddressInDotDecimalNotation) {
- try {
- String[] parts = ipv4AddressInDotDecimalNotation.split("\\.");
- if (parts.length != 4) {
- throw new Exception();
- } else {
- Formatter byteHexFormatter = new Formatter();
-
- for (String part : parts) {
- int i = Integer.parseInt(part);
- if ( i > 0xff || i < 0)
- throw new Exception();
- byteHexFormatter.format("%02x", i);
- }
-
- return byteHexFormatter.toString();
- }
- } catch( Exception e ) {
- logger.log(Level.WARNING, "IPv4 address not in dot decimal notation: " +
- ipv4AddressInDotDecimalNotation);
- return "00000000";
- }
- }
-
- private void setIpV4Address(String ipV4AddressInDotDecimalNotation) {
- this.ipV4AddressInDotDecimalNotation = ipV4AddressInDotDecimalNotation;
- }
-
- private void setTimeStampMillis(long numMillisSince1Jan1970AtMidnightUTC) {
- int unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000);
-
- if (numMillisSince1Jan1970AtMidnightUTC/1000 > 0x7fffffff) {
- logger.log(Level.WARNING, "A year 2038 problem occurred.");
- logger.log(Level.INFO, "numMillisSince1Jan1970AtMidnightUTC: "
- + numMillisSince1Jan1970AtMidnightUTC);
- unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000 % 0x7fffffff);
- }
-
- unixTimeStamp = to8ByteHexString(unixTime);
- }
-
- private void setDurationBetweenRequestResponseMillis(long timeInMillis) {
- long timeInMicroSeconds = timeInMillis*1000;
- if (timeInMicroSeconds > 0xffffffffL) {
- logger.log(Level.WARNING, "Duration too long: " + timeInMillis);
- timeInMicroSeconds = 0xffffffffL;
- }
-
- durationBetweenRequestResponseInMS = to8ByteHexString(timeInMicroSeconds);
- }
-
- private void setReturnedContentSize(long byteCount) {
- numBytesReturned = to8ByteHexString(byteCount);
- }
-
- private void setURI(final URI uri) {
- setNormalizedURI(uri.normalize());
- }
-
- private void setNormalizedURI(final URI normalizedUri) {
- String uriString = normalizedUri.getPath();
- if (normalizedUri.getRawQuery() != null) {
- uriString = uriString + "?" + normalizedUri.getRawQuery();
- }
-
- this.uri = uriString;
- }
-
- private void setRemoteAddress(String remoteAddress) {
- this.remoteAddress = remoteAddress;
- }
-
- private void setRemotePort(int remotePort) {
- this.remotePort = remotePort;
- }
-
- private void setPeerAddress(final String peerAddress) {
- this.peerAddress = peerAddress;
- }
-
- private void setPeerPort(int peerPort) {
- this.peerPort = peerPort;
- }
-
- /** Sets the status code, which will end up in the "s" field. If this is 200 the field is not written (by spec). */
- private void setStatusCode(int statusCode) {
- if (statusCode == 0) {
- return;
- }
- if (statusCode!=200)
- addDecimalField('s', statusCode);
- }
-
-}
diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java
deleted file mode 100644
index 30442089f18..00000000000
--- a/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java
+++ /dev/null
@@ -1,289 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.logging;
-
-import com.yahoo.container.core.AccessLogConfig;
-import org.hamcrest.MatcherAssert;
-import org.junit.Test;
-
-import java.io.*;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Tony Vaagenes
- */
-public class YApacheLogTestCase {
-
- private static String ipAddress = "152.200.54.243";
- private static final String EMPTY_REFERRER = "";
- private static final String EMPTY_USERAGENT = "";
-
- @Test
- public void testIt() throws Exception {
- AccessLogEntry entry = new AccessLogEntry();
- addCommonEntries(entry);
-
- entry.setAdSpaceID("676817");
-
- AccessLogEntry.AdInfo ad1 = new AccessLogEntry.AdInfo();
- AccessLogEntry.AdInfo ad2 = new AccessLogEntry.AdInfo();
- AccessLogEntry.AdInfo ad3 = new AccessLogEntry.AdInfo();
-
- ad1.setAdID("134263");
- ad1.setMatchID("29213.323310.2048738.221486");
-
- ad2.setAdID("127077");
- ad2.setMatchID("26036.316814.2030021.40354");
-
- ad3.setAdID("127611");
- ad3.setMatchID("26036.330708.2043270.64665");
-
- entry.addAdInfo(ad1);
- entry.addAdInfo(ad2);
- entry.addAdInfo(ad3);
-
- entry.setUserAgent("Mozilla/4.05 [en] (Win95; I)");
- entry.setWebfactsDigitalSignature("Wk6_cAzC`4IKP\7&)$");
-
- entry.setRemoteAddress("17.6.5.4");
-
- String expectedOutput =
- "98c836f3" +
- "36e38385" +
- "0001dc90" +
- "00002693" +
- "/Business/Companies/Financial_Services/Investment_Services/Mutual_Funds/" +
- "\u0005gMozilla/4.05 [en] (Win95; I)" +
- "\u0005dWk6_cAzC`4IKP\7&)$" +
- "\u0005AA17.6.5.4\u0001B12345" +
- "\u0005b\u0001676817" + //adinfo
- "\u0002134263" + "\u000329213.323310.2048738.221486" +
- "\u0002127077" + "\u000326036.316814.2030021.40354" +
- "\u0002127611" + "\u000326036.330708.2043270.64665";
-
- assertEquals(expectedOutput, new YApacheFormatter(entry).format());
- }
-
- private void addCommonEntries(AccessLogEntry entry) throws URISyntaxException {
- entry.setIpV4Address(ipAddress);
- entry.setTimeStamp(920880005L*1000);
- entry.setDurationBetweenRequestResponse(122);
- entry.setReturnedContentSize(9875);
- entry.setRawPath("/Business/Companies/Financial_Services/Investment_Services/Mutual_Funds/");
- entry.setRemotePort(12345);
- }
-
- @Test
- public void test_remote_address_different_from_ip_address() throws Exception {
- AccessLogEntry entry = new AccessLogEntry();
- addCommonEntries(entry);
-
- entry.setRemoteAddress("FE80:0000:0000:0000:0202:B3FF:FE1E:8329");
-
- assertEquals("98c836f336e383850001dc9000002693/Business/Companies/Financial_Services/Investment_Services/Mutual_Funds/\u0005AAFE80:0000:0000:0000:0202:B3FF:FE1E:8329\u0001B12345",
- new YApacheFormatter(entry).format());
- }
-
- @Test
- public void test_remote_address_same_as_ip_address_does_not_cause_double_adding() throws Exception {
- AccessLogEntry entry = new AccessLogEntry();
- addCommonEntries(entry);
- entry.setRemoteAddress(ipAddress);
-
- assertThat(new YApacheFormatter(entry).format(), not(containsString(ipAddress)));
- }
-
- @Test
- public void test_status_code_stored_as_decimal() throws Exception {
- AccessLogEntry entry = new AccessLogEntry();
- addCommonEntries(entry);
- entry.setStatusCode(404);
-
- assertThat(new YApacheFormatter(entry).format(), containsString("s404"));
- }
-
- /**
- * author someone-else. Please rewrite this.
- */
- @Test
- public void testYApacheAccessLogWithDateNamingScheme() {
- AccessLogConfig.Builder builder = new AccessLogConfig.Builder().
- fileHandler(new AccessLogConfig.FileHandler.Builder().
- pattern("yapachetest/testaccess.%Y%m%d%H%M%S").
- symlink("testaccess"));
- AccessLogConfig config = new AccessLogConfig(builder);
- YApacheAccessLog accessLog = new YApacheAccessLog(config);
- try {
- final AccessLogEntry entry = newAccessLogEntry("hans");
- accessLog.log(entry);
-
- // wait for the log writing thread to do all its work, then check it did it right
-
- // check that symlink appears
- int waitTimeMs=0;
- while ( ! new File("yapachetest/testaccess").exists()) {
- Thread.sleep(2);
- waitTimeMs+=2;
- if (waitTimeMs>40*1000)
- throw new RuntimeException("Waited 40 seconds for the configured symlink to be created, giving up");
- }
- // ..and check that the log entry is written
- waitTimeMs=0;
- while ( ! containsExpectedLine("00000000000000010000271000000008?query=hans","yapachetest/testaccess",0)) {
- Thread.sleep(2);
- waitTimeMs+=2;
- if (waitTimeMs>40*1000)
- throw new RuntimeException("Waited 40 seconds for a log file entry to be written, giving up");
- }
- }
- catch (IOException e) {
- throw new RuntimeException("yapache log io exception",e);
- }
- catch (InterruptedException e) {
- throw new RuntimeException("Interruption",e);
- }
- finally {
- accessLog.shutdown();
- deleteDirectory("yapachetest");
- }
- }
-
- @Test
- public void testThatQueryWithEncodedCharactersIsLoggedInEncodedForm() {
- final String query = "%5E%3B%22";
- final AccessLogEntry entry = new AccessLogEntry();
- entry.setRawPath(newQueryPath(query));
- assertThat(new YApacheFormatter(entry).format(), containsString(query));
- }
-
- private AccessLogEntry newAccessLogEntry(final String query) {
- final AccessLogEntry entry = new AccessLogEntry();
- entry.setIpV4Address("0.0.0.0");
- entry.setUser("user");
- entry.setHttpMethod("GET");
- entry.setRawPath(newQueryPath(query));
- entry.setHttpVersion("HTTP/1.1");
- entry.setReferer(EMPTY_REFERRER);
- entry.setUserAgent(EMPTY_USERAGENT);
- entry.setRemoteAddress(new InetSocketAddress(0));
- entry.setTimeStamp(1000);
- entry.setDurationBetweenRequestResponse(10);
- entry.setReturnedContentSize(8);
- entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10));
- entry.setStatusCode(200);
- return entry;
- }
-
- // Prefixes that don't collide with any in the specified log format.
- private static final char FIELD_KEY_REQUEST_EXTRA = '@';
- private static final char FIELD_KEY_RESPONSE_EXTRA = '#';
-
-
- private static List<String> subArrayAsList(final String[] fields, final int fromIndex) {
- return Arrays.asList(fields).subList(fromIndex, fields.length);
- }
-
- private static Map<Character, String> makeFieldMap(final Iterable<String> fields) {
- final Map<Character, String> fieldMap = new HashMap<>();
- fields.forEach(field -> {
- final String existingValue = fieldMap.putIfAbsent(field.charAt(0), field.substring(1));
- MatcherAssert.assertThat("Attempt to insert field " + field + " would overwrite value", existingValue, is(nullValue()));
- });
- return fieldMap;
- }
-
- /**
- * author someone-else. Please rewrite this.
- */
- private void assertCorrectSequenceBehavior(int startN) throws IOException, InterruptedException {
- AccessLogConfig.Builder builder = new AccessLogConfig.Builder().
- fileHandler(new AccessLogConfig.FileHandler.Builder().
- pattern("yapachetest2/access").
- compressOnRotation(false));
-
- AccessLogConfig config = new AccessLogConfig(builder);
- YApacheAccessLog accessLog = new YApacheAccessLog(config);
- try {
- // log and rotate trice
- accessLog.log(newAccessLogEntry("query1"));
- accessLog.rotateNow();
- accessLog.log(newAccessLogEntry("query2"));
- accessLog.rotateNow();
- accessLog.log(newAccessLogEntry("query3.1"));
- accessLog.log(newAccessLogEntry("query3.2"));
- accessLog.rotateNow();
- accessLog.log(newAccessLogEntry("query4"));
-
- // wait for the last rotation, which should cause us to have an "access" file containing query4
- int waitTimeMs=0;
- while ( ! containsExpectedLine("00000000000000010000271000000008?query=query4","yapachetest2/access",0)) {
- Thread.sleep(2);
- waitTimeMs+=2;
- if (waitTimeMs>40*1000)
- throw new RuntimeException("Waited 40 seconds for the right log file entry to be written, giving up");
- }
-
- // Should now have 3 rotated away files
- assertTrue(containsExpectedLine("00000000000000010000271000000008?query=query1","yapachetest2/access." + (startN+0),0));
- assertTrue(containsExpectedLine("00000000000000010000271000000008?query=query2","yapachetest2/access." + (startN+1),0));
- assertTrue(containsExpectedLine("00000000000000010000271000000008?query=query3.1","yapachetest2/access." + (startN+2),0));
- assertTrue(containsExpectedLine("00000000000000010000271000000008?query=query3.2","yapachetest2/access." + (startN+2),1));
- }
- finally {
- accessLog.shutdown();
- deleteDirectory("yapachetest2");
- }
- }
-
- private void deleteDirectory(String name) {
- File dir=new File(name);
- if (! dir.exists()) return;
- for (File f : dir.listFiles())
- f.delete();
- dir.delete();
- }
-
- /**
- * Returns whether this file contains this line as the first one.
- * If line is null: Checks that the file is empty.
- *
- * author someone-else. Please rewrite this.
- */
- private boolean containsExpectedLine(String line,String file,int lineNumber) throws IOException {
- BufferedReader reader=null;
- try {
- reader=new BufferedReader(new FileReader(file));
- if (line==null) return reader.readLine()==null;
- while (lineNumber-- > 0) {
- String l = reader.readLine();
- }
- String l = reader.readLine();
- return l != null && l.startsWith(line);
- }
- catch (FileNotFoundException e) {
- return false;
- }
- finally {
- if (reader!=null)
- reader.close();
- }
- }
-
- private static String newQueryPath(final String query) {
- return "http://localhost?query=" + query;
- }
-
-}