diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-03-11 15:28:10 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2019-03-11 15:28:10 +0100 |
commit | 79d1a0a92f4548204ade47159204bf1b3c90d2fa (patch) | |
tree | 8b7718d7cc391cbf5d2ba451198d8840180a5964 /container-accesslogging/src/main/java/com/yahoo | |
parent | 48ce50681ad29a1a17446dbb1f0413615ca35725 (diff) |
GC unused yapache log and derivatives.
Diffstat (limited to 'container-accesslogging/src/main/java/com/yahoo')
3 files changed, 1 insertions, 808 deletions
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index 46e5aa95dab..72eaf129453 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -7,7 +7,6 @@ import org.apache.commons.lang.builder.ReflectionToStringBuilder; import javax.security.auth.x500.X500Principal; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.URI; import java.security.Principal; import java.util.ArrayList; import java.util.Collections; @@ -29,23 +28,13 @@ import static java.util.stream.Collectors.toMap; * but the parameters should be different for each * invocation of the same method.</p> * - * This class is thread-safe, but the inner class {@link AdInfo} is not. + * This class is thread-safe. * * @author Tony Vaagenes * @author bakksjo * @author bjorncs */ public class AccessLogEntry { - public enum CookieType { - b, - l, - n, - geocookie, - I, - R, - Y, - M; - } // Sadly, there's no way to do compile-time validation of these field references. private static final String[] FIELDS_EXCLUDED_FROM_TOSTRING = new String[] { @@ -54,42 +43,25 @@ public class AccessLogEntry { private final Object monitor = new Object(); - private List<AdInfo> adInfos; - private String spaceID; - private String ipV4AddressInDotDecimalNotation; private long timeStampMillis; private long durationBetweenRequestResponseMillis; private long numBytesReturned; - private String remoteAddress; private int remotePort; private String peerAddress; private int peerPort; - private CookieType cookieType; - private String cookie; - private String weekOfRegistration; private String profile; - private String internationalInfo; - private String contentAttribute; - private String webfactsDigitalSignature; private String errorMessage; private String fileName; private String userAgent; private String referer; private String user; private HitCounts hitCounts; - private String requestExtra; - private String responseExtra; - private Boolean resultFromCache; private String httpMethod; private String httpVersion; - private String partner; - private String adRationale; - private String incrementSlotByOneRequest; - private String zDataIncrementSlotByOneRequest; private String hostString; private int statusCode; private String scheme; @@ -101,40 +73,6 @@ public class AccessLogEntry { private ListMap<String,String> keyValues=null; - public void setCookie( CookieType type, String cookie) { - synchronized (monitor) { - requireNull(this.cookieType); - requireNull(this.cookie); - this.cookieType = type; - this.cookie = cookie; - } - } - - public CookieType getCookieType() { - synchronized (monitor) { - return cookieType; - } - } - - public String getCookie() { - synchronized (monitor) { - return cookie; - } - } - - public void setWeekOfRegistration( String weekOfRegistration ) { - synchronized (monitor) { - requireNull(this.weekOfRegistration); - this.weekOfRegistration = weekOfRegistration; - } - } - - public String getWeekOfRegistration() { - synchronized (monitor) { - return weekOfRegistration; - } - } - public void setProfile( String profile ) { synchronized (monitor) { requireNull(this.profile); @@ -148,168 +86,6 @@ public class AccessLogEntry { } } - public void setInternationalInfo( String intl ) { - synchronized (monitor) { - requireNull(this.internationalInfo); - this.internationalInfo = intl; - } - } - - public String getInternationalInfo() { - synchronized (monitor) { - return internationalInfo; - } - } - - public void setContentAttribute( String contentAttribute ) { - synchronized (monitor) { - requireNull(this.contentAttribute); - this.contentAttribute = contentAttribute; - } - } - - public String getContentAttribute() { - synchronized (monitor) { - return contentAttribute; - } - } - - public void setAdSpaceID(String spaceID) { - synchronized (monitor) { - requireNull(this.spaceID); - this.spaceID = spaceID; - } - } - - public String getAdSpaceID() { - synchronized (monitor) { - return spaceID; - } - } - - public void addAdInfo(AdInfo adInfo) { - synchronized (monitor) { - if (adInfos == null) { - adInfos = new ArrayList<>(); - } - adInfos.add( adInfo ); - } - } - - public List<AdInfo> getAdInfos() { - synchronized (monitor) { - if (adInfos == null) { - return Collections.emptyList(); - } - // TODO: The returned list is unmodifiable, but its elements are not. But we're all friendly here, right? - return Collections.unmodifiableList(adInfos); - } - } - - /** - * This class is NOT thread-safe. It is assumed that a single instance is created/written by a single thread, - * and all reads happen-after creation/population, i.e. no mutation after the instance is shared between threads. - */ - public static class AdInfo { - - private String adServerString; - private String adId; - private String matchId; - private String position; - private String property; - private String cpc; - private String adClientVersion; - private String linkId; - private String bidPosition; - - public void setAdID(String id) { - this.adId = id; - } - - public String getAdID() { - return adId; - } - - public void setMatchID(String id) { - this.matchId = id; - } - - public String getMatchID() { - return matchId; - } - - public void setPosition(String position) { - this.position = position; - } - - public String getPosition() { - return position; - } - - public void setProperty(String property) { - this.property = property; - } - - public String getProperty() { - return property; - } - - public void setCPC(String cpc) { - this.cpc = cpc; - } - - public String getCPC() { - return cpc; - } - - public void setAdClientVersion(String adClientVersion) { - this.adClientVersion = adClientVersion; - } - - public String getAdClientVersion() { - return adClientVersion; - } - - public void setLinkID(String id) { - this.linkId = id; - } - - public String getLinkID() { - return linkId; - } - - public void setBidPosition(String bidPosition) { - this.bidPosition = bidPosition; - } - - public String getBidPosition() { - return bidPosition; - } - - public AdInfo() {} - - AdInfo(String adServerString) { - this.adServerString = adServerString; - } - - String getAdServerString() { - return adServerString; - } - } - - public void setWebfactsDigitalSignature(String signature) { - synchronized (monitor) { - requireNull(this.webfactsDigitalSignature); - this.webfactsDigitalSignature = signature; - } - } - - public String getWebfactsDigitalSignature() { - synchronized (monitor) { - return webfactsDigitalSignature; - } - } - public void setErrorMessage(String errorMessage) { synchronized (monitor) { requireNull(this.errorMessage); @@ -388,18 +164,6 @@ public class AccessLogEntry { } } - public String getRequestExtra() { - synchronized (monitor) { - return requestExtra; - } - } - - public String getResponseExtra() { - synchronized (monitor) { - return responseExtra; - } - } - public void addKeyValue(String key,String value) { synchronized (monitor) { if (keyValues == null) { @@ -431,19 +195,6 @@ public class AccessLogEntry { entry -> valueConverter.apply(entry.getValue()))); } - public void setResultFromCache(boolean fromCache) { - synchronized (monitor) { - requireNull(this.resultFromCache); - this.resultFromCache = fromCache; - } - } - - public Boolean getResultFromCache() { - synchronized (monitor) { - return resultFromCache; - } - } - public enum HttpMethod { GET, POST; } @@ -478,58 +229,6 @@ public class AccessLogEntry { } } - public void setPartner(String partner) { - synchronized (monitor) { - requireNull(this.partner); - this.partner = partner; - } - } - - public String getPartner() { - synchronized (monitor) { - return partner; - } - } - - public void setAdRationale(String adRationale) { - synchronized (monitor) { - requireNull(this.adRationale); - this.adRationale = adRationale; - } - } - - public String getAdRationale() { - synchronized (monitor) { - return adRationale; - } - } - - public void setIncrementSlotByOneRequest(String slotName) { - synchronized (monitor) { - requireNull(this.incrementSlotByOneRequest); - this.incrementSlotByOneRequest = slotName; - } - } - - public String getIncrementSlotByOneRequest() { - synchronized (monitor) { - return incrementSlotByOneRequest; - } - } - - public void setZDataIncrementSlotByOneRequest(String slotName) { - synchronized (monitor) { - requireNull(this.zDataIncrementSlotByOneRequest); - this.zDataIncrementSlotByOneRequest = slotName; - } - } - - public String getZDataIncrementSlotByOneRequest() { - synchronized (monitor) { - return zDataIncrementSlotByOneRequest; - } - } - public void setHostString(String hostString) { synchronized (monitor) { requireNull(this.hostString); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java deleted file mode 100644 index 6fb0cd8161a..00000000000 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheAccessLog.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.logging; - -import com.yahoo.container.core.AccessLogConfig; - -import java.util.logging.Level; - -/** - * Log a message in yApache log format. - * - * @author tinyv - */ -public final class YApacheAccessLog implements AccessLogInterface { - - private final AccessLogHandler logHandler; - - public YApacheAccessLog(AccessLogConfig config) { - logHandler = new AccessLogHandler(config.fileHandler()); - } - - @Override - public void log(final AccessLogEntry logEntry) { - logHandler.access.log(Level.INFO, new YApacheFormatter(logEntry).format() + '\n'); - } - - - // TODO: This is never called. We should have a DI provider and call this method from its deconstruct. - public void shutdown() { - logHandler.shutdown(); - } - - void rotateNow() { - logHandler.rotateNow(); - } - -} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java deleted file mode 100644 index f63d02b1a77..00000000000 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/YApacheFormatter.java +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.logging; - -import java.net.URI; -import java.util.Formatter; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.container.logging.AccessLogEntry.CookieType; - -/** - * Formatting of an {@link AccessLogEntry} in the yapache access log format. - * - * @author Tony Vaagenes - * @author bakksjo - */ -public class YApacheFormatter { - - private AccessLogEntry accessLogEntry; - - public YApacheFormatter(final AccessLogEntry entry) { - accessLogEntry = entry; - } - - public String format() { - // Initial 32-byte fixed block of mandatory, non-prefixed fields. - setIpV4Address(accessLogEntry.getIpV4Address()); - setTimeStampMillis(accessLogEntry.getTimeStampMillis()); - setDurationBetweenRequestResponseMillis(accessLogEntry.getDurationBetweenRequestResponseMillis()); - setReturnedContentSize(accessLogEntry.getReturnedContentSize()); - - // Optional, prefixed fields in arbitrary order. - setStatusCode(accessLogEntry.getStatusCode()); - setRemoteAddress(accessLogEntry.getRemoteAddress()); - setRemotePort(accessLogEntry.getRemotePort()); - setURI(URI.create(accessLogEntry.getRawPath())); - setCookie(accessLogEntry.getCookieType(), accessLogEntry.getCookie()); - setWeekOfRegistration(accessLogEntry.getWeekOfRegistration()); - setProfile(accessLogEntry.getProfile()); - setInternationalInfo(accessLogEntry.getInternationalInfo()); - setContentAttribute(accessLogEntry.getContentAttribute()); - setAdSpaceID(accessLogEntry.getAdSpaceID()); - setErrorMessage(accessLogEntry.getErrorMessage()); - setFileName(accessLogEntry.getFileName()); - setUserAgent(accessLogEntry.getUserAgent()); - setWebfactsDigitalSignature(accessLogEntry.getWebfactsDigitalSignature()); - setReferer(accessLogEntry.getReferer()); - setRequestExtra(accessLogEntry.getRequestExtra()); - setResponseExtra(accessLogEntry.getResponseExtra()); - setResultFromCache(accessLogEntry.getResultFromCache()); - setHttpMethod(accessLogEntry.getHttpMethod()); - setPartner(accessLogEntry.getPartner()); - setAdRationale(accessLogEntry.getAdRationale()); - setIncrementSlotByOneRequest(accessLogEntry.getIncrementSlotByOneRequest()); - setZDataIncrementSlotByOneRequest(accessLogEntry.getZDataIncrementSlotByOneRequest()); - setHostString(accessLogEntry.getHostString()); - setPeerAddress(accessLogEntry.getPeerAddress()); - setPeerPort(accessLogEntry.getPeerPort()); - adInfos = accessLogEntry.getAdInfos(); - keyValues = accessLogEntry.getKeyValues(); - - return toYApacheAccessEntry(); - } - - private static final Map<CookieType, Character> cookieTypeFirstCharMap = new HashMap<>(); - static { - cookieTypeFirstCharMap.put(CookieType.b, '1'); - cookieTypeFirstCharMap.put(CookieType.l, '2'); - cookieTypeFirstCharMap.put(CookieType.n, '3'); - cookieTypeFirstCharMap.put(CookieType.geocookie,'5'); - cookieTypeFirstCharMap.put(CookieType.I, '7'); - cookieTypeFirstCharMap.put(CookieType.R, '9'); - cookieTypeFirstCharMap.put(CookieType.Y, 'c'); - cookieTypeFirstCharMap.put(CookieType.M, 'M'); - } - - private List<AccessLogEntry.AdInfo> adInfos; - private String spaceID; - - private String ipV4AddressInDotDecimalNotation; - private String unixTimeStamp; - private String durationBetweenRequestResponseInMS; - private String numBytesReturned; - private String uri; - - - private String remoteAddress; - private int remotePort; - private String peerAddress; - private int peerPort; - - private Map<String,List<String>> keyValues; - - private static Logger logger = Logger.getLogger(YApacheFormatter.class.getName()); - - private void setCookie(CookieType type, String cookie) { - final Character firstChar = cookieTypeFirstCharMap.get(type); - if (firstChar == null) { - return; - } - addField(firstChar, cookie); - } - - private void setWeekOfRegistration( String weekOfRegistration ) { - addField('4', weekOfRegistration); - } - - private void setProfile( String profile ) { - addField('6', profile); - } - - private void setInternationalInfo( String intl ) { - addField('8', intl); - } - - private void setContentAttribute( String contentAttribute ) { - addField('a', contentAttribute); - } - - private void setAdSpaceID(String spaceID) { - this.spaceID = spaceID; - } - - private static class AdInfo { - - private void setAdID(String id) { - add('B', id); - } - - private void setMatchID(String id) { - add('C', id); - } - - private void setPosition(String position) { - add('D', position); - } - - private void setProperty(String property) { - add('F', property); - } - - private void setCPC(String cpc) { - add('G', cpc); - } - - private void setAdClientVersion(String adClientVersion) { - add('K', adClientVersion); - } - - private void setLinkID(String id) { - add('L', id); - } - - private void setBidPosition(String bidPosition) { - add('P', bidPosition); - } - - private StringBuilder adInfo = new StringBuilder(); - - AdInfo(final AccessLogEntry.AdInfo model) { - final String modelAdServerString = model.getAdServerString(); - if (modelAdServerString != null) { - adInfo.append(modelAdServerString); - } - setAdID(model.getAdID()); - setMatchID(model.getMatchID()); - setPosition(model.getPosition()); - setProperty(model.getProperty()); - setCPC(model.getCPC()); - setAdClientVersion(model.getAdClientVersion()); - setLinkID(model.getLinkID()); - setBidPosition(model.getBidPosition()); - } - - String adInfoString() { - return adInfo.toString(); - } - - private void add(char controlChar, String value) { - if (value == null) { - return; - } - adInfo.append( controlCharacter(controlChar) ); - adInfo.append( value ); - } - } - - private void setWebfactsDigitalSignature(String signature) { - addField('d', signature); - } - - private void setErrorMessage(String errorMessage) { - addField('e', errorMessage); - } - - private void setFileName(String fileName) { - addField('f', fileName); - } - - private void setUserAgent(String userAgent) { - addField('g', userAgent); - } - - private void setReferer(String referer) { - addField('r', referer); - } - - private void setRequestExtra(String requestExtra) { - if (requestExtra == null || requestExtra.isEmpty()) { - return; - } - if (fieldPrefix != requestExtra.charAt(0)) { - yApacheEntry.append(fieldPrefix); - } - yApacheEntry.append(requestExtra); - } - - private void setResponseExtra(String responseExtra) { - if (responseExtra == null || responseExtra.isEmpty()) { - return; - } - if (fieldPrefix != responseExtra.charAt(0)) { - yApacheEntry.append(fieldPrefix); - } - yApacheEntry.append(responseExtra); - } - - private void setResultFromCache(Boolean fromCache) { - if (fromCache == null) { - return; - } - addField('h', - fromCache ? "1" : "0"); - } - - private void setHttpMethod(String method) { - addField('m', method); - } - - private void setPartner(String partner) { - addField('p', partner); - } - - private void setAdRationale(String adRationale) { - addField('R', adRationale); - } - - private void setIncrementSlotByOneRequest(String slotName) { - addField('t', slotName); - } - - private void setZDataIncrementSlotByOneRequest(String slotName) { - addField('z', slotName); - } - - private void setHostString(String hostString) { - addField('w', hostString); - } - - private StringBuilder yApacheEntry = new StringBuilder(); - - private static char fieldPrefix = controlCharacter('E'); - private static char valueSeparator = controlCharacter('A'); - - - //assumes A <= c <= Z - private static char controlCharacter(char c) { - return (char)((c - 'A') + 1); - } - - private void appendStartOfField(StringBuilder builder, char firstChar) { - builder.append(fieldPrefix); - builder.append(firstChar); - } - - private void addField(char firstChar, String field) { - if (field == null) { - return; - } - appendStartOfField(yApacheEntry, firstChar); - yApacheEntry.append(field); - } - - private void addDecimalField(char firstChar, int field) { - addField(firstChar, Integer.toString(field)); - } - - private String to8ByteHexString(final long value) { - Formatter formatter = new Formatter(); - formatter.format("%08x", value); - return formatter.toString(); - } - - private void appendAdInfo(StringBuilder buf) { - if (spaceID != null || ! adInfos.isEmpty()) { - buf.append(fieldPrefix); - buf.append('b'); - - if (spaceID != null) { - buf.append( controlCharacter('A') ); - buf.append( spaceID ); - } - - for (AccessLogEntry.AdInfo adInfo : adInfos ) { - buf.append( new AdInfo(adInfo).adInfoString() ); - } - } - } - - private void appendFirst32Bytes(StringBuilder buf) { - buf.append(ipv4AddressToHexString(ipV4AddressInDotDecimalNotation)); - buf.append(unixTimeStamp); - buf.append(durationBetweenRequestResponseInMS); - buf.append(numBytesReturned); - - assert(buf.length() == 32); - } - - private String toYApacheAccessEntry() { - StringBuilder b = new StringBuilder(); - - appendFirst32Bytes(b); - b.append(uri); - - b.append(yApacheEntry); - appendIPv6AddressInfo(b); - appendAdInfo(b); - - appendKeyValues(b); - return b.toString(); - } - - private void appendIPv6AddressInfo(StringBuilder builder) { - appendStartOfField(builder, 'A'); - final boolean remoteAddressesAreEqual = Objects.equals(ipV4AddressInDotDecimalNotation, remoteAddress); - if (!remoteAddressesAreEqual && remoteAddress != null) { - builder.append('A').append(remoteAddress).append(valueSeparator); - } - - builder.append('B').append(remotePort); - - if (peerAddress != null) { - builder.append(valueSeparator).append('C').append(peerAddress); - } - - if (peerPort > 0 && peerPort != remotePort) { - builder.append(valueSeparator).append('D').append(peerPort); - } - } - - /** - * Encodes key-values added to this entry at the "property extension" key 'X' as - * <code>^EY[key1]^F[value1.1]^B[value1.2]^A[key2]^F[value2]</code>, - */ - private void appendKeyValues(StringBuilder b) { - if (keyValues==null) return; - b.append(fieldPrefix); - b.append('X'); - for (Map.Entry<String,List<String>> entry : keyValues.entrySet()) { - b.append(entry.getKey()); - b.append(controlCharacter('F')); - for (Iterator<String> i=entry.getValue().iterator(); i.hasNext(); ) { - b.append(i.next()); - if (i.hasNext()) - b.append(controlCharacter('B')); - } - b.append(controlCharacter('A')); - } - b.deleteCharAt(b.length()-1); // Deletes the last ^A to be able to do foreach looping :-) - } - - private String ipv4AddressToHexString(String ipv4AddressInDotDecimalNotation) { - try { - String[] parts = ipv4AddressInDotDecimalNotation.split("\\."); - if (parts.length != 4) { - throw new Exception(); - } else { - Formatter byteHexFormatter = new Formatter(); - - for (String part : parts) { - int i = Integer.parseInt(part); - if ( i > 0xff || i < 0) - throw new Exception(); - byteHexFormatter.format("%02x", i); - } - - return byteHexFormatter.toString(); - } - } catch( Exception e ) { - logger.log(Level.WARNING, "IPv4 address not in dot decimal notation: " + - ipv4AddressInDotDecimalNotation); - return "00000000"; - } - } - - private void setIpV4Address(String ipV4AddressInDotDecimalNotation) { - this.ipV4AddressInDotDecimalNotation = ipV4AddressInDotDecimalNotation; - } - - private void setTimeStampMillis(long numMillisSince1Jan1970AtMidnightUTC) { - int unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000); - - if (numMillisSince1Jan1970AtMidnightUTC/1000 > 0x7fffffff) { - logger.log(Level.WARNING, "A year 2038 problem occurred."); - logger.log(Level.INFO, "numMillisSince1Jan1970AtMidnightUTC: " - + numMillisSince1Jan1970AtMidnightUTC); - unixTime = (int)(numMillisSince1Jan1970AtMidnightUTC/1000 % 0x7fffffff); - } - - unixTimeStamp = to8ByteHexString(unixTime); - } - - private void setDurationBetweenRequestResponseMillis(long timeInMillis) { - long timeInMicroSeconds = timeInMillis*1000; - if (timeInMicroSeconds > 0xffffffffL) { - logger.log(Level.WARNING, "Duration too long: " + timeInMillis); - timeInMicroSeconds = 0xffffffffL; - } - - durationBetweenRequestResponseInMS = to8ByteHexString(timeInMicroSeconds); - } - - private void setReturnedContentSize(long byteCount) { - numBytesReturned = to8ByteHexString(byteCount); - } - - private void setURI(final URI uri) { - setNormalizedURI(uri.normalize()); - } - - private void setNormalizedURI(final URI normalizedUri) { - String uriString = normalizedUri.getPath(); - if (normalizedUri.getRawQuery() != null) { - uriString = uriString + "?" + normalizedUri.getRawQuery(); - } - - this.uri = uriString; - } - - private void setRemoteAddress(String remoteAddress) { - this.remoteAddress = remoteAddress; - } - - private void setRemotePort(int remotePort) { - this.remotePort = remotePort; - } - - private void setPeerAddress(final String peerAddress) { - this.peerAddress = peerAddress; - } - - private void setPeerPort(int peerPort) { - this.peerPort = peerPort; - } - - /** Sets the status code, which will end up in the "s" field. If this is 200 the field is not written (by spec). */ - private void setStatusCode(int statusCode) { - if (statusCode == 0) { - return; - } - if (statusCode!=200) - addDecimalField('s', statusCode); - } - -} |