diff options
9 files changed, 184 insertions, 20 deletions
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java new file mode 100644 index 00000000000..266437bb8bc --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java @@ -0,0 +1,63 @@ +package com.yahoo.container.logging; + +/** + * Carry information about how the query covered the document corpus. + */ +public class Coverage { + private final long docs; + private final long active; + private final long soonActive; + private final int degradedReason; + private final static int DEGRADED_BY_MATCH_PHASE = 1; + private final static int DEGRADED_BY_TIMEOUT = 2; + private final static int DEGRADED_BY_ADAPTIVE_TIMEOUT = 4; + public Coverage(long docs, long active, long soonActive, int degradedReason) { + this.docs = docs; + this.active = active; + this.soonActive = soonActive; + this.degradedReason = degradedReason; + } + + public long getDocs() { + return docs; + } + + public long getActive() { + return active; + } + + public static int toDegradation(boolean degradeByMatchPhase, boolean degradedByTimeout, boolean degradedByAdaptiveTimeout) { + int v = 0; + if (degradeByMatchPhase) { + v |= DEGRADED_BY_MATCH_PHASE; + } + if (degradedByTimeout) { + v |= DEGRADED_BY_TIMEOUT; + } + if (degradedByAdaptiveTimeout) { + v |= DEGRADED_BY_ADAPTIVE_TIMEOUT; + } + return v; + } + + public long getSoonActive() { return soonActive; } + + public boolean isDegraded() { return (degradedReason != 0) || isDegradedByNonIdealState(); } + public boolean isDegradedByMatchPhase() { return (degradedReason & DEGRADED_BY_MATCH_PHASE) != 0; } + public boolean isDegradedByTimeout() { return (degradedReason & DEGRADED_BY_TIMEOUT) != 0; } + public boolean isDegradedByAdapativeTimeout() { return (degradedReason & DEGRADED_BY_ADAPTIVE_TIMEOUT) != 0; } + public boolean isDegradedByNonIdealState() { return (degradedReason == 0) && (getResultPercentage() != 100);} + + /** + * An int between 0 (inclusive) and 100 (inclusive) representing how many + * percent coverage the result sets this Coverage instance contains information + * about had. + */ + public int getResultPercentage() { + if (docs < active) { + return (int) Math.round(docs * 100.0d / active); + } + return 100; + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java index 84ee3b505b1..fed12281962 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java @@ -16,19 +16,23 @@ public class HitCounts { private final long totalHitCount; private final int requestedHits; private final int requestedOffset; + private final Coverage coverage; - public HitCounts( - int retrievedHits, - int summaryCount, - long totalHitCount, - int requestedHits, - int requestedOffset) { + HitCounts(int retrievedHits, int summaryCount, long totalHitCount, int requestedHits, int requestedOffset) { + this(retrievedHits, summaryCount, totalHitCount, requestedHits, requestedOffset, + new Coverage(1,1,1,0)); + } + + public HitCounts(int retrievedHits, int summaryCount, long totalHitCount, + int requestedHits, int requestedOffset, Coverage coverage) + { this.retrievedHits = retrievedHits; this.summaryCount = summaryCount; this.totalHitCount = totalHitCount; this.requestedHits = requestedHits; this.requestedOffset = requestedOffset; + this.coverage = coverage; } /** @@ -69,4 +73,6 @@ public class HitCounts { return requestedOffset; } + public Coverage getCoverage() { return coverage; } + } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 595bd99a759..556b97ced62 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -23,6 +23,14 @@ import java.util.logging.Logger; * @author frodelu */ public class JSONFormatter { + private static final String COVERAGE = "coverage"; + private static final String COVERAGE_COVERAGE = "coverage"; + private static final String COVERAGE_DOCUMENTS = "documents"; + private static final String COVERAGE_DEGRADE = "degraded"; + private static final String COVERAGE_DEGRADE_MATCHPHASE = "match-phase"; + private static final String COVERAGE_DEGRADE_TIMEOUT = "timeout"; + private static final String COVERAGE_DEGRADE_ADAPTIVE_TIMEOUT = "adaptive-timeout"; + private static final String COVERAGE_DEGRADED_NON_IDEAL_STATE = "non-ideal-state"; private AccessLogEntry accessLogEntry; private final JsonFactory generatorFactory; @@ -92,6 +100,25 @@ public class JSONFormatter { generator.writeObjectFieldStart("search"); generator.writeNumberField("totalhits", getTotalHitCount(accessLogEntry.getHitCounts())); generator.writeNumberField("hits", getRetrievedHitCount(accessLogEntry.getHitCounts())); + Coverage c = accessLogEntry.getHitCounts().getCoverage(); + if (c != null) { + generator.writeObjectFieldStart(COVERAGE); + generator.writeNumberField(COVERAGE_COVERAGE, c.getResultPercentage()); + generator.writeNumberField(COVERAGE_DOCUMENTS, c.getDocs()); + if (c.isDegraded()) { + generator.writeObjectFieldStart(COVERAGE_DEGRADE); + if (c.isDegradedByMatchPhase()) + generator.writeBooleanField(COVERAGE_DEGRADE_MATCHPHASE, c.isDegradedByMatchPhase()); + if (c.isDegradedByTimeout()) + generator.writeBooleanField(COVERAGE_DEGRADE_TIMEOUT, c.isDegradedByTimeout()); + if (c.isDegradedByAdapativeTimeout()) + generator.writeBooleanField(COVERAGE_DEGRADE_ADAPTIVE_TIMEOUT, c.isDegradedByAdapativeTimeout()); + if (c.isDegradedByNonIdealState()) + generator.writeBooleanField(COVERAGE_DEGRADED_NON_IDEAL_STATE, c.isDegradedByNonIdealState()); + generator.writeEndObject(); + } + generator.writeEndObject(); + } generator.writeEndObject(); } diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index ceafbc18272..8bbb8500cfd 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -18,6 +18,9 @@ public class JSONLogTestCase { private static final String EMPTY_USERAGENT = ""; private AccessLogEntry newAccessLogEntry(final String query) { + return newAccessLogEntry(query, new Coverage(100,100,100,0)); + } + private AccessLogEntry newAccessLogEntry(final String query, Coverage coverage) { final AccessLogEntry entry = new AccessLogEntry(); entry.setRawQuery("query="+query); entry.setRawPath(""); @@ -25,7 +28,7 @@ public class JSONLogTestCase { entry.setHttpMethod("GET"); entry.setHttpVersion("HTTP/1.1"); entry.setUserAgent("Mozilla/4.05 [en] (Win95; I)"); - entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10)); + entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10, coverage)); entry.setHostString("localhost"); entry.setStatusCode(200); entry.setTimeStamp(920880005023L); @@ -58,7 +61,8 @@ public class JSONLogTestCase { "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + - "\"hits\":0" + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + "}" + "}"; @@ -87,7 +91,8 @@ public class JSONLogTestCase { "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + - "\"hits\":0" + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + "}," + "\"attributes\":{" + "\"singlevalue\":\"value1\"," + @@ -121,7 +126,8 @@ public class JSONLogTestCase { "\"remoteaddr\":\"FE80:0000:0000:0000:0202:B3FF:FE1E:8329\"," + "\"search\":{" + "\"totalhits\":1234," + - "\"hits\":0" + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + "}" + "}"; @@ -147,7 +153,8 @@ public class JSONLogTestCase { "\"remoteport\":1234," + "\"search\":{" + "\"totalhits\":1234," + - "\"hits\":0" + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + "}" + "}"; @@ -172,7 +179,7 @@ public class JSONLogTestCase { entry.setHttpMethod("GET"); entry.setHttpVersion("HTTP/1.1"); entry.setUserAgent("Mozilla/4.05 [en] (Win95; I; \"Best Browser Ever\")"); - entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10)); + entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10, new Coverage(100,200,200,0))); entry.setHostString("localhost"); entry.setStatusCode(200); entry.setTimeStamp(920880005023L); @@ -194,11 +201,44 @@ public class JSONLogTestCase { "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + - "\"hits\":0" + + "\"hits\":0," + + "\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"non-ideal-state\":true}}" + "}" + "}"; assertEquals(expectedOutput, new JSONFormatter(entry).format()); } + private void verifyCoverage(String coverage, AccessLogEntry entry) { + assertEquals("{\"ip\":\"152.200.54.243\"," + + "\"time\":920880005.023," + + "\"duration\":0.122," + + "\"responsesize\":9875," + + "\"code\":200," + + "\"method\":\"GET\"," + + "\"uri\":\"?query=test\"," + + "\"version\":\"HTTP/1.1\"," + + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + + "\"search\":{" + + "\"totalhits\":1234," + + "\"hits\":0," + + coverage + + "}" + + "}", new JSONFormatter(entry).format()); + } + + @Test + public void test_with_coverage_degradation() { + verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"non-ideal-state\":true}}", + newAccessLogEntry("test", new Coverage(100,200,200,0))); + verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"match-phase\":true}}", + newAccessLogEntry("test", new Coverage(100,200,200,1))); + verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"timeout\":true}}", + newAccessLogEntry("test", new Coverage(100,200,200,2))); + verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"adaptive-timeout\":true}}", + newAccessLogEntry("test", new Coverage(100,200,200,4))); + } } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 345911a810f..d20d5643647 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -71,7 +71,8 @@ "public com.yahoo.container.handler.Coverage setNodesTried(int)", "public int getFullResultSets()", "public int getResultSets()", - "public int getResultPercentage()" + "public int getResultPercentage()", + "public com.yahoo.container.logging.Coverage toLoggingCoverage()" ], "fields": [ "protected long docs", diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 84cc0734e7c..2ec5d34a0a6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -194,4 +194,11 @@ public class Coverage { return getFullResultSets() * 100 / getResultSets(); } + public com.yahoo.container.logging.Coverage toLoggingCoverage() { + int degradation = com.yahoo.container.logging.Coverage.toDegradation(isDegradedByMatchPhase(), + isDegradedByTimeout(), + isDegradedByAdapativeTimeout()); + return new com.yahoo.container.logging.Coverage(getDocs(), getActive(), getSoonActive(), degradation); + } + } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java index 5af94c798d1..92fe347f7d1 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingRequestHandlerTestCase.java @@ -58,7 +58,8 @@ public class LoggingRequestHandlerTestCase { @Override public HitCounts getHitCounts() { - return new HitCounts(1, 1, 1, 1, 1); + return new HitCounts(1, 1, 1, 1, 1, + getCoverage().toLoggingCoverage()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java index df138364b5b..1e19c6e7345 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java @@ -2,6 +2,7 @@ package com.yahoo.search.handler; import com.yahoo.container.handler.Timing; +import com.yahoo.container.logging.Coverage; import com.yahoo.container.logging.HitCounts; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -57,11 +58,13 @@ public class SearchResponse { } public static HitCounts createHitCounts(Query query, Result result) { - return new HitCounts(result.getHitCount(), - result.getConcreteHitCount(), - result.getTotalHitCount(), - query.getHits(), - query.getOffset()); + com.yahoo.container.handler.Coverage coverage = result.getCoverage(false); + + return (coverage != null) + ? new HitCounts(result.getHitCount(), result.getConcreteHitCount(), result.getTotalHitCount(), + query.getHits(), query.getOffset(), coverage.toLoggingCoverage()) + : new HitCounts(result.getHitCount(), result.getConcreteHitCount(), result.getTotalHitCount(), + query.getHits(), query.getOffset(), null); } } diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java index 7624185481e..b73da781cd6 100644 --- a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java @@ -67,4 +67,20 @@ public class CoverageTestCase { assertEquals(1, federationSearcherResult.getCoverage(create).getResultSets()); } + @Test + public void testCoverageConversion() { + Coverage c = new Coverage(6, 10); + c.setDegradedReason(7); + com.yahoo.container.logging.Coverage lc = c.toLoggingCoverage(); + assertEquals(lc.getDocs(), c.getDocs()); + assertEquals(lc.getActive(), c.getActive()); + assertEquals(lc.getSoonActive(), c.getSoonActive()); + assertEquals(lc.getResultPercentage(), c.getResultPercentage()); + assertEquals(lc.isDegraded(), c.isDegraded()); + assertEquals(lc.isDegradedByNonIdealState(), c.isDegradedByNonIdealState()); + assertEquals(lc.isDegradedByAdapativeTimeout(), c.isDegradedByAdapativeTimeout()); + assertEquals(lc.isDegradedByMatchPhase(), c.isDegradedByMatchPhase()); + assertEquals(lc.isDegradedByTimeout(), c.isDegradedByTimeout()); + } + } |