Add coverage info to access log

4 files changed, 149 insertions, 13 deletions

diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java
new file mode 100644
index 00000000000..266437bb8bc
--- /dev/null
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/Coverage.java
@@ -0,0 +1,63 @@
+package com.yahoo.container.logging;
+
+/**
+ * Carry information about how the query covered the document corpus.
+ */
+public class Coverage {
+    private final long docs;
+    private final long active;
+    private final long soonActive;
+    private final int degradedReason;
+    private final static int DEGRADED_BY_MATCH_PHASE = 1;
+    private final static int DEGRADED_BY_TIMEOUT = 2;
+    private final static int DEGRADED_BY_ADAPTIVE_TIMEOUT = 4;
+    public Coverage(long docs, long active, long soonActive, int degradedReason) {
+        this.docs = docs;
+        this.active = active;
+        this.soonActive = soonActive;
+        this.degradedReason = degradedReason;
+    }
+
+    public long getDocs() {
+        return docs;
+    }
+
+    public long getActive() {
+        return active;
+    }
+
+    public static int toDegradation(boolean degradeByMatchPhase, boolean degradedByTimeout, boolean degradedByAdaptiveTimeout) {
+        int v = 0;
+        if (degradeByMatchPhase) {
+            v |= DEGRADED_BY_MATCH_PHASE;
+        }
+        if (degradedByTimeout) {
+            v |= DEGRADED_BY_TIMEOUT;
+        }
+        if (degradedByAdaptiveTimeout) {
+            v |= DEGRADED_BY_ADAPTIVE_TIMEOUT;
+        }
+        return v;
+    }
+
+    public long getSoonActive() { return soonActive; }
+
+    public boolean isDegraded() { return (degradedReason != 0) || isDegradedByNonIdealState(); }
+    public boolean isDegradedByMatchPhase() { return (degradedReason & DEGRADED_BY_MATCH_PHASE) != 0; }
+    public boolean isDegradedByTimeout() { return (degradedReason & DEGRADED_BY_TIMEOUT) != 0; }
+    public boolean isDegradedByAdapativeTimeout() { return (degradedReason & DEGRADED_BY_ADAPTIVE_TIMEOUT) != 0; }
+    public boolean isDegradedByNonIdealState() { return (degradedReason == 0) && (getResultPercentage() != 100);}
+
+    /**
+     * An int between 0 (inclusive) and 100 (inclusive) representing how many
+     * percent coverage the result sets this Coverage instance contains information
+     * about had.
+     */
+    public int getResultPercentage() {
+        if (docs < active) {
+            return (int) Math.round(docs * 100.0d / active);
+        }
+        return 100;
+    }
+
+}
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java
index 84ee3b505b1..fed12281962 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/HitCounts.java
@@ -16,19 +16,23 @@ public class HitCounts {
     private final long totalHitCount;
     private final int requestedHits;
     private final int requestedOffset;
+    private final Coverage coverage;
 
-    public HitCounts(
-            int retrievedHits,
-            int summaryCount,
-            long totalHitCount,
-            int requestedHits,
-            int requestedOffset) {
+    HitCounts(int retrievedHits, int summaryCount, long totalHitCount, int requestedHits, int requestedOffset) {
+        this(retrievedHits, summaryCount, totalHitCount, requestedHits, requestedOffset,
+             new Coverage(1,1,1,0));
+    }
+
+    public HitCounts(int retrievedHits, int summaryCount, long totalHitCount,
+                     int requestedHits, int requestedOffset, Coverage coverage)
+    {
 
         this.retrievedHits = retrievedHits;
         this.summaryCount = summaryCount;
         this.totalHitCount = totalHitCount;
         this.requestedHits = requestedHits;
         this.requestedOffset = requestedOffset;
+        this.coverage = coverage;
     }
 
     /**
@@ -69,4 +73,6 @@ public class HitCounts {
         return requestedOffset;
     }
 
+    public Coverage getCoverage() { return coverage; }
+
 }
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
index 595bd99a759..556b97ced62 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
@@ -23,6 +23,14 @@ import java.util.logging.Logger;
  * @author frodelu
  */
 public class JSONFormatter {
+    private static final String COVERAGE = "coverage";
+    private static final String COVERAGE_COVERAGE = "coverage";
+    private static final String COVERAGE_DOCUMENTS = "documents";
+    private static final String COVERAGE_DEGRADE = "degraded";
+    private static final String COVERAGE_DEGRADE_MATCHPHASE = "match-phase";
+    private static final String COVERAGE_DEGRADE_TIMEOUT = "timeout";
+    private static final String COVERAGE_DEGRADE_ADAPTIVE_TIMEOUT = "adaptive-timeout";
+    private static final String COVERAGE_DEGRADED_NON_IDEAL_STATE = "non-ideal-state";
 
     private AccessLogEntry accessLogEntry;
     private final JsonFactory generatorFactory;
@@ -92,6 +100,25 @@ public class JSONFormatter {
                 generator.writeObjectFieldStart("search");
                 generator.writeNumberField("totalhits", getTotalHitCount(accessLogEntry.getHitCounts()));
                 generator.writeNumberField("hits", getRetrievedHitCount(accessLogEntry.getHitCounts()));
+                Coverage c = accessLogEntry.getHitCounts().getCoverage();
+                if (c != null) {
+                    generator.writeObjectFieldStart(COVERAGE);
+                    generator.writeNumberField(COVERAGE_COVERAGE, c.getResultPercentage());
+                    generator.writeNumberField(COVERAGE_DOCUMENTS, c.getDocs());
+                    if (c.isDegraded()) {
+                        generator.writeObjectFieldStart(COVERAGE_DEGRADE);
+                        if (c.isDegradedByMatchPhase())
+                            generator.writeBooleanField(COVERAGE_DEGRADE_MATCHPHASE, c.isDegradedByMatchPhase());
+                        if (c.isDegradedByTimeout())
+                            generator.writeBooleanField(COVERAGE_DEGRADE_TIMEOUT, c.isDegradedByTimeout());
+                        if (c.isDegradedByAdapativeTimeout())
+                            generator.writeBooleanField(COVERAGE_DEGRADE_ADAPTIVE_TIMEOUT, c.isDegradedByAdapativeTimeout());
+                        if (c.isDegradedByNonIdealState())
+                            generator.writeBooleanField(COVERAGE_DEGRADED_NON_IDEAL_STATE, c.isDegradedByNonIdealState());
+                        generator.writeEndObject();
+                    }
+                    generator.writeEndObject();
+                }
                 generator.writeEndObject();
             }
 
diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
index ceafbc18272..8bbb8500cfd 100644
--- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
+++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
@@ -18,6 +18,9 @@ public class JSONLogTestCase {
     private static final String EMPTY_USERAGENT = "";
 
     private AccessLogEntry newAccessLogEntry(final String query) {
+        return newAccessLogEntry(query, new Coverage(100,100,100,0));
+    }
+    private AccessLogEntry newAccessLogEntry(final String query, Coverage coverage) {
         final AccessLogEntry entry = new AccessLogEntry();
         entry.setRawQuery("query="+query);
         entry.setRawPath("");
@@ -25,7 +28,7 @@ public class JSONLogTestCase {
         entry.setHttpMethod("GET");
         entry.setHttpVersion("HTTP/1.1");
         entry.setUserAgent("Mozilla/4.05 [en] (Win95; I)");
-        entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10));
+        entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10, coverage));
         entry.setHostString("localhost");
         entry.setStatusCode(200);
         entry.setTimeStamp(920880005023L);
@@ -58,7 +61,8 @@ public class JSONLogTestCase {
             "\"localport\":0," +
             "\"search\":{" +
             "\"totalhits\":1234," +
-            "\"hits\":0" +
+            "\"hits\":0," +
+            "\"coverage\":{\"coverage\":100,\"documents\":100}" +
             "}" +
             "}";
 
@@ -87,7 +91,8 @@ public class JSONLogTestCase {
             "\"localport\":0," +
             "\"search\":{" +
             "\"totalhits\":1234," +
-            "\"hits\":0" +
+            "\"hits\":0," +
+            "\"coverage\":{\"coverage\":100,\"documents\":100}" +
             "}," +
             "\"attributes\":{" +
             "\"singlevalue\":\"value1\"," +
@@ -121,7 +126,8 @@ public class JSONLogTestCase {
             "\"remoteaddr\":\"FE80:0000:0000:0000:0202:B3FF:FE1E:8329\"," +
             "\"search\":{" +
             "\"totalhits\":1234," +
-            "\"hits\":0" +
+            "\"hits\":0," +
+            "\"coverage\":{\"coverage\":100,\"documents\":100}" +
             "}" +
             "}";
 
@@ -147,7 +153,8 @@ public class JSONLogTestCase {
             "\"remoteport\":1234," +
             "\"search\":{" +
             "\"totalhits\":1234," +
-            "\"hits\":0" +
+            "\"hits\":0," +
+            "\"coverage\":{\"coverage\":100,\"documents\":100}" +
             "}" +
             "}";
 
@@ -172,7 +179,7 @@ public class JSONLogTestCase {
         entry.setHttpMethod("GET");
         entry.setHttpVersion("HTTP/1.1");
         entry.setUserAgent("Mozilla/4.05 [en] (Win95; I; \"Best Browser Ever\")");
-        entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10));
+        entry.setHitCounts(new HitCounts(0, 10, 1234, 0, 10, new Coverage(100,200,200,0)));
         entry.setHostString("localhost");
         entry.setStatusCode(200);
         entry.setTimeStamp(920880005023L);
@@ -194,11 +201,44 @@ public class JSONLogTestCase {
             "\"localport\":0," +
             "\"search\":{" +
             "\"totalhits\":1234," +
-            "\"hits\":0" +
+            "\"hits\":0," +
+            "\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"non-ideal-state\":true}}" +
             "}" +
             "}";
 
         assertEquals(expectedOutput, new JSONFormatter(entry).format());
     }
 
+    private void verifyCoverage(String coverage, AccessLogEntry entry) {
+        assertEquals("{\"ip\":\"152.200.54.243\"," +
+                "\"time\":920880005.023," +
+                "\"duration\":0.122," +
+                "\"responsesize\":9875," +
+                "\"code\":200," +
+                "\"method\":\"GET\"," +
+                "\"uri\":\"?query=test\"," +
+                "\"version\":\"HTTP/1.1\"," +
+                "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," +
+                "\"host\":\"localhost\"," +
+                "\"scheme\":null," +
+                "\"localport\":0," +
+                "\"search\":{" +
+                "\"totalhits\":1234," +
+                "\"hits\":0," +
+                coverage +
+                "}" +
+                "}", new JSONFormatter(entry).format());
+    }
+
+    @Test
+    public void test_with_coverage_degradation() {
+        verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"non-ideal-state\":true}}",
+                       newAccessLogEntry("test",  new Coverage(100,200,200,0)));
+        verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"match-phase\":true}}",
+                       newAccessLogEntry("test",  new Coverage(100,200,200,1)));
+        verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"timeout\":true}}",
+                       newAccessLogEntry("test",  new Coverage(100,200,200,2)));
+        verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"adaptive-timeout\":true}}",
+                       newAccessLogEntry("test",  new Coverage(100,200,200,4)));
+    }
 }