diff options
author | Harald Musum <musum@yahooinc.com> | 2023-06-06 13:30:30 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-06-06 13:30:30 +0200 |
commit | 09d00888d1cb60a92ba75b4707262e08987f1bad (patch) | |
tree | 599f0fefd708120031c51921e86f55e206623fba /container-search/src/test | |
parent | d0c8f93a7b9dae9acae144857176c1dd1a375c6f (diff) |
Track failures and move file to failures path if more than max failures
Diffstat (limited to 'container-search/src/test')
-rw-r--r-- | container-search/src/test/java/com/yahoo/search/logging/SpoolerTest.java | 99 |
1 files changed, 83 insertions, 16 deletions
diff --git a/container-search/src/test/java/com/yahoo/search/logging/SpoolerTest.java b/container-search/src/test/java/com/yahoo/search/logging/SpoolerTest.java index eb5d7f85481..dc8f82e4d5a 100644 --- a/container-search/src/test/java/com/yahoo/search/logging/SpoolerTest.java +++ b/container-search/src/test/java/com/yahoo/search/logging/SpoolerTest.java @@ -27,8 +27,7 @@ public class SpoolerTest { public void testSpoolingLogger() throws IOException { Path spoolDir = tempDir.resolve("spool"); - int maxEntriesPerFile = 1; - Spooler spooler = new Spooler(spoolDir, maxEntriesPerFile, clock, true); + Spooler spooler = createSpooler(spoolDir, 1); TestLogger logger = new TestLogger(spooler); assertTrue(sendEntry(logger, "Yo entry")); @@ -53,14 +52,15 @@ public class SpoolerTest { assertReadyFiles(spooler, 0); assertSuccessFiles(spooler, 2); assertFailureFiles(spooler, 0); + + assertTrue(spooler.failures().isEmpty(), spooler.failures().toString()); } @Test public void testSpoolingManyEntriesPerFile() throws IOException { Path spoolDir = tempDir.resolve("spool"); - int maxEntriesPerFile = 2; - Spooler spooler = new Spooler(spoolDir, maxEntriesPerFile, clock, true); + Spooler spooler = createSpooler(spoolDir, 2); TestLogger logger = new TestLogger(spooler); assertTrue(sendEntry(logger, "Yo entry")); @@ -111,8 +111,8 @@ public class SpoolerTest { @Test public void failingToTransportIsRetried() throws IOException { Path spoolDir = tempDir.resolve("spool"); - Spooler spooler = new Spooler(spoolDir, 1, clock, true); - FailingToTransportSecondEntryLogger logger = new FailingToTransportSecondEntryLogger(spooler); + Spooler spooler = createSpooler(spoolDir, 1, true, 2); + FailingToTransportNthEntryLogger logger = new FailingToTransportNthEntryLogger(spooler, 2); assertTrue(sendEntry(logger, "Yo entry")); logger.manualRun(); // Success for first message @@ -121,17 +121,47 @@ public class SpoolerTest { assertTrue(sendEntry(logger, "Yo entry 2")); logger.manualRun(); // Failure for second message, so still just 1 file in successes path assertEquals(1, spooler.listFilesInPath(spooler.successesPath()).size()); + assertEquals(0, spooler.listFilesInPath(spooler.failuresPath()).size()); logger.manualRun(); // Success when retrying second message, so 2 files in successes path assertEquals(2, spooler.listFilesInPath(spooler.successesPath()).size()); } @Test + public void failingToTransportGivesUpAfterNTries() throws IOException { + Path spoolDir = tempDir.resolve("spool"); + Spooler spooler = createSpooler(spoolDir, 1, true, 2); + FailingToTransportAfterNEntriesLogger logger = new FailingToTransportAfterNEntriesLogger(spooler, 2); + + assertTrue(sendEntry(logger, "Yo entry")); + assertEquals(1, spooler.listFilesInPath(spooler.readyPath()).size()); + logger.manualRun(); // Success for first message + assertEquals(1, spooler.listFilesInPath(spooler.successesPath()).size()); + assertEquals(0, spooler.listFilesInPath(spooler.failuresPath()).size()); + + assertTrue(sendEntry(logger, "Yo entry 2")); + assertEquals(1, spooler.listFilesInPath(spooler.readyPath()).size()); + logger.manualRun(); // Failure for second message, so still just 1 file in successes path + assertEquals(1, spooler.listFilesInPath(spooler.successesPath()).size()); + assertEquals(0, spooler.listFilesInPath(spooler.failuresPath()).size()); + + logger.manualRun(); // Fails again, but should be retried + assertEquals(1, spooler.listFilesInPath(spooler.readyPath()).size()); + assertEquals(1, spooler.listFilesInPath(spooler.successesPath()).size()); + assertEquals(0, spooler.listFilesInPath(spooler.failuresPath()).size()); + + logger.manualRun(); // Fails again, should be moved to failures path + assertEquals(0, spooler.listFilesInPath(spooler.readyPath()).size()); + assertEquals(1, spooler.listFilesInPath(spooler.successesPath()).size()); + assertEquals(1, spooler.listFilesInPath(spooler.failuresPath()).size()); + } + + @Test public void noSuccessFiles() throws IOException { Path spoolDir = tempDir.resolve("spool"); boolean keepSuccessFiles = false; - Spooler spooler = new Spooler(spoolDir, 1, clock, keepSuccessFiles); - FailingToTransportSecondEntryLogger logger = new FailingToTransportSecondEntryLogger(spooler); + Spooler spooler = createSpooler(spoolDir, 1, keepSuccessFiles, 2); + FailingToTransportNthEntryLogger logger = new FailingToTransportNthEntryLogger(spooler, 2); assertTrue(sendEntry(logger, "Yo entry")); logger.manualRun(); // Success for first message @@ -163,6 +193,14 @@ public class SpoolerTest { assertTrue(content.contains(Base64.getEncoder().encodeToString(expectedContent.getBytes()))); } + private static Spooler createSpooler(Path spoolDir, int maxEntriesPerFile) { + return new Spooler(spoolDir, maxEntriesPerFile, clock, true, 1000); + } + + private static Spooler createSpooler(Path spoolDir, int maxEntriesPerFile, boolean keepSuccessFiles, int maxFailures) { + return new Spooler(spoolDir, maxEntriesPerFile, clock, keepSuccessFiles, maxFailures); + } + private static class TestLogger extends AbstractSpoolingLogger { private final List<LoggerEntry> entriesSent = new ArrayList<>(); @@ -178,9 +216,7 @@ public class SpoolerTest { } @Override - public void run() { - // Do nothing, use manualRun - } + public void run() {} // do nothing, call manualRun() to do something @Override public boolean send(LoggerEntry entry) { @@ -198,12 +234,14 @@ public class SpoolerTest { } - private static class FailingToTransportSecondEntryLogger extends AbstractSpoolingLogger { + private static class FailingToTransportNthEntryLogger extends AbstractSpoolingLogger { private int transportCount = 0; + private final int entriesToFail; - public FailingToTransportSecondEntryLogger(Spooler spooler) { + public FailingToTransportNthEntryLogger(Spooler spooler, int entriesToFail) { super(spooler); + this.entriesToFail = entriesToFail; } @Override @@ -215,14 +253,43 @@ public class SpoolerTest { @Override public boolean transport(LoggerEntry entry) { transportCount++; - return transportCount != 2; + return transportCount != entriesToFail; + } + + @Override + public void run() {} // do nothing, call manualRun() to do something + + public void manualRun() { + super.run(); + } + + } + + private static class FailingToTransportAfterNEntriesLogger extends AbstractSpoolingLogger { + + private int transportCount = 0; + private final int entriesToFailAfter; + + public FailingToTransportAfterNEntriesLogger(Spooler spooler, int entriesToFailAfter) { + super(spooler); + this.entriesToFailAfter = entriesToFailAfter; } @Override - public void run() { - // do nothing + public boolean send(LoggerEntry entry) { + spooler.write(entry); + return true; } + @Override + public boolean transport(LoggerEntry entry) { + transportCount++; + return transportCount < entriesToFailAfter; + } + + @Override + public void run() {} // do nothing, call manualRun() to do something + public void manualRun() { super.run(); } |