diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-06-14 15:35:11 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-06-14 15:35:11 +0200 |
commit | 1d0c5d6dec23623ebb25a51012826f7c9dfa94af (patch) | |
tree | 960c9b44d0cf6a42e52b8ed44c6767fd89b7ad47 /filedistribution | |
parent | ada5176c09d5383d7ce2d705b39c7fd9ebbbd39a (diff) |
Split out download and download status into its own class
Diffstat (limited to 'filedistribution')
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java | 90 | ||||
-rw-r--r-- | filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java | 93 |
2 files changed, 95 insertions, 88 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java new file mode 100644 index 00000000000..ee5602f9534 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java @@ -0,0 +1,90 @@ +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; + +import java.time.Instant; +import java.util.Collections; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +class Downloads { + private final Map<FileReference, FileReferenceDownload> downloads = new ConcurrentHashMap<>(); + + void add(FileReferenceDownload fileReferenceDownload) { + downloads.put(fileReferenceDownload.fileReference(), fileReferenceDownload); + } + + void remove(FileReference fileReference) { + downloads.remove(fileReference); + } + + Optional<FileReferenceDownload> get(FileReference fileReference) { + return Optional.ofNullable(downloads.get(fileReference)); + } + + /* Status for ongoing and completed downloads, keeps at most status for 100 last downloads */ + static class DownloadStatuses { + + private static final int maxEntries = 100; + + private final Map<FileReference, DownloadStatus> downloadStatus = new ConcurrentHashMap<>(); + + void add(FileReference fileReference) { + add(fileReference, 0.0); + } + + void add(FileReference fileReference, double progress) { + DownloadStatus ds = new DownloadStatus(fileReference); + ds.setProgress(progress); + downloadStatus.put(fileReference, ds); + if (downloadStatus.size() > maxEntries) { + Map.Entry<FileReference, DownloadStatus> oldest = + Collections.min(downloadStatus.entrySet(), Comparator.comparing(e -> e.getValue().created)); + downloadStatus.remove(oldest.getKey()); + } + } + + Optional<DownloadStatus> get(FileReference fileReference) { + return Optional.ofNullable(downloadStatus.get(fileReference)); + } + + Map<FileReference, DownloadStatus> all() { + return Map.copyOf(downloadStatus); + } + + } + + static class DownloadStatus { + private final FileReference fileReference; + private double progress; // between 0 and 1 + private final Instant created; + + DownloadStatus(FileReference fileReference) { + this.fileReference = fileReference; + this.progress = 0.0; + this.created = Instant.now(); + } + + public FileReference fileReference() { + return fileReference; + } + + public double progress() { + return progress; + } + + public void setProgress(double progress) { + this.progress = progress; + } + + public void finished() { + setProgress(1.0); + } + + public Instant created() { + return created; + } + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 0ce69c182ce..8cd13f972b7 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -12,11 +12,8 @@ import com.yahoo.vespa.config.ConnectionPool; import java.io.File; import java.time.Duration; import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -43,7 +40,7 @@ public class FileReferenceDownloader { /* Ongoing downloads */ private final Downloads downloads = new Downloads(); /* Status for ongoing and finished downloads */ - private final DownloadStatuses downloadStatuses = new DownloadStatuses(); + private final Downloads.DownloadStatuses downloadStatuses = new Downloads.DownloadStatuses(); private final Duration downloadTimeout; private final Duration sleepBetweenRetries; private final Duration rpcTimeout; @@ -98,7 +95,7 @@ public class FileReferenceDownloader { void completedDownloading(FileReference fileReference, File file) { Optional<FileReferenceDownload> download = downloads.get(fileReference); if (download.isPresent()) { - downloadStatuses.get(fileReference).ifPresent(DownloadStatus::finished); + downloadStatuses.get(fileReference).ifPresent(Downloads.DownloadStatus::finished); downloads.remove(fileReference); download.get().future().complete(Optional.of(file)); } else { @@ -157,7 +154,7 @@ public class FileReferenceDownloader { double downloadStatus(String file) { double status = 0.0; - Optional<DownloadStatus> downloadStatus = downloadStatuses.get(new FileReference(file)); + Optional<Downloads.DownloadStatus> downloadStatus = downloadStatuses.get(new FileReference(file)); if (downloadStatus.isPresent()) { status = downloadStatus.get().progress(); } @@ -165,7 +162,7 @@ public class FileReferenceDownloader { } void setDownloadStatus(FileReference fileReference, double completeness) { - Optional<DownloadStatus> downloadStatus = downloadStatuses.get(fileReference); + Optional<Downloads.DownloadStatus> downloadStatus = downloadStatuses.get(fileReference); if (downloadStatus.isPresent()) downloadStatus.get().setProgress(completeness); else @@ -173,7 +170,7 @@ public class FileReferenceDownloader { } Map<FileReference, Double> downloadStatus() { - return downloadStatuses.all().values().stream().collect(Collectors.toMap(DownloadStatus::fileReference, DownloadStatus::progress)); + return downloadStatuses.all().values().stream().collect(Collectors.toMap(Downloads.DownloadStatus::fileReference, Downloads.DownloadStatus::progress)); } public ConnectionPool connectionPool() { @@ -189,84 +186,4 @@ public class FileReferenceDownloader { } } - private static class Downloads { - private final Map<FileReference, FileReferenceDownload> downloads = new ConcurrentHashMap<>(); - - void add(FileReferenceDownload fileReferenceDownload) { - downloads.put(fileReferenceDownload.fileReference(), fileReferenceDownload); - } - - void remove(FileReference fileReference) { - downloads.remove(fileReference); - } - - Optional<FileReferenceDownload> get(FileReference fileReference) { - return Optional.ofNullable(downloads.get(fileReference)); - } - } - - private static class DownloadStatus { - private final FileReference fileReference; - private double progress; // between 0 and 1 - private final Instant created; - - DownloadStatus(FileReference fileReference) { - this.fileReference = fileReference; - this.progress = 0.0; - this.created = Instant.now(); - } - - public FileReference fileReference() { - return fileReference; - } - - public double progress() { - return progress; - } - - public void setProgress(double progress) { - this.progress = progress; - } - - public void finished() { - setProgress(1.0); - } - - public Instant created() { - return created; - } - } - - /* Status for ongoing and completed downloads, keeps at most status for 100 last downloads */ - private static class DownloadStatuses { - - private static final int maxEntries = 100; - - private final Map<FileReference, DownloadStatus> downloadStatus = new ConcurrentHashMap<>(); - - void add(FileReference fileReference) { - add(fileReference, 0.0); - } - - void add(FileReference fileReference, double progress) { - DownloadStatus ds = new DownloadStatus(fileReference); - ds.setProgress(progress); - downloadStatus.put(fileReference, ds); - if (downloadStatus.size() > maxEntries) { - Map.Entry<FileReference, DownloadStatus> oldest = - Collections.min(downloadStatus.entrySet(), Comparator.comparing(e -> e.getValue().created)); - downloadStatus.remove(oldest.getKey()); - } - } - - Optional<DownloadStatus> get(FileReference fileReference) { - return Optional.ofNullable(downloadStatus.get(fileReference)); - } - - Map<FileReference, DownloadStatus> all() { - return Map.copyOf(downloadStatus); - } - - } - } |