summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-06-14 15:35:11 +0200
committerHarald Musum <musum@verizonmedia.com>2021-06-14 15:35:11 +0200
commit1d0c5d6dec23623ebb25a51012826f7c9dfa94af (patch)
tree960c9b44d0cf6a42e52b8ed44c6767fd89b7ad47
parentada5176c09d5383d7ce2d705b39c7fd9ebbbd39a (diff)
Split out download and download status into its own class
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java90
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java93
2 files changed, 95 insertions, 88 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java
new file mode 100644
index 00000000000..ee5602f9534
--- /dev/null
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java
@@ -0,0 +1,90 @@
+package com.yahoo.vespa.filedistribution;
+
+import com.yahoo.config.FileReference;
+
+import java.time.Instant;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+class Downloads {
+ private final Map<FileReference, FileReferenceDownload> downloads = new ConcurrentHashMap<>();
+
+ void add(FileReferenceDownload fileReferenceDownload) {
+ downloads.put(fileReferenceDownload.fileReference(), fileReferenceDownload);
+ }
+
+ void remove(FileReference fileReference) {
+ downloads.remove(fileReference);
+ }
+
+ Optional<FileReferenceDownload> get(FileReference fileReference) {
+ return Optional.ofNullable(downloads.get(fileReference));
+ }
+
+ /* Status for ongoing and completed downloads, keeps at most status for 100 last downloads */
+ static class DownloadStatuses {
+
+ private static final int maxEntries = 100;
+
+ private final Map<FileReference, DownloadStatus> downloadStatus = new ConcurrentHashMap<>();
+
+ void add(FileReference fileReference) {
+ add(fileReference, 0.0);
+ }
+
+ void add(FileReference fileReference, double progress) {
+ DownloadStatus ds = new DownloadStatus(fileReference);
+ ds.setProgress(progress);
+ downloadStatus.put(fileReference, ds);
+ if (downloadStatus.size() > maxEntries) {
+ Map.Entry<FileReference, DownloadStatus> oldest =
+ Collections.min(downloadStatus.entrySet(), Comparator.comparing(e -> e.getValue().created));
+ downloadStatus.remove(oldest.getKey());
+ }
+ }
+
+ Optional<DownloadStatus> get(FileReference fileReference) {
+ return Optional.ofNullable(downloadStatus.get(fileReference));
+ }
+
+ Map<FileReference, DownloadStatus> all() {
+ return Map.copyOf(downloadStatus);
+ }
+
+ }
+
+ static class DownloadStatus {
+ private final FileReference fileReference;
+ private double progress; // between 0 and 1
+ private final Instant created;
+
+ DownloadStatus(FileReference fileReference) {
+ this.fileReference = fileReference;
+ this.progress = 0.0;
+ this.created = Instant.now();
+ }
+
+ public FileReference fileReference() {
+ return fileReference;
+ }
+
+ public double progress() {
+ return progress;
+ }
+
+ public void setProgress(double progress) {
+ this.progress = progress;
+ }
+
+ public void finished() {
+ setProgress(1.0);
+ }
+
+ public Instant created() {
+ return created;
+ }
+ }
+}
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
index 0ce69c182ce..8cd13f972b7 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -12,11 +12,8 @@ import com.yahoo.vespa.config.ConnectionPool;
import java.io.File;
import java.time.Duration;
import java.time.Instant;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.Map;
import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -43,7 +40,7 @@ public class FileReferenceDownloader {
/* Ongoing downloads */
private final Downloads downloads = new Downloads();
/* Status for ongoing and finished downloads */
- private final DownloadStatuses downloadStatuses = new DownloadStatuses();
+ private final Downloads.DownloadStatuses downloadStatuses = new Downloads.DownloadStatuses();
private final Duration downloadTimeout;
private final Duration sleepBetweenRetries;
private final Duration rpcTimeout;
@@ -98,7 +95,7 @@ public class FileReferenceDownloader {
void completedDownloading(FileReference fileReference, File file) {
Optional<FileReferenceDownload> download = downloads.get(fileReference);
if (download.isPresent()) {
- downloadStatuses.get(fileReference).ifPresent(DownloadStatus::finished);
+ downloadStatuses.get(fileReference).ifPresent(Downloads.DownloadStatus::finished);
downloads.remove(fileReference);
download.get().future().complete(Optional.of(file));
} else {
@@ -157,7 +154,7 @@ public class FileReferenceDownloader {
double downloadStatus(String file) {
double status = 0.0;
- Optional<DownloadStatus> downloadStatus = downloadStatuses.get(new FileReference(file));
+ Optional<Downloads.DownloadStatus> downloadStatus = downloadStatuses.get(new FileReference(file));
if (downloadStatus.isPresent()) {
status = downloadStatus.get().progress();
}
@@ -165,7 +162,7 @@ public class FileReferenceDownloader {
}
void setDownloadStatus(FileReference fileReference, double completeness) {
- Optional<DownloadStatus> downloadStatus = downloadStatuses.get(fileReference);
+ Optional<Downloads.DownloadStatus> downloadStatus = downloadStatuses.get(fileReference);
if (downloadStatus.isPresent())
downloadStatus.get().setProgress(completeness);
else
@@ -173,7 +170,7 @@ public class FileReferenceDownloader {
}
Map<FileReference, Double> downloadStatus() {
- return downloadStatuses.all().values().stream().collect(Collectors.toMap(DownloadStatus::fileReference, DownloadStatus::progress));
+ return downloadStatuses.all().values().stream().collect(Collectors.toMap(Downloads.DownloadStatus::fileReference, Downloads.DownloadStatus::progress));
}
public ConnectionPool connectionPool() {
@@ -189,84 +186,4 @@ public class FileReferenceDownloader {
}
}
- private static class Downloads {
- private final Map<FileReference, FileReferenceDownload> downloads = new ConcurrentHashMap<>();
-
- void add(FileReferenceDownload fileReferenceDownload) {
- downloads.put(fileReferenceDownload.fileReference(), fileReferenceDownload);
- }
-
- void remove(FileReference fileReference) {
- downloads.remove(fileReference);
- }
-
- Optional<FileReferenceDownload> get(FileReference fileReference) {
- return Optional.ofNullable(downloads.get(fileReference));
- }
- }
-
- private static class DownloadStatus {
- private final FileReference fileReference;
- private double progress; // between 0 and 1
- private final Instant created;
-
- DownloadStatus(FileReference fileReference) {
- this.fileReference = fileReference;
- this.progress = 0.0;
- this.created = Instant.now();
- }
-
- public FileReference fileReference() {
- return fileReference;
- }
-
- public double progress() {
- return progress;
- }
-
- public void setProgress(double progress) {
- this.progress = progress;
- }
-
- public void finished() {
- setProgress(1.0);
- }
-
- public Instant created() {
- return created;
- }
- }
-
- /* Status for ongoing and completed downloads, keeps at most status for 100 last downloads */
- private static class DownloadStatuses {
-
- private static final int maxEntries = 100;
-
- private final Map<FileReference, DownloadStatus> downloadStatus = new ConcurrentHashMap<>();
-
- void add(FileReference fileReference) {
- add(fileReference, 0.0);
- }
-
- void add(FileReference fileReference, double progress) {
- DownloadStatus ds = new DownloadStatus(fileReference);
- ds.setProgress(progress);
- downloadStatus.put(fileReference, ds);
- if (downloadStatus.size() > maxEntries) {
- Map.Entry<FileReference, DownloadStatus> oldest =
- Collections.min(downloadStatus.entrySet(), Comparator.comparing(e -> e.getValue().created));
- downloadStatus.remove(oldest.getKey());
- }
- }
-
- Optional<DownloadStatus> get(FileReference fileReference) {
- return Optional.ofNullable(downloadStatus.get(fileReference));
- }
-
- Map<FileReference, DownloadStatus> all() {
- return Map.copyOf(downloadStatus);
- }
-
- }
-
}