1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
|
package com.yahoo.vespa.config.proxy.filedistribution;
import com.yahoo.config.FileReference;
import com.yahoo.io.IOUtils;
import com.yahoo.jrt.Int32Value;
import com.yahoo.jrt.Request;
import com.yahoo.jrt.RequestWaiter;
import com.yahoo.jrt.StringValue;
import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Transport;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.Connection;
import com.yahoo.vespa.config.ConnectionPool;
import net.jpountz.xxhash.XXHash64;
import net.jpountz.xxhash.XXHashFactory;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class FileDownloaderTest {
private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
private MockConnection connection;
private FileDownloader fileDownloader;
private File downloadDir;
@Before
public void setup() {
try {
downloadDir = Files.createTempDirectory("filedistribution").toFile();
connection = new MockConnection();
fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000));
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
public void getFile() throws IOException {
File downloadDir = fileDownloader.downloadDirectory();
{
// fileReference already exists on disk, does not have to be downloaded
String fileReferenceString = "foo";
String filename = "foo.jar";
FileReference fileReference = new FileReference(fileReferenceString);
File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
writeFileReference(downloadDir, fileReferenceString, filename);
// Check that we get correct path and content when asking for file reference
Optional<File> pathToFile = fileDownloader.getFile(fileReference);
assertTrue(pathToFile.isPresent());
String downloadedFile = new File(fileReferenceFullPath, filename).getAbsolutePath();
assertEquals(new File(fileReferenceFullPath, filename).getAbsolutePath(), downloadedFile);
assertEquals("content", IOUtils.readFile(pathToFile.get()));
// Verify download status when downloaded
assertDownloadStatus(fileDownloader, fileReference, 100.0);
}
{
// fileReference does not exist on disk, needs to be downloaded, but fails when asking upstream for file)
connection.setResponseHandler(new MockConnection.UnknownFileReferenceResponseHandler());
FileReference fileReference = new FileReference("bar");
File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
// Verify download status when unable to download
assertDownloadStatus(fileDownloader, fileReference, 0.0);
}
{
// fileReference does not exist on disk, needs to be downloaded)
FileReference fileReference = new FileReference("fileReference");
File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent());
// Verify download status
assertDownloadStatus(fileDownloader, fileReference, 0.0);
// Receives fileReference, should return and make it available to caller
String filename = "abc.jar";
receiveFile(fileReference, filename, "some other content");
Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
assertTrue(downloadedFile.isPresent());
File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath());
assertEquals("some other content", IOUtils.readFile(downloadedFile.get()));
// Verify download status when downloaded
assertDownloadStatus(fileDownloader, fileReference, 100.0);
}
}
@Test
public void setFilesToDownload() throws IOException {
Duration timeout = Duration.ofMillis(200);
File downloadDir = Files.createTempDirectory("filedistribution").toFile();
MockConnection connectionPool = new MockConnection();
connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000))));
FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, timeout);
FileReference foo = new FileReference("foo");
FileReference bar = new FileReference("bar");
List<FileReference> fileReferences = Arrays.asList(foo, bar);
fileDownloader.queueForDownload(fileReferences);
// Verify download status
assertDownloadStatus(fileDownloader, foo, 0.0);
assertDownloadStatus(fileDownloader, bar, 0.0);
}
@Test
public void receiveFile() throws IOException {
FileReference foo = new FileReference("foo");
String filename = "foo.jar";
receiveFile(foo, filename, "content");
File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename);
assertEquals("content", IOUtils.readFile(downloadedFile));
}
private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException {
File file = new File(new File(dir, fileReferenceString), fileName);
IOUtils.writeFile(file, "content", false);
}
private File fileReferenceFullPath(File dir, FileReference fileReference) {
return new File(dir, fileReference.value());
}
private void assertDownloadStatus(FileDownloader fileDownloader, FileReference fileReference, double expectedDownloadStatus) {
double downloadStatus = fileDownloader.downloadStatus(fileReference);
assertEquals(expectedDownloadStatus, downloadStatus, 0.0001);
}
private void receiveFile(FileReference fileReference, String filename, String content) {
byte[] contentBytes = Utf8.toBytes(content);
long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0);
fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent);
}
private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
private ResponseHandler responseHandler;
MockConnection() {
this(new FileReferenceFoundResponseHandler());
}
MockConnection(ResponseHandler responseHandler) {
this.responseHandler = responseHandler;
}
@Override
public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) {
responseHandler.request(request);
}
@Override
public void invokeSync(Request request, double jrtTimeout) {
responseHandler.request(request);
}
@Override
public void setError(int errorCode) {
}
@Override
public void setSuccess() {
}
@Override
public String getAddress() {
return null;
}
@Override
public void close() {
}
@Override
public void setError(Connection connection, int errorCode) {
connection.setError(errorCode);
}
@Override
public Connection getCurrent() {
return this;
}
@Override
public Connection setNewCurrentConnection() {
return this;
}
@Override
public int getSize() {
return 1;
}
@Override
public Supervisor getSupervisor() {
return new Supervisor(new Transport());
}
void setResponseHandler(ResponseHandler responseHandler) {
this.responseHandler = responseHandler;
}
public interface ResponseHandler {
void request(Request request);
}
static class FileReferenceFoundResponseHandler implements MockConnection.ResponseHandler {
@Override
public void request(Request request) {
if (request.methodName().equals("filedistribution.serveFile")) {
request.returnValues().add(new Int32Value(0));
request.returnValues().add(new StringValue("OK"));
}
}
}
static class UnknownFileReferenceResponseHandler implements MockConnection.ResponseHandler {
@Override
public void request(Request request) {
if (request.methodName().equals("filedistribution.serveFile")) {
request.returnValues().add(new Int32Value(1));
request.returnValues().add(new StringValue("Internal error"));
}
}
}
static class WaitResponseHandler implements MockConnection.ResponseHandler {
private final Duration waitUntilAnswering;
WaitResponseHandler(Duration waitUntilAnswering) {
super();
this.waitUntilAnswering = waitUntilAnswering;
}
@Override
public void request(Request request) {
try { Thread.sleep(waitUntilAnswering.toMillis());} catch (InterruptedException e) { /* do nothing */ }
if (request.methodName().equals("filedistribution.serveFile")) {
request.returnValues().add(new Int32Value(0));
request.returnValues().add(new StringValue("OK"));
}
}
}
}
}
|