1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.filedistribution;
import com.google.common.collect.ImmutableMap;
import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.net.HostName;
import com.yahoo.path.Path;
import com.yahoo.text.Utf8;
import net.jpountz.xxhash.XXHashFactory;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
/**
* File registry for one application package
*
* @author Tony Vaagenes
*/
public class FileDBRegistry implements FileRegistry {
private final boolean silenceNonExistingFiles;
private final AddFileInterface manager;
private final Map<String, FileReference> fileReferenceCache = new HashMap<>();
private static final String entryDelimiter = "\t";
private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL);
public FileDBRegistry(AddFileInterface manager) {
this(manager, Map.of(), false);
}
private FileDBRegistry(AddFileInterface manager, Map<String, FileReference> knownReferences, boolean silenceNonExistingFiles) {
this.silenceNonExistingFiles = silenceNonExistingFiles;
this.manager = manager;
fileReferenceCache.putAll(knownReferences);
}
public static FileDBRegistry create(AddFileInterface manager, Reader persistedState) {
try (BufferedReader reader = new BufferedReader(persistedState)) {
String ignoredFileSourceHost = reader.readLine();
if (ignoredFileSourceHost == null)
throw new RuntimeException("No file source host");
return new FileDBRegistry(manager, decode(reader), true);
} catch (IOException e) {
throw new RuntimeException("Error while reading pre-generated file registry", e);
}
}
static Map<String, FileReference> decode(BufferedReader reader) {
Map<String, FileReference> refs = new HashMap<>();
try {
String line;
while ((line = reader.readLine()) != null) {
String[] parts = entryDelimiterPattern.split(line);
if (parts.length < 2)
throw new IllegalArgumentException("Cannot split '" + line + "' into two parts");
refs.put(parts[0], new FileReference(parts[1]));
}
} catch (IOException e) {
throw new RuntimeException("Error while reading pre-generated file registry", e);
}
return refs;
}
@Override
public synchronized FileReference addFile(String relativePath) {
if (relativePath.startsWith("/"))
throw new IllegalArgumentException(relativePath + " is not relative");
Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
return cachedReference.orElseGet(() -> {
try {
FileReference newRef = manager.addFile(Path.fromString(relativePath));
fileReferenceCache.put(relativePath, newRef);
return newRef;
} catch (FileNotFoundException e) {
if (silenceNonExistingFiles) {
return new FileReference("non-existing-file");
} else {
throw new IllegalArgumentException(e);
}
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
);
}
@Override
public synchronized FileReference addUri(String uri) {
String relativePath = uriToRelativeFile(uri);
Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
return cachedReference.orElseGet(() -> {
FileReference newRef = manager.addUri(uri, Path.fromString(relativePath));
fileReferenceCache.put(uri, newRef);
return newRef;
});
}
@Override
public synchronized FileReference addBlob(String blobName, ByteBuffer blob) {
String relativePath = blobToRelativeFile(blobName);
Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName));
return cachedReference.orElseGet(() -> {
FileReference newRef = manager.addBlob(blob, Path.fromString(relativePath));
fileReferenceCache.put(blobName, newRef);
return newRef;
});
}
@Override
public synchronized List<Entry> export() {
List<Entry> entries = new ArrayList<>();
for (Map.Entry<String, FileReference> entry : fileReferenceCache.entrySet()) {
entries.add(new Entry(entry.getKey(), entry.getValue()));
}
return entries;
}
// Used for testing only
synchronized Map<String, FileReference> getMap() {
return ImmutableMap.copyOf(fileReferenceCache);
}
public static String exportRegistry(FileRegistry registry) {
List<Entry> entries = registry.export();
StringBuilder builder = new StringBuilder();
builder.append(HostName.getLocalhost()).append('\n');
for (FileRegistry.Entry entry : entries) {
builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n');
}
return builder.toString();
}
private static String uriToRelativeFile(String uri) {
String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0);
if (uri.endsWith(".json")) {
relative += ".json";
} else if (uri.endsWith(".json.lz4")) {
relative += ".json.lz4";
} else if (uri.endsWith(".lz4")) {
relative += ".lz4";
}
return relative;
}
private static String blobToRelativeFile(String blobName) {
return "blob/" + blobName;
}
}
|
