1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.filedistribution;
import com.yahoo.config.FileReference;
import com.yahoo.config.model.api.ConfigServerSpec;
import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.vespa.model.ConfigProxy;
import com.yahoo.vespa.model.Host;
import java.util.*;
import java.util.stream.Collectors;
/**
* Responsible for directing distribution of files to hosts.
*
* @author Tony Vaagenes
*/
public class FileDistributor {
private final FileRegistry fileRegistry;
private final List<ConfigServerSpec> configServerSpecs;
/** A map from files to the hosts to which that file should be distributed */
private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>();
/**
* Adds the given file to the associated application packages' registry of file and marks the file
* for distribution to the given hosts.
* <b>Note: This class receives ownership of the given collection.</b>
*
* @return the reference to the file, created by the application package
*/
public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) {
FileReference reference = fileRegistry.addFile(relativePath);
addToFilesToDistribute(reference, hosts);
return reference;
}
/**
* Adds the given file to the associated application packages' registry of file and marks the file
* for distribution to the given hosts.
* <b>Note: This class receives ownership of the given collection.</b>
*
* @return the reference to the file, created by the application package
*/
public FileReference sendUriToHosts(String uri, Collection<Host> hosts) {
FileReference reference = fileRegistry.addUri(uri);
if (reference != null) {
addToFilesToDistribute(reference, hosts);
}
return reference;
}
/** Same as sendFileToHost(relativePath,Collections.singletonList(host) */
public FileReference sendFileToHost(String relativePath, Host host) {
return sendFileToHosts(relativePath, Arrays.asList(host));
}
public FileReference sendUriToHost(String uri, Host host) {
return sendUriToHosts(uri, Arrays.asList(host));
}
private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) {
Set<Host> oldHosts = getHosts(reference);
oldHosts.addAll(hosts);
}
private Set<Host> getHosts(FileReference reference) {
Set<Host> hosts = filesToHosts.get(reference);
if (hosts == null) {
hosts = new HashSet<>();
filesToHosts.put(reference, hosts);
}
return hosts;
}
public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs) {
this.fileRegistry = fileRegistry;
this.configServerSpecs = configServerSpecs;
}
/** Returns the files which has been marked for distribution to the given host */
public Set<FileReference> filesToSendToHost(Host host) {
Set<FileReference> files = new HashSet<>();
for (Map.Entry<FileReference,Set<Host>> e : filesToHosts.entrySet()) {
if (e.getValue().contains(host)) {
files.add(e.getKey());
}
}
return files;
}
public Set<Host> getTargetHosts() {
Set<Host> hosts = new HashSet<>();
for (Set<Host> hostSubset: filesToHosts.values())
hosts.addAll(hostSubset);
return hosts;
}
public Set<String> getTargetHostnames() {
return getTargetHosts().stream().map(Host::getHostname).collect(Collectors.toSet());
}
/** Returns the host which is the source of the files */
public String fileSourceHost() {
return fileRegistry.fileSourceHost();
}
public Set<FileReference> allFilesToSend() {
return filesToHosts.keySet();
}
// should only be called during deploy
public void sendDeployedFiles(FileDistribution dbHandler) {
String fileSourceHost = fileSourceHost();
for (Host host : getTargetHosts()) {
if ( ! host.getHostname().equals(fileSourceHost)) {
dbHandler.sendDeployedFiles(host.getHostname(), filesToSendToHost(host));
dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host));
}
}
// Ask other config servers to download, for redundancy
if (configServerSpecs != null)
configServerSpecs.stream()
.filter(configServerSpec -> !configServerSpec.getHostName().equals(fileSourceHost))
.forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend());
dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames());
}
// should only be called during deploy, and only once, since it leads to file distributor
// rescanning all files, which is very expensive ATM (April 2016)
public void reloadDeployFileDistributor(FileDistribution dbHandler) {
dbHandler.reloadDeployFileDistributor();
}
}
|
