1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.model.api.ModelFactory;
import com.yahoo.config.model.provision.Host;
import com.yahoo.config.model.provision.Hosts;
import com.yahoo.config.model.provision.InMemoryProvisioner;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.component.Version;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.handler.VipStatus;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
import com.yahoo.container.jdisc.state.StateMonitor;
import com.yahoo.jdisc.core.SystemTimer;
import com.yahoo.path.Path;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.server.deploy.DeployTester;
import com.yahoo.vespa.config.server.rpc.RpcServer;
import com.yahoo.vespa.config.server.version.VersionState;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.function.BooleanSupplier;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
* @author Harald Musum
*/
public class ConfigServerBootstrapTest {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
public void testBootstrap() throws Exception {
ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3");
DeployTester tester = new DeployTester(configserverConfig, provisioner);
tester.deployApp("src/test/apps/hosted/");
File versionFile = temporaryFolder.newFile();
VersionState versionState = new VersionState(versionFile);
assertTrue(versionState.isUpgraded());
RpcServer rpcServer = createRpcServer(configserverConfig);
VipStatus vipStatus = new VipStatus();
// Take a host away so that there are too few for the application, to verify we can still bootstrap
provisioner.allocations().values().iterator().next().remove(0);
ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer,
versionState, createStateMonitor(), vipStatus,
ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
assertFalse(vipStatus.isInRotation());
bootstrap.start();
waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
bootstrap.deconstruct();
assertEquals(StateMonitor.Status.down, bootstrap.status());
assertFalse(rpcServer.isRunning());
assertFalse(vipStatus.isInRotation());
}
@Test
public void testBootstrapWhenRedeploymentFails() throws Exception {
ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
DeployTester tester = new DeployTester(configserverConfig);
tester.deployApp("src/test/apps/hosted/");
File versionFile = temporaryFolder.newFile();
VersionState versionState = new VersionState(versionFile);
assertTrue(versionState.isUpgraded());
// Manipulate application package so that it will fail deployment when config server starts
java.nio.file.Files.delete(Paths.get(configserverConfig.configServerDBDir())
.resolve("tenants/")
.resolve(tester.tenant().getName().value())
.resolve("sessions/2/services.xml"));
RpcServer rpcServer = createRpcServer(configserverConfig);
VipStatus vipStatus = new VipStatus();
ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
createStateMonitor(), vipStatus,
ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
assertFalse(vipStatus.isInRotation());
// Call method directly, to be sure that it is finished redeploying all applications and we can check status
bootstrap.start();
// App is invalid, bootstrapping was unsuccessful. Status should be 'initializing',
// rpc server should not be running and it should be out of rotation
assertEquals(StateMonitor.Status.initializing, bootstrap.status());
assertFalse(rpcServer.isRunning());
assertFalse(vipStatus.isInRotation());
bootstrap.deconstruct();
}
// Tests that we do not try to create the config model version stored in zookeeper when not on hosted vespa, since
// we are then only able to create the latest version
@Test
public void testBootstrapNonHostedOneConfigModel() throws Exception {
ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder);
String vespaVersion = "1.2.3";
List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString(vespaVersion)));
List<Host> hosts = createHosts(vespaVersion);
InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true);
Curator curator = new MockCurator();
DeployTester tester = new DeployTester(modelFactories, configserverConfig,
Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()),
provisioner, curator);
tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now());
ApplicationId applicationId = tester.applicationId();
File versionFile = temporaryFolder.newFile();
VersionState versionState = new VersionState(versionFile);
assertTrue(versionState.isUpgraded());
// Ugly hack, but I see no other way of doing it:
// Manipulate application version in zookeeper so that it is an older version than the model we know, which is
// the case when upgrading on non-hosted installations
curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2"));
RpcServer rpcServer = createRpcServer(configserverConfig);
VipStatus vipStatus = new VipStatus();
ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
createStateMonitor(), vipStatus,
ConfigServerBootstrap.Mode.BOOTSTRAP_IN_SEPARATE_THREAD);
waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
}
private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException {
Duration timeout = Duration.ofSeconds(60);
Instant endTime = Instant.now().plus(timeout);
while (Instant.now().isBefore(endTime)) {
if (booleanSupplier.getAsBoolean())
return;
Thread.sleep(10);
}
throw new RuntimeException(messageIfWaitingFails);
}
private MockRpc createRpcServer(ConfigserverConfig configserverConfig) throws IOException {
return new MockRpc(configserverConfig.rpcport(), temporaryFolder.newFolder());
}
private StateMonitor createStateMonitor() {
return new StateMonitor(new HealthMonitorConfig(new HealthMonitorConfig.Builder().initialStatus("initializing")),
new SystemTimer());
}
private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder) throws IOException {
return createConfigserverConfig(temporaryFolder, true);
}
private static ConfigserverConfig createConfigserverConfigNonHosted(TemporaryFolder temporaryFolder) throws IOException {
return createConfigserverConfig(temporaryFolder, false);
}
private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder, boolean hosted) throws IOException {
return new ConfigserverConfig(new ConfigserverConfig.Builder()
.configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath())
.configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())
.hostedVespa(hosted)
.multitenant(hosted)
.maxDurationOfBootstrap(2) /* seconds */
.sleepTimeWhenRedeployingFails(0)); /* seconds */
}
private List<Host> createHosts(String vespaVersion) {
return Arrays.asList(createHost("host1", vespaVersion), createHost("host2", vespaVersion), createHost("host3", vespaVersion));
}
private Host createHost(String hostname, String version) {
return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version)));
}
public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc {
volatile boolean isRunning = false;
MockRpc(int port, File tempDir) {
super(port, tempDir);
}
@Override
public void run() {
isRunning = true;
}
@Override
public void stop() {
isRunning = false;
}
@Override
public boolean isRunning() {
return isRunning;
}
}
}
|
