diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-07-28 16:02:28 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2020-07-28 16:02:28 +0200 |
commit | 5d1171de674bf0bc3cc5a6171feeda87a2379fd0 (patch) | |
tree | d879791b6a1a9c684ff4f479d1710a8d88ae07f8 | |
parent | 42c483f2e3cb9d001e79e7f4846d60cc3c0fb1bc (diff) |
Use new file distribution mocks for config server
* Use new mocks without the hardcoded hacks in the old ones
* Let HostSystem avoid using hack for localhost for new file distribution mocks
9 files changed, 122 insertions, 65 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 3d2918e0ee1..0192e9d42c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.net.HostName; import java.net.UnknownHostException; import java.util.LinkedHashMap; @@ -31,7 +32,7 @@ import static java.util.logging.Level.FINE; */ public class HostSystem extends AbstractConfigProducer<Host> { - private static Logger log = Logger.getLogger(HostSystem.class.getName()); + private static final Logger log = Logger.getLogger(HostSystem.class.getName()); private final Map<String, HostResource> hostname2host = new LinkedHashMap<>(); private final HostProvisioner provisioner; @@ -65,15 +66,19 @@ public class HostSystem extends AbstractConfigProducer<Host> { * @return the host with the given hostname, or null if no such host */ public HostResource getHostByHostname(String name) { - // TODO: please eliminate the following ugly hack - if ("localhost.fortestingpurposesonly".equals(name)) { - String localhost = "localhost"; - if ( ! getChildren().containsKey(localhost)) { - new Host(this, localhost); + String localhost = "localhost"; + HostResource hostResource = hostname2host.get(name); + if (hostResource == null) { + // Create a new HostResource if this is the host this code is running on (as it is when running tests) + // TODO: please eliminate the ugly hack using "localhost.fortestingpurposesonly" + if (HostName.getLocalhost().equals(name) || "localhost.fortestingpurposesonly".equals(name)) { + if (! getChildren().containsKey(localhost)) { + new Host(this, localhost); + } + hostResource = new HostResource(getChildren().get(localhost)); } - return new HostResource(getChildren().get(localhost)); } - return hostname2host.get(name); + return hostResource; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java index 1cfe30270c3..ae18c3e6e95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java @@ -16,7 +16,7 @@ import java.io.File; @SuppressWarnings("WeakerAccess") public class FileDistributionFactory { - private final ConfigserverConfig configserverConfig; + protected final ConfigserverConfig configserverConfig; private final Supervisor supervisor = new Supervisor(new Transport()); @Inject diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java deleted file mode 100644 index db70a51b2b4..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.config.model.api.FileDistribution; -import com.yahoo.config.model.application.provider.MockFileRegistry; - -import java.io.File; - -/** - * @author Ulf Lilleengen - */ -public class MockFileDistributionProvider extends FileDistributionProvider { - public int timesCalled = 0; - - public MockFileDistributionProvider(File fileReferencesDir) { - super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir)); - } - - public FileDistribution getFileDistribution() { - timesCalled++; - return super.getFileDistribution(); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index e20fef0909c..f74378b32a9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -121,14 +121,15 @@ public class ApplicationRepositoryTest { public void setup(FlagSource flagSource) throws IOException { Curator curator = new MockCurator(); configCurator = ConfigCurator.create(curator); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .flagSource(flagSource) .clock(clock) .build(); @@ -142,7 +143,11 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, - clock); + configserverConfig, + new MockLogRetriever(), + clock, + new MockTesterClient(), + new NullMetric()); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java index 78729156b93..af55dc6a90e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java @@ -10,16 +10,13 @@ import java.io.File; */ public class MockFileDistributionFactory extends FileDistributionFactory { - public final MockFileDistributionProvider mockFileDistributionProvider; - public MockFileDistributionFactory(ConfigserverConfig configserverConfig) { super(configserverConfig); - mockFileDistributionProvider = new MockFileDistributionProvider(new File(configserverConfig.fileReferencesDir())); } @Override public com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider createProvider(File applicationFile) { - return mockFileDistributionProvider; + return new MockFileDistributionProvider(applicationFile, new File(configserverConfig.fileReferencesDir())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java new file mode 100644 index 00000000000..45e00e2ece8 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.model.api.FileDistribution; + +import java.io.File; + +/** + * @author hmusum + */ +public class MockFileDistributionProvider extends FileDistributionProvider { + + public MockFileDistributionProvider(File applicationDir, File fileReferencesDir) { + super(new MockFileRegistry(applicationDir, fileReferencesDir.toPath()), + new MockFileDistribution(fileReferencesDir)); + } + + public FileDistribution getFileDistribution() { + return super.getFileDistribution(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java new file mode 100644 index 00000000000..343e0c50520 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.net.HostName; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * A file registry for config server tests + * + * @author hmusum + */ +public class MockFileRegistry implements FileRegistry { + + private final List<Entry> entries = new ArrayList<>(); + private final AddFileInterface addFileInterface; + + public MockFileRegistry(File applicationDir, Path rootPath) { + FileDirectory fileDirectory = new FileDirectory(rootPath.toFile()); + this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory); + } + + public FileReference addFile(String relativePath) { + if (relativePath.isEmpty()) + relativePath = "./"; + addFileInterface.addFile(relativePath); + + FileReference fileReference = new FileReference(relativePath); + entries.add(new Entry(relativePath, fileReference)); + return fileReference; + } + + @Override + public String fileSourceHost() { return HostName.getLocalhost(); } + + public List<Entry> export() { return entries; } + + @Override + public FileReference addUri(String uri) { + throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 72bed784195..7d1554c3e19 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; @@ -33,7 +34,6 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudgetTest; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.model.TestModelFactory; @@ -59,7 +59,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -68,6 +67,7 @@ import java.util.Set; import java.util.logging.Level; import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.APPLICATION_PACKAGE_REFERENCE_PATH; +import static com.yahoo.vespa.config.server.session.SessionPreparer.PrepareResult; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -85,27 +85,31 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); - private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + private final KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private final X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; private ConfigCurator configCurator; private SessionPreparer preparer; private TestComponentRegistry componentRegistry; - private MockFileDistributionFactory fileDistributionFactory; - private MockSecretStore secretStore = new MockSecretStore(); + private final MockSecretStore secretStore = new MockSecretStore(); @Rule public TemporaryFolder folder = new TemporaryFolder(); @Before - public void setUp() { + public void setUp() throws IOException { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); - componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - fileDistributionFactory = (MockFileDistributionFactory)componentRegistry.getFileDistributionFactory(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .fileReferencesDir(folder.newFolder().getAbsolutePath()) + .configServerDBDir(folder.newFolder().getAbsolutePath()) + .configDefinitionsDir(folder.newFolder().getAbsolutePath()) + .build()) + .build(); preparer = createPreparer(); } @@ -115,7 +119,7 @@ public class SessionPreparerTest { private SessionPreparer createPreparer(HostProvisionerProvider hostProvisionerProvider) { ModelFactoryRegistry modelFactoryRegistry = - new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(version123), new TestModelFactory(version321))); + new ModelFactoryRegistry(List.of(new TestModelFactory(version123), new TestModelFactory(version321))); return createPreparer(modelFactoryRegistry, hostProvisionerProvider); } @@ -152,14 +156,13 @@ public class SessionPreparerTest { @Test public void require_that_filedistribution_is_ignored_on_dryrun() throws IOException { - prepare(testApp, new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build()); - assertThat(fileDistributionFactory.mockFileDistributionProvider.timesCalled, is(0)); + PrepareResult result = prepare(testApp, new PrepareParams.Builder().dryRun(true).build()); + assertTrue(result.getFileRegistries().get(version321).export().isEmpty()); } @Test public void require_that_application_is_prepared() throws Exception { prepare(testApp); - assertThat(fileDistributionFactory.mockFileDistributionProvider.timesCalled, is(1)); // Only builds the newest version assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } @@ -327,11 +330,11 @@ public class SessionPreparerTest { prepare(app, new PrepareParams.Builder().build()); } - private void prepare(File app, PrepareParams params) throws IOException { + private PrepareResult prepare(File app, PrepareParams params) throws IOException { FilesApplicationPackage applicationPackage = getApplicationPackage(app); - preparer.prepare(new HostRegistry<>(), getLogger(), params, - Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), - applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); + return preparer.prepare(new HostRegistry<>(), getLogger(), params, + Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), + applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); } private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java index 1dcd63e8d9f..7f38083797e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java @@ -30,9 +30,9 @@ public class SessionTest { @Override public PrepareResult prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, - Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, - SessionZooKeeperClient sessionZooKeeperClient) { + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { isPrepared = true; return new PrepareResult(AllocatedHosts.withHosts(Set.of()), List.of()); } |