diff options
56 files changed, 529 insertions, 350 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index b3501d2d9d3..1fd67ee0c80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ add_subdirectory(searchcore) add_subdirectory(searchcorespi) add_subdirectory(searchlib) add_subdirectory(searchsummary) +add_subdirectory(security-tools) add_subdirectory(security-utils) add_subdirectory(service-monitor) add_subdirectory(simplemetrics) diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index b5cadedd454..dea2095d8e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -49,8 +49,8 @@ public class AdminModel extends ConfigModel { public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { verifyClusterControllersOnlyDefinedForContent(configModelRepo); if (admin == null) return; - if (admin.getClusterControllers() != null) - admin.getClusterControllers().prepare(deployState); + if (admin.getClusterControllers() != null) admin.getClusterControllers().prepare(deployState); + if (admin.getMetricsProxyCluster() != null) admin.getMetricsProxyCluster().prepare(deployState); admin.getLogServerContainerCluster().ifPresent((ContainerCluster cc) -> cc.prepare(deployState)); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index ca6bdee82cf..b381a5f28db 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -42,6 +42,9 @@ import java.util.stream.Collectors; */ public class MockApplicationPackage implements ApplicationPackage { + public static final String DEPLOYED_BY_USER = "user"; + public static final String APPLICATION_NAME = "application"; + public static final long APPLICATION_GENERATION = 1L; public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); @@ -70,7 +73,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), asNamedReaderList(queryProfile)); - applicationMetaData = new ApplicationMetaData("user", "dir", 0L, false, "application", "checksum", 1L, 0L); + applicationMetaData = new ApplicationMetaData(DEPLOYED_BY_USER, "dir", 0L, false, APPLICATION_NAME, "checksum", APPLICATION_GENERATION, 0L); } /** Returns the root of this application package relative to the current dir */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index cc3da1df7dc..44627ade4ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -45,9 +45,11 @@ public class Admin extends AbstractConfigProducer implements Serializable { private final boolean isHostedVespa; private final Monitoring monitoring; + private final List<Configserver> configservers = new ArrayList<>(); + private final Metrics metrics; + private MetricsProxyContainerCluster metricsProxyCluster; private MetricSet additionalDefaultMetrics = emptyMetricSet(); - private final List<Configserver> configservers = new ArrayList<>(); private final List<Slobrok> slobroks = new ArrayList<>(); private Configserver defaultConfigserver; @@ -99,6 +101,10 @@ public class Admin extends AbstractConfigProducer implements Serializable { public Metrics getUserMetrics() { return metrics; } + public MetricsProxyContainerCluster getMetricsProxyCluster() { + return metricsProxyCluster; + } + public void setAdditionalDefaultMetrics(MetricSet additionalDefaultMetrics) { if (additionalDefaultMetrics == null) return; this.additionalDefaultMetrics = additionalDefaultMetrics; @@ -202,7 +208,8 @@ public class Admin extends AbstractConfigProducer implements Serializable { if (slobroks.isEmpty()) // TODO: Move to caller slobroks.addAll(createDefaultSlobrokSetup(deployState.getDeployLogger())); - addMetricsProxyCluster(hosts, deployState); + if (deployState.getProperties().enableMetricsProxyContainer()) + addMetricsProxyCluster(hosts, deployState); for (HostResource host : hosts) { if (!host.getHost().runsConfigServer()) { @@ -212,7 +219,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { } private void addMetricsProxyCluster(List<HostResource> hosts, DeployState deployState) { - var metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); + metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); int index = 0; for (var host : hosts) { var container = new MetricsProxyContainer(metricsProxyCluster, index++, deployState.isHosted()); diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index 074ff610e9a..dc72fb8a42b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -116,7 +116,7 @@ public class SystemModelTestCase { // all three host aliases are for the same host, so the number of services should be 3 + 8 // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver and metricsproxy) - assertEquals(10, host1.getServices().size()); + assertEquals(9, host1.getServices().size()); assertNotNull(host1.getService("simpleservice")); assertNotNull(host1.getService("simpleservice2")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index a490b7afd4c..b7ceb0923b7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -29,7 +29,6 @@ import org.junit.Test; import java.util.Set; -import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -102,12 +101,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(5)); + assertThat(sentinelConfig.service().size(), is(4)); assertThat(sentinelConfig.service(0).name(), is("logserver")); assertThat(sentinelConfig.service(1).name(), is("slobrok")); assertThat(sentinelConfig.service(2).name(), is("slobrok2")); - assertThat(sentinelConfig.service(3).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(4).name(), is("logd")); + //assertThat(sentinelConfig.service(3).name(), is(METRICS_PROXY_CONTAINER.serviceName)); + assertThat(sentinelConfig.service(3).name(), is("logd")); } /** @@ -138,11 +137,11 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(4)); + assertThat(sentinelConfig.service().size(), is(3)); assertThat(sentinelConfig.service(0).name(), is("logserver")); assertThat(sentinelConfig.service(1).name(), is("slobrok")); - assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(3).name(), is("logd")); + //assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); + assertThat(sentinelConfig.service(2).name(), is("logd")); assertThat(sentinelConfig.service(0).affinity().cpuSocket(), is(-1)); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); @@ -293,11 +292,11 @@ public class AdminTestCase { TestRoot root = new TestDriver().buildModel(state); String localhost = HostName.getLocalhost(); SentinelConfig sentinelConfig = root.getConfig(SentinelConfig.class, "hosts/" + localhost); - assertThat(sentinelConfig.service().size(), is(4)); + assertThat(sentinelConfig.service().size(), is(3)); assertThat(sentinelConfig.service(0).name(), is("logserver")); assertThat(sentinelConfig.service(1).name(), is("slobrok")); - assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); - assertThat(sentinelConfig.service(3).name(), is("logd")); + //assertThat(sentinelConfig.service(2).name(), is(METRICS_PROXY_CONTAINER.serviceName)); + assertThat(sentinelConfig.service(2).name(), is("logd")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 77f31993a7f..fc10e941091 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -75,13 +75,10 @@ public class DedicatedAdminV4Test { VespaModel model = createModel(hosts, services); assertEquals(3, model.getHosts().size()); - assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName); - assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName); - // Note: A logserver container is always added on logserver host - assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd", - METRICS_PROXY_CONTAINER.serviceName, LOGSERVER_CONTAINER.serviceName); + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd"); + assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd"); + // Note: A container is always added on logserver host + assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd", LOGSERVER_CONTAINER.serviceName); Monitoring monitoring = model.getAdmin().getMonitoring(); assertEquals("vespa.routing", monitoring.getClustername()); @@ -134,13 +131,13 @@ public class DedicatedAdminV4Test { // 4 slobroks, 2 per cluster where possible assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + QRSERVER.serviceName); assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + QRSERVER.serviceName); assertHostContainsServices(model, "hosts/myhost2", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + QRSERVER.serviceName); assertHostContainsServices(model, "hosts/myhost3", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + QRSERVER.serviceName); } @Test @@ -158,13 +155,10 @@ public class DedicatedAdminV4Test { VespaModel model = createModel(hosts, services); assertEquals(3, model.getHosts().size()); - assertHostContainsServices(model, "hosts/myhost0", "logd", "logforwarder", "slobrok", - METRICS_PROXY_CONTAINER.serviceName); - assertHostContainsServices(model, "hosts/myhost1", "logd", "logforwarder", "slobrok", - METRICS_PROXY_CONTAINER.serviceName); - // Note: A logserver container is always added on logserver host - assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver", - METRICS_PROXY_CONTAINER.serviceName, LOGSERVER_CONTAINER.serviceName); + assertHostContainsServices(model, "hosts/myhost0", "logd", "logforwarder", "slobrok"); + assertHostContainsServices(model, "hosts/myhost1", "logd", "logforwarder", "slobrok"); + // Note: A container is always added on logserver host + assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver", LOGSERVER_CONTAINER.serviceName); Set<String> configIds = model.getConfigIds(); // 1 logforwarder on each host @@ -209,9 +203,8 @@ public class DedicatedAdminV4Test { .zone(new Zone(SystemName.cd, Environment.dev, RegionName.defaultName())) .properties(new TestProperties().setHostedVespa(true))); assertEquals(1, model.getHosts().size()); - // Should create a logserver container on the same node as logserver - assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", - METRICS_PROXY_CONTAINER.serviceName , LOGSERVER_CONTAINER.serviceName); + // Should create a container on the same node as logserver + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", LOGSERVER_CONTAINER.serviceName); } private Set<String> serviceNames(VespaModel model, String hostname) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 0ba5b51bfe1..6aaae7f991a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -6,8 +6,10 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Zone; import com.yahoo.container.BundlesConfig; +import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; import com.yahoo.vespa.model.admin.monitoring.Metric; @@ -65,6 +67,17 @@ public class MetricsProxyContainerClusterTest { } @Test + public void cluster_is_prepared_so_that_application_metadata_config_is_produced() { + VespaModel model = getModel(servicesWithAdminOnly()); + var builder = new ApplicationMetadataConfig.Builder(); + model.getConfig(builder, CLUSTER_CONFIG_ID); + ApplicationMetadataConfig config = builder.build(); + assertEquals(MockApplicationPackage.APPLICATION_GENERATION, config.generation()); + assertEquals(MockApplicationPackage.APPLICATION_NAME, config.name()); + assertEquals(MockApplicationPackage.DEPLOYED_BY_USER, config.user()); + } + + @Test public void default_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly()); assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index 3fbfbf33fb3..01f868d2ad7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -58,10 +58,10 @@ public class ConfigValueChangeValidatorTest { createVespaModel(createQrStartConfigSegment(true, 2096)), createVespaModel(createQrStartConfigSegment(false, 2096)) ); - assertEquals(3, changes.size()); + assertEquals(2, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "admin/metrics/metricsproxy.0", 2); + //assertComponentsEquals(changes, "admin/metrics/0", 2); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index e7a1cbd6d8b..6a5e3bcb9d6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -241,7 +241,7 @@ public class ContentBuilderTest extends DomBuilderTest { String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "logserver", "slobrok", "storagenode", "distributor","searchnode","transactionlogserver", - CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName + CLUSTERCONTROLLER_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index a5284ac8720..5f6cec00b0c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -265,7 +265,7 @@ public class VespaModelTestCase { assertThat(hosts.size(), is(1)); //logd, config proxy, sentinel, config server, slobrok, log server HostInfo host = hosts.iterator().next(); - assertThat(host.getServices().size(), is(7)); + assertThat(host.getServices().size(), is(6)); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } diff --git a/config-provisioning/src/main/resources/configdefinitions/node-repository.def b/config-provisioning/src/main/resources/configdefinitions/node-repository.def index 8ea9265aa23..5217cba4302 100644 --- a/config-provisioning/src/main/resources/configdefinitions/node-repository.def +++ b/config-provisioning/src/main/resources/configdefinitions/node-repository.def @@ -5,4 +5,6 @@ namespace=config.provisioning # version. Example: my-docker-registry.domain.tld:8080/dist/vespa dockerImage string default="dummyImage" +serverNodeType enum {config, controller} default=config + useCuratorClientCache bool default=false diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index e850312158c..3f45e8e8f00 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -106,7 +106,7 @@ public class ClusterSearcher extends Searcher { super(id); this.fs4ResourcePool = fs4ResourcePool; - Dispatcher dispatcher = new Dispatcher(id.stringValue(), dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus, metric); + Dispatcher dispatcher = Dispatcher.create(id.stringValue(), dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus, metric); monitor = (dispatcher.searchCluster().directDispatchTarget().isPresent()) // dispatcher should decide vip status instead ? new ClusterMonitor(this, monitorConfig, Optional.empty()) diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java deleted file mode 100644 index b9af60089f8..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.fs4.mplex.Backend; -import com.yahoo.prelude.Pong; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; -import com.yahoo.search.dispatch.FillInvoker; -import com.yahoo.search.dispatch.InterleavedFillInvoker; -import com.yahoo.search.dispatch.InvokerFactory; -import com.yahoo.search.dispatch.SearchInvoker; -import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.Pinger; -import com.yahoo.search.dispatch.searchcluster.SearchCluster; -import com.yahoo.search.result.Hit; - -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Callable; - -/** - * FS4InvokerFactory constructs {@link FillInvoker} and {@link SearchInvoker} objects that communicate with - * content nodes or dispatchers over the fnet/FS4 protocol - * - * @author ollivir - */ -public class FS4InvokerFactory extends InvokerFactory { - private final FS4ResourcePool fs4ResourcePool; - private final ImmutableMap<Integer, Node> nodesByKey; - - public FS4InvokerFactory(FS4ResourcePool fs4ResourcePool, SearchCluster searchCluster) { - super(searchCluster); - this.fs4ResourcePool = fs4ResourcePool; - - ImmutableMap.Builder<Integer, Node> builder = ImmutableMap.builder(); - searchCluster.groups().values().forEach(group -> group.nodes().forEach(node -> builder.put(node.key(), node))); - this.nodesByKey = builder.build(); - } - - public SearchInvoker createDirectSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { - Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); - return new FS4SearchInvoker(searcher, query, backend.openChannel(), Optional.of(node)); - } - - @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { - Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); - if (backend.probeConnection()) { - return Optional.of(new FS4SearchInvoker(searcher, query, backend.openChannel(), Optional.of(node))); - } else { - return Optional.empty(); - } - } - - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result, Node node) { - return new FS4FillInvoker(searcher, result.getQuery(), fs4ResourcePool, node.hostname(), node.fs4port()); - } - - /** - * Create a {@link FillInvoker} for a the hits in a {@link Result}. - * - * @param searcher the searcher processing the query - * @param result the Result containing hits that need to be filled - * @return Optional containing the FillInvoker or <i>empty</i> if some hit is from an unknown content node - */ - public Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result) { - Collection<Integer> requiredNodes = requiredFillNodes(result); - - Map<Integer, FillInvoker> invokers = new HashMap<>(); - for (Integer distKey : requiredNodes) { - Node node = nodesByKey.get(distKey); - if (node == null) { - return Optional.empty(); - } - invokers.put(distKey, createFillInvoker(searcher, result, node)); - } - - if (invokers.size() == 1) { - return Optional.of(invokers.values().iterator().next()); - } else { - return Optional.of(new InterleavedFillInvoker(invokers)); - } - } - - private static Collection<Integer> requiredFillNodes(Result result) { - Set<Integer> requiredNodes = new HashSet<>(); - for (Iterator<Hit> i = result.hits().unorderedDeepIterator(); i.hasNext();) { - Hit h = i.next(); - if (h instanceof FastHit) { - FastHit hit = (FastHit) h; - requiredNodes.add(hit.getDistributionKey()); - } - } - return requiredNodes; - } - - @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - return new Pinger(node, monitor, fs4ResourcePool); - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4PingFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4PingFactory.java new file mode 100644 index 00000000000..2abaf341c58 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4PingFactory.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.prelude.Pong; +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; + +import java.util.concurrent.Callable; + +/** + * FS4PingFactory constructs {@link Pinger} objects that communicate with + * content nodes or dispatchers over the fnet/FS4 protocol + * + * @author ollivir + */ +public class FS4PingFactory implements PingFactory { + private final FS4ResourcePool fs4ResourcePool; + + public FS4PingFactory(FS4ResourcePool fs4ResourcePool) { + this.fs4ResourcePool = fs4ResourcePool; + } + + @Override + public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + return new Pinger(node, monitor, fs4ResourcePool); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 5f6d4220dd4..0c8ebd70b5e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -5,7 +5,6 @@ import com.yahoo.fs4.BasicPacket; import com.yahoo.fs4.ChannelTimeoutException; import com.yahoo.fs4.PingPacket; import com.yahoo.fs4.PongPacket; -import com.yahoo.fs4.QueryPacket; import com.yahoo.fs4.mplex.Backend; import com.yahoo.fs4.mplex.FS4Channel; import com.yahoo.fs4.mplex.InvalidChannelException; @@ -53,6 +52,7 @@ public class FastSearcher extends VespaBackEndSearcher { private final Dispatcher dispatcher; private final Backend dispatchBackend; + private final FS4ResourcePool fs4ResourcePool; /** * Creates a Fastsearcher. @@ -75,6 +75,7 @@ public class FastSearcher extends VespaBackEndSearcher { init(fs4ResourcePool.getServerId(), docSumParams, clusterParams, documentdbInfoConfig); this.dispatchBackend = dispatchBackend; this.dispatcher = dispatcher; + this.fs4ResourcePool = fs4ResourcePool; } /** @@ -213,7 +214,9 @@ public class FastSearcher extends VespaBackEndSearcher { Optional<Node> direct = getDirectNode(query); if(direct.isPresent()) { - return dispatcher.getFS4InvokerFactory().createDirectSearchInvoker(this, query, direct.get()); + var node = direct.get(); + Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); + return new FS4SearchInvoker(this, query, backend.openChannel(), direct); } return new FS4SearchInvoker(this, query, dispatchBackend.openChannel(), Optional.empty()); } @@ -232,7 +235,9 @@ public class FastSearcher extends VespaBackEndSearcher { Optional<Node> direct = getDirectNode(query); if (direct.isPresent()) { - return dispatcher.getFS4InvokerFactory().createFillInvoker(this, result, direct.get()); + var node = direct.get(); + Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); + return new FS4FillInvoker(this, query, backend); } return new FS4FillInvoker(this, query, dispatchBackend); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 4f546201e76..fb587e394d9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -4,7 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.component.AbstractComponent; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; -import com.yahoo.prelude.fastsearch.FS4InvokerFactory; +import com.yahoo.prelude.fastsearch.FS4PingFactory; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; @@ -15,6 +15,7 @@ import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; @@ -56,40 +57,33 @@ public class Dispatcher extends AbstractComponent { private final LoadBalancer loadBalancer; private final boolean multilevelDispatch; private final boolean internalDispatchByDefault; - private final boolean dispatchWithProtobuf; - private final FS4InvokerFactory fs4InvokerFactory; - private final RpcInvokerFactory rpcInvokerFactory; + private final InvokerFactory invokerFactory; private final Metric metric; private final Metric.Context metricContext; - public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, + public static Dispatcher create(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus, Metric metric) { - this(new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus), dispatchConfig, fs4ResourcePool, metric); - } + var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus); + var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, !dispatchConfig.useFdispatchByDefault()); + var pingFactory = dispatchConfig.useFdispatchByDefault()? new FS4PingFactory(fs4ResourcePool) : rpcFactory; - public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, Metric metric) { - this(searchCluster, dispatchConfig, new FS4InvokerFactory(fs4ResourcePool, searchCluster), - new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, dispatchConfig.dispatchWithProtobuf()), metric); + return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, pingFactory, metric); } - public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4InvokerFactory fs4InvokerFactory, - RpcInvokerFactory rpcInvokerFactory, Metric metric) { + public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, PingFactory pingFactory, + Metric metric) { this.searchCluster = searchCluster; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.multilevelDispatch = dispatchConfig.useMultilevelDispatch(); this.internalDispatchByDefault = !dispatchConfig.useFdispatchByDefault(); - this.dispatchWithProtobuf = dispatchConfig.dispatchWithProtobuf(); - - this.fs4InvokerFactory = fs4InvokerFactory; - this.rpcInvokerFactory = rpcInvokerFactory; - + this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); - searchCluster.startClusterMonitoring(dispatchWithProtobuf ? rpcInvokerFactory : fs4InvokerFactory); + searchCluster.startClusterMonitoring(pingFactory); } /** Returns the search cluster this dispatches to */ @@ -99,19 +93,13 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - rpcInvokerFactory.release(); + invokerFactory.release(); } public Optional<FillInvoker> getFillInvoker(Result result, VespaBackEndSearcher searcher) { - Optional<FillInvoker> rpcInvoker = rpcInvokerFactory.createFillInvoker(searcher, result); - if (rpcInvoker.isPresent()) { - return rpcInvoker; - } - if (result.getQuery().properties().getBoolean(dispatchInternal, internalDispatchByDefault)) { - Optional<FillInvoker> fs4Invoker = fs4InvokerFactory.createFillInvoker(searcher, result); - if (fs4Invoker.isPresent()) { - return fs4Invoker; - } + Optional<FillInvoker> invoker = invokerFactory.createFillInvoker(searcher, result); + if (invoker.isPresent()) { + return invoker; } return Optional.empty(); } @@ -122,13 +110,10 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } - InvokerFactory factory = query.properties().getBoolean(dispatchProtobuf, dispatchWithProtobuf) - ? rpcInvokerFactory : fs4InvokerFactory; - - Optional<SearchInvoker> invoker = getSearchPathInvoker(query, factory, searcher); + Optional<SearchInvoker> invoker = getSearchPathInvoker(query, searcher); if (!invoker.isPresent()) { - invoker = getInternalInvoker(query, factory, searcher); + invoker = getInternalInvoker(query, searcher); } if (invoker.isPresent() && query.properties().getBoolean(com.yahoo.search.query.Model.ESTIMATE)) { query.setHits(0); @@ -140,12 +125,8 @@ public class Dispatcher extends AbstractComponent { return invoker; } - public FS4InvokerFactory getFS4InvokerFactory() { - return fs4InvokerFactory; - } - // build invoker based on searchpath - private Optional<SearchInvoker> getSearchPathInvoker(Query query, InvokerFactory invokerFactory, VespaBackEndSearcher searcher) { + private Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackEndSearcher searcher) { String searchPath = query.getModel().getSearchPath(); if (searchPath == null) { return Optional.empty(); @@ -163,7 +144,7 @@ public class Dispatcher extends AbstractComponent { } } - private Optional<SearchInvoker> getInternalInvoker(Query query, InvokerFactory invokerFactory, VespaBackEndSearcher searcher) { + private Optional<SearchInvoker> getInternalInvoker(Query query, VespaBackEndSearcher searcher) { Optional<Node> directNode = searchCluster.directDispatchTarget(); if (directNode.isPresent()) { Node node = directNode.get(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 0ce1b74c02d..31af74a39b2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -1,11 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; @@ -17,7 +15,6 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; -import java.util.concurrent.Callable; /** * @author ollivir @@ -33,8 +30,6 @@ public abstract class InvokerFactory { public abstract Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result); - public abstract Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); - /** * Create a {@link SearchInvoker} for a list of content nodes. * @@ -113,4 +108,6 @@ public abstract class InvokerFactory { coverage.setNodesTried(count); return new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError(down.toString()), coverage); } + + public void release() {} } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index b0a418241f8..0cd646914a1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -13,6 +13,7 @@ import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; @@ -21,7 +22,7 @@ import java.util.concurrent.Callable; /** * @author ollivir */ -public class RpcInvokerFactory extends InvokerFactory { +public class RpcInvokerFactory extends InvokerFactory implements PingFactory { /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java new file mode 100644 index 00000000000..c9f722ef79b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -0,0 +1,10 @@ +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.prelude.Pong; +import com.yahoo.search.cluster.ClusterMonitor; + +import java.util.concurrent.Callable; + +public interface PingFactory { + Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 603932d56f9..3657e0b5c76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -10,7 +10,6 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; -import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; @@ -46,7 +45,7 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableList<Group> orderedGroups; private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; - private InvokerFactory pingFactory; + private PingFactory pingFactory; /** * A search node on this local machine having the entire corpus, which we therefore @@ -89,7 +88,7 @@ public class SearchCluster implements NodeManager<Node> { this.clusterMonitor = new ClusterMonitor<>(this); } - public void startClusterMonitoring(InvokerFactory pingFactory) { + public void startClusterMonitoring(PingFactory pingFactory) { this.pingFactory = pingFactory; for (var group : orderedGroups) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index e78fd920adc..92ccd7e46b2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -2,9 +2,8 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.fastsearch.FS4InvokerFactory; +import com.yahoo.prelude.fastsearch.FS4PingFactory; import com.yahoo.prelude.fastsearch.FS4ResourcePool; -import com.yahoo.search.Result; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; @@ -31,8 +30,8 @@ class MockDispatcher extends Dispatcher { private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, RpcResourcePool rpcResourcePool) { - super(searchCluster, dispatchConfig, new FS4InvokerFactory(fs4ResourcePool, searchCluster), - new RpcInvokerFactory(rpcResourcePool, searchCluster, dispatchConfig.dispatchWithProtobuf()), new MockMetric()); + super(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster, !dispatchConfig.useFdispatchByDefault()), + new FS4PingFactory(fs4ResourcePool), new MockMetric()); } private static DispatchConfig toDispatchConfig(List<Node> nodes) { @@ -49,8 +48,4 @@ class MockDispatcher extends Dispatcher { } return new DispatchConfig(dispatchConfigBuilder); } - - public void fill(Result result, String summaryClass) { - } - } diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index a35cbdf289e..84c10991293 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -196,6 +196,7 @@ public class ClusteredConnectionTestCase { public SimpleQuery(int hashValue) { this.hashValue = hashValue; + this.setTimeout(50); } @Override diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 25aed879a48..03a417c9bbb 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -1,13 +1,15 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.prelude.fastsearch.FS4InvokerFactory; +import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.prelude.fastsearch.test.MockMetric; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.Test; @@ -15,10 +17,12 @@ import org.junit.Test; import java.util.List; import java.util.Optional; import java.util.OptionalInt; +import java.util.concurrent.Callable; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; /** * @author ollivir @@ -39,7 +43,9 @@ public class DispatcherTest { builder.useMultilevelDispatch(true); DispatchConfig dc = new DispatchConfig(builder); - Dispatcher disp = new Dispatcher(cl, dc, new MockFS4InvokerFactory(cl), new MockRpcInvokerFactory(), new MockMetric()); + var invokerFactory = new MockInvokerFactory(cl); + + Dispatcher disp = new Dispatcher(cl, dc, invokerFactory, invokerFactory, new MockMetric()); assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false)); } @@ -48,12 +54,12 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 2); Query q = query(); q.getModel().setSearchPath("1/0"); // second node in first group - MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (nodes, a) -> { + MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (nodes, a) -> { assertThat(nodes.size(), is(1)); assertThat(nodes.get(0).key(), is(2)); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(q, null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -67,8 +73,8 @@ public class DispatcherTest { return Optional.of(new Node(1, "test", 123, 1)); } }; - MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> true); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); + MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -78,14 +84,14 @@ public class DispatcherTest { public void requireThatInvokerConstructionIsRetriedAndLastAcceptsAnyCoverage() { SearchCluster cl = new MockSearchCluster("1", 2, 1); - MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, acceptIncompleteCoverage) -> { + MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, acceptIncompleteCoverage) -> { assertThat(acceptIncompleteCoverage, is(false)); return false; }, (n, acceptIncompleteCoverage) -> { assertThat(acceptIncompleteCoverage, is(true)); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -95,8 +101,8 @@ public class DispatcherTest { public void requireThatInvokerConstructionDoesNotRepeatGroups() { SearchCluster cl = new MockSearchCluster("1", 2, 1); - MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, null, new MockMetric()); + MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(false)); invokerFactory.verifyAllEventsProcessed(); @@ -106,12 +112,12 @@ public class DispatcherTest { public boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage); } - private static class MockFS4InvokerFactory extends FS4InvokerFactory { + private static class MockInvokerFactory extends InvokerFactory implements PingFactory { private final FactoryStep[] events; private int step = 0; - public MockFS4InvokerFactory(SearchCluster cl, FactoryStep... events) { - super(null, cl); + public MockInvokerFactory(SearchCluster cl, FactoryStep... events) { + super(cl); this.events = events; } @@ -133,15 +139,23 @@ public class DispatcherTest { void verifyAllEventsProcessed() { assertThat(step, is(events.length)); } - } - public class MockRpcInvokerFactory extends RpcInvokerFactory { - public MockRpcInvokerFactory() { - super(null, null, true); + @Override + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { + fail("Unexpected call to createNodeSearchInvoker"); + return null; + } + + @Override + public Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result) { + fail("Unexpected call to createFillInvoker"); + return null; } @Override - public void release() { + public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + fail("Unexpected call to createPinger"); + return null; } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index bedfbc5bdc1..2af89cc1191 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -97,7 +97,8 @@ public class NodeRepository extends AbstractComponent { */ @Inject public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) { - this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), + NodeType.valueOf(config.serverNodeType().name()), config.useCuratorClientCache()); } /** @@ -105,14 +106,14 @@ public class NodeRepository extends AbstractComponent { * which will be used for time-sensitive decisions. */ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver, - DockerImage dockerImage, boolean useCuratorClientCache) { + DockerImage dockerImage, NodeType serverNodeType, boolean useCuratorClientCache) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; this.flavors = flavors; this.nameResolver = nameResolver; this.osVersions = new OsVersions(db); - this.infrastructureVersions = new InfrastructureVersions(db); + this.infrastructureVersions = new InfrastructureVersions(db, serverNodeType); this.firmwareChecks = new FirmwareChecks(db, clock); this.dockerImages = new DockerImages(db, dockerImage); this.jobControl = new JobControl(db); @@ -128,7 +129,7 @@ public class NodeRepository extends AbstractComponent { /** Returns the Docker image to use for nodes in this */ public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); } - /** @return The name resolver used to resolve hostname and ip addresses */ + /** Returns the name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } /** Returns the OS versions to use for nodes in this */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java index 7e2e729149b..2cfe27ee01d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -23,25 +24,30 @@ public class InfrastructureVersions { private static final Logger logger = Logger.getLogger(InfrastructureVersions.class.getName()); private final CuratorDatabaseClient db; + private final NodeType serverNodeType; + private final Version defaultVersion; - public InfrastructureVersions(CuratorDatabaseClient db) { + public InfrastructureVersions(CuratorDatabaseClient db, NodeType serverNodeType) { + this(db, serverNodeType, Vtag.currentVersion); + } + + InfrastructureVersions(CuratorDatabaseClient db, NodeType serverNodeType, Version defaultVersion) { this.db = db; + this.serverNodeType = serverNodeType; + this.defaultVersion = defaultVersion; } public void setTargetVersion(NodeType nodeType, Version newTargetVersion, boolean force) { - switch (nodeType) { - case config: - case confighost: - case proxyhost: - case controller: - case controllerhost: - break; - default: - throw new IllegalArgumentException("Cannot set version for type " + nodeType); - } - if (newTargetVersion.isEmpty()) { + if (!isNodeTypeAllowedForTargetVersion(nodeType)) + throw new IllegalArgumentException("Cannot set version for type " + nodeType); + + if (!isNodeTypeAllowedForThisServer(nodeType)) + throw new IllegalArgumentException( + "Cannot set version for " + nodeType + " on a " + serverNodeType.description()); + + if (newTargetVersion.isEmpty()) throw new IllegalArgumentException("Invalid target version: " + newTargetVersion.toFullString()); - } + try (Lock lock = db.lockInfrastructureVersions()) { Map<NodeType, Version> infrastructureVersions = db.readInfrastructureVersions(); @@ -64,11 +70,44 @@ public class InfrastructureVersions { } } - public Optional<Version> getTargetVersionFor(NodeType nodeType) { - return Optional.ofNullable(db.readInfrastructureVersions().get(nodeType)); + /** + * @return the target version for a given nodeType. If the nodeType has no target version set, current version + * of this server will be set and returned + * @throws IllegalArgumentException if the given nodeType is not allowed to have a target version on this server + */ + public Version getTargetVersionFor(NodeType nodeType) { + if (!isNodeTypeAllowedForTargetVersion(nodeType) || !isNodeTypeAllowedForThisServer(nodeType)) + throw new IllegalArgumentException(nodeType + " has no target version"); + + return Optional.ofNullable(db.readInfrastructureVersions().get(nodeType)).orElseGet(() -> { + // Target version has never been set for this node type, set it to the default version of this server. + // We need to set the version (in ZK) to prevent another config server from returning a different version. + // No lock needed since this is only an issue in bootstrap case and in a potential race, all versions + // are equally valid + setTargetVersion(nodeType, defaultVersion, false); + return defaultVersion; + }); } + /** @return all set target versions from db */ public Map<NodeType, Version> getTargetVersions() { return Collections.unmodifiableMap(db.readInfrastructureVersions()); } + + private boolean isNodeTypeAllowedForThisServer(NodeType nodeType) { + return nodeType == NodeType.proxyhost || + nodeType == serverNodeType || (nodeType.isDockerHost() && nodeType.childNodeType() == serverNodeType); + } + + private static boolean isNodeTypeAllowedForTargetVersion(NodeType nodeType) { + switch (nodeType) { + case config: + case confighost: + case proxyhost: + case controller: + case controllerhost: + return true; + } + return false; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index f2b90b93abb..15863e78039 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -89,7 +89,7 @@ public class CapacityPolicies { if (zone.system() == SystemName.cd && zone.environment().isTest()) new NodeResources(4, 4, 50); - return new NodeResources(2, 8, 50); + return new NodeResources(1.5, 8, 50); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index d002d81cc2c..73aadb93f2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -81,13 +81,6 @@ public class InfraDeployerImpl implements InfraDeployer { try (Mutex lock = nodeRepository.lock(application.getApplicationId())) { NodeType nodeType = application.getCapacity().type(); - Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); - if (targetVersion.isEmpty()) { - logger.log(LogLevel.DEBUG, "No target version set for " + nodeType + ", removing application"); - removeApplication(application.getApplicationId()); - return; - } - candidateNodes = nodeRepository .getNodes(nodeType, Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive); if (candidateNodes.isEmpty()) { @@ -96,10 +89,11 @@ public class InfraDeployerImpl implements InfraDeployer { return; } - if (!allActiveNodesOn(targetVersion.get(), candidateNodes)) { + Version targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); + if (!allActiveNodesOn(targetVersion, candidateNodes)) { hostSpecs = provisioner.prepare( application.getApplicationId(), - application.getClusterSpecWithVersion(targetVersion.get()), + application.getClusterSpecWithVersion(targetVersion), application.getCapacity(), 1, // groups logger::log); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 9edf368fa9c..de00f38ea5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -52,6 +52,7 @@ public class MockNodeRepository extends NodeRepository { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); this.flavors = flavors; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 9b9e7397323..6a2e983a995 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -27,9 +27,12 @@ public class NodeRepositoryTester { private final NodeRepository nodeRepository; private final Clock clock; private final MockCurator curator; - - + public NodeRepositoryTester() { + this(NodeType.config); + } + + public NodeRepositoryTester(NodeType nodeType) { nodeFlavors = new NodeFlavors(createConfig()); clock = new ManualClock(); curator = new MockCurator(); @@ -37,6 +40,7 @@ public class NodeRepositoryTester { nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + nodeType, true); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 6fa5afd0c20..f8ad43f90fd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -274,6 +274,7 @@ public class FailedExpirerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), + NodeType.config, true); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java index a70bd323b63..eb0d36475de 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java @@ -132,7 +132,8 @@ public class HostProvisionMaintainerTest { private final ManualClock clock = new ManualClock(); private final NodeRepository nodeRepository = new NodeRepository( - nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); + nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-image"), NodeType.config, true); Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { Node node = createNode(hostname, parentHostname, nodeType, state, application); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java index 4ee3ade351a..a28991452b6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import org.junit.Test; -import java.util.HashMap; import java.util.Map; -import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -19,9 +17,10 @@ import static org.junit.Assert.fail; */ public class InfrastructureVersionsTest { + private final Version defaultVersion = Version.fromString("6.13.37"); private final NodeRepositoryTester tester = new NodeRepositoryTester(); private final InfrastructureVersions infrastructureVersions = - new InfrastructureVersions(tester.nodeRepository().database()); + new InfrastructureVersions(tester.nodeRepository().database(), NodeType.config, defaultVersion); private final Version version = Version.fromString("6.123.456"); @@ -29,23 +28,21 @@ public class InfrastructureVersionsTest { public void can_only_downgrade_with_force() { assertTrue(infrastructureVersions.getTargetVersions().isEmpty()); - assertEquals(Optional.empty(), infrastructureVersions.getTargetVersionFor(NodeType.config)); + assertEquals(defaultVersion, infrastructureVersions.getTargetVersionFor(NodeType.config)); infrastructureVersions.setTargetVersion(NodeType.config, version, false); - assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + assertEquals(version, infrastructureVersions.getTargetVersionFor(NodeType.config)); // Upgrading to new version without force is fine - Version new_version = Version.fromString("6.123.457"); // version + 1 - infrastructureVersions.setTargetVersion(NodeType.config, new_version, false); - assertEquals(Optional.of(new_version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + Version newVersion = Version.fromString("6.123.457"); // version + 1 + infrastructureVersions.setTargetVersion(NodeType.config, newVersion, false); + assertEquals(newVersion, infrastructureVersions.getTargetVersionFor(NodeType.config)); // Downgrading to old version without force fails - try { - infrastructureVersions.setTargetVersion(NodeType.config, version, false); - fail("Should not be able to downgrade without force"); - } catch (IllegalArgumentException ignored) { } + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.config, version, false)); infrastructureVersions.setTargetVersion(NodeType.config, version, true); - assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + assertEquals(version, infrastructureVersions.getTargetVersionFor(NodeType.config)); } @Test @@ -53,35 +50,61 @@ public class InfrastructureVersionsTest { // We can set version for config infrastructureVersions.setTargetVersion(NodeType.config, version, false); - try { - infrastructureVersions.setTargetVersion(NodeType.tenant, version, false); - fail("Should not be able to set version for tenant nodes"); - } catch (IllegalArgumentException ignored) { } + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.tenant, version, false)); - try { - // Using 'force' does not help, force only applies to version downgrade - infrastructureVersions.setTargetVersion(NodeType.tenant, version, true); - fail("Should not be able to set version for tenant nodes"); - } catch (IllegalArgumentException ignored) { } + // Using 'force' does not help, force only applies to version downgrade + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.tenant, version, true)); } @Test - public void can_store_multiple_versions() { - Version version2 = Version.fromString("6.456.123"); - + public void store_all_valid_for_config() { infrastructureVersions.setTargetVersion(NodeType.config, version, false); - infrastructureVersions.setTargetVersion(NodeType.confighost, version2, false); + infrastructureVersions.setTargetVersion(NodeType.confighost, version, false); infrastructureVersions.setTargetVersion(NodeType.proxyhost, version, false); + + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.controller, version, false)); + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.controllerhost, version, false)); + + Map<NodeType, Version> expected = Map.of( + NodeType.config, version, + NodeType.confighost, version, + NodeType.proxyhost, version); + + assertEquals(expected, infrastructureVersions.getTargetVersions()); + } + + @Test + public void store_all_valid_for_controller() { + InfrastructureVersions infrastructureVersions = + new InfrastructureVersions(tester.nodeRepository().database(), NodeType.controller, defaultVersion); + infrastructureVersions.setTargetVersion(NodeType.controller, version, false); - infrastructureVersions.setTargetVersion(NodeType.controllerhost, version2, false); + infrastructureVersions.setTargetVersion(NodeType.controllerhost, version, false); + infrastructureVersions.setTargetVersion(NodeType.proxyhost, version, false); - Map<NodeType, Version> expected = new HashMap<>(); - expected.put(NodeType.config, version); - expected.put(NodeType.confighost, version2); - expected.put(NodeType.proxyhost, version); - expected.put(NodeType.controller, version); - expected.put(NodeType.controllerhost, version2); + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.config, version, false)); + assertThrows(IllegalArgumentException.class, + () -> infrastructureVersions.setTargetVersion(NodeType.confighost, version, false)); + + Map<NodeType, Version> expected = Map.of( + NodeType.controller, version, + NodeType.controllerhost, version, + NodeType.proxyhost, version); assertEquals(expected, infrastructureVersions.getTargetVersions()); } + + private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { + try { + runnable.run(); + fail("Expected " + clazz); + } catch (Throwable e) { + if (!clazz.isInstance(e)) throw e; + } + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 798df66f6cd..efb6dffb23b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -36,6 +36,7 @@ public class MaintenanceTester { public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index e1a0f478b86..435c005143e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -54,6 +54,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(Collections.singletonList(node)); @@ -115,6 +116,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); // Allow 4 containers diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 9e57ae6dcd6..60ba388f78c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -88,7 +88,7 @@ public class NodeFailTester { clock = new ManualClock(); curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), NodeType.config, true); provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java index 832c2fc512b..b4375f16c0b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java @@ -76,7 +76,7 @@ public class NodeRetirerTester { NodeRetirerTester(NodeFlavors nodeFlavors) { Curator curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), NodeType.config, true); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); deployer = new MockDeployer(provisioner, clock, apps); flavors = nodeFlavors.getFlavors().stream().sorted(Comparator.comparing(Flavor::name)).collect(Collectors.toList()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index cd136360b0d..707692a69e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -62,6 +62,7 @@ public class OperatorChangeApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 49a14dc3777..467f1864810 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -68,6 +68,7 @@ public class PeriodicApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 491f506c3eb..75d4aefbe94 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -48,6 +48,7 @@ public class ReservationExpirerTest { NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), + NodeType.config, true); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index e37a139700f..feee3387931 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -65,7 +65,7 @@ public class RetiredExpirerTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), NodeType.config, true); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 079574b7206..664430a2de8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -379,6 +379,26 @@ public class DynamicDockerAllocationTest { NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed()); } + @Test + public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 140)), NodeType.host, 10, true); + deployZoneApp(tester); + + ApplicationId application = tester.makeApplicationId(); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); + + List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.fromNodeCount(2, Optional.of("d-2-8-50"), false, true), 1); + tester.activate(application, hosts1); + + NodeResources resources = new NodeResources(1.5, 8, 50); + System.out.println("Redeploying with " + resources); + List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.fromCount(2, resources), 1); + tester.activate(application, hosts2); + + assertEquals(hosts1, hosts2); + } + private ApplicationId makeApplicationId(String tenant, String appName) { return ApplicationId.from(tenant, appName, "default"); } @@ -420,6 +440,7 @@ public class DynamicDockerAllocationTest { b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2-8-50", 2, 8, 50, Flavor.Type.DOCKER_CONTAINER); return b.build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 44bdec75f51..ede9c7cb378 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -51,10 +50,9 @@ public class InfraDeployerImplTest { ); } - private final NodeRepositoryTester tester = new NodeRepositoryTester(); + private final NodeRepositoryTester tester; + private final InfrastructureVersions infrastructureVersions; private final Provisioner provisioner = mock(Provisioner.class); - private final NodeRepository nodeRepository = tester.nodeRepository(); - private final InfrastructureVersions infrastructureVersions = nodeRepository.infrastructureVersions(); private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class); private final InfraDeployerImpl infraDeployer; @@ -71,16 +69,9 @@ public class InfraDeployerImplTest { when(duperModelInfraApi.getSupportedInfraApplications()).thenReturn(List.of(application)); this.application = application; this.nodeType = application.getCapacity().type(); - this.infraDeployer = new InfraDeployerImpl(nodeRepository, provisioner, duperModelInfraApi); - } - - @Test - public void remove_application_if_without_target_version() { - addNode(1, Node.State.active, Optional.of(target)); - when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true); - infraDeployer.getDeployment(application.getApplicationId()).orElseThrow().activate(); - verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId()); - verifyRemoved(1); + this.tester = new NodeRepositoryTester(nodeType); + this.infrastructureVersions = tester.nodeRepository().infrastructureVersions(); + this.infraDeployer = new InfraDeployerImpl(tester.nodeRepository(), provisioner, duperModelInfraApi); } @Test @@ -239,7 +230,7 @@ public class InfraDeployerImplTest { Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false); return node.with(allocation); }); - return nodeRepository.database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty()); + return tester.nodeRepository().database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty()); } }
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c8051c3bdee..020eddbc948 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -87,7 +87,7 @@ public class ProvisioningTester { this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver, - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), NodeType.config, true); this.orchestrator = orchestrator; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, provisionServiceProvider, flagSource); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 3fbfc7c2487..6510cb570d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -636,15 +636,10 @@ public class RestApiTest { Utf8.toBytes("{\"version\": \"6.123.456\"}"), Request.Method.PATCH), "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/controller", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.123.456 for nodes of type controller\"}"); - // Verify versions are set assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); // Setting empty version fails assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -660,6 +655,13 @@ public class RestApiTest { 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set version for type tenant\"}"); + // Setting version for controller on a config server fails + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/controller", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set version for controller on a Config server\"}"); + // Omitting version field fails assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{}"), @@ -683,7 +685,7 @@ public class RestApiTest { // Verify version has been updated assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{},\"dockerImages\":{}}"); // Upgrade OS for confighost and host assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -697,7 +699,7 @@ public class RestApiTest { // OS versions are set assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"},\"dockerImages\":{}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"},\"dockerImages\":{}}"); // Upgrade OS and Vespa together assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -743,7 +745,7 @@ public class RestApiTest { "{\"message\":\"Set docker image to my-repo.my-domain.example:1234/repo/image for nodes of type config\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); // Cannot set docker image for non docker node type assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -111,6 +111,7 @@ <module>searchcore</module> <module>searchlib</module> <module>searchsummary</module> + <module>security-tools</module> <module>security-utils</module> <module>serviceview</module> <module>service-monitor</module> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java index 237c3a1d0b1..047d9d761ce 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java @@ -107,7 +107,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable { @Override public final double getDouble(int index) { double value = doubleValues()[index]; - if (value == Double.NaN) + if (Double.isNaN(value)) throw new UnsupportedOperationException("Value at " + index + " has no double representation"); return value; } @@ -119,7 +119,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable { public ArrayContext clone() { ArrayContext clone = (ArrayContext)super.clone(); clone.values = new Value[nameToIndex().size()]; - Arrays.fill(values, constantZero); + Arrays.fill(clone.values, constantZero); return clone; } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java index 3ad5c0f7316..4254bec2348 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java @@ -22,44 +22,33 @@ import static org.junit.Assert.assertEquals; public class ContextReuseTestCase { private String contextString = - "CONCEPTTYPE = 0.0\n" + - "REGEXTYPE = 0.0\n" + - "POS_18 = 0.0\n" + - "POS_19 = 0.0\n" + "ORDER_IN_CLUSTER = 2.0\n" + "GOOD_SYNTAX = 1.0\n" + - "POS_20 = 0.0\n" + - "POS_11 = 0.0\n" + - "POS_10 = 0.0\n" + - "CHUNKTYPE = 0.0\n" + - "POS_13 = 0.0\n" + - "STOP_WORD_1 = 0.0\n" + - "TERM_CASE_2 = 0.0\n" + - "TERM_CASE_3 = 0.0\n" + - "STOP_WORD_3 = 0.0\n" + - "POS_15 = 0.0\n" + - "TERM_CASE_1 = 0.0\n" + - "STOP_WORD_2 = 0.0\n" + - "POS_1 = 0.0\n" + "TERM_CASE_4 = 1.0\n" + - "LENGTH = 6.0\n" + - "EXTENDEDTYPE = 0.0\n" + - "ENTITYPLACETYPE = 0.0\n"; + "LENGTH = 6.0\n"; + + private static final double delta = 0.00000001; @Test public void testIt() throws ParseException, IOException { // Prepare - RankingExpression expression=new RankingExpression(IOUtils.readFile(new File("src/test/files/s-expression.vre"))); - ArrayContext contextPrototype=new ArrayContext(expression); - new ExpressionOptimizer().optimize(expression,contextPrototype); + RankingExpression expression = new RankingExpression(IOUtils.readFile(new File("src/test/files/s-expression.vre"))); + ArrayContext contextPrototype = new ArrayContext(expression); + new ExpressionOptimizer().optimize(expression, contextPrototype); + + assertExecution(expression, contextPrototype); + assertExecution(expression, contextPrototype); // reuse + } - // Execute - ArrayContext context=contextPrototype.clone(); + private void assertExecution(RankingExpression expression, ArrayContext contextPrototype) { + ArrayContext context = contextPrototype.clone(); for (String contextValueString : contextString.split("\n")) { String[] contextValueParts = contextValueString.split("="); context.put(contextValueParts[0].trim(), Double.valueOf(contextValueParts[1].trim())); } - assertEquals(-2.3450294999999994, expression.evaluate(context).asDouble(), 0.000000000001); + assertEquals("Context values not set are initialized to 0 doubles", + 0.0, context.get("CHUNKTYPE").asDouble(), delta); + assertEquals(-2.3450294999999994, expression.evaluate(context).asDouble(), delta); } } diff --git a/security-tools/CMakeLists.txt b/security-tools/CMakeLists.txt new file mode 100644 index 00000000000..234bb7f7497 --- /dev/null +++ b/security-tools/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(security-tools) + +vespa_install_script(src/main/sh/vespa-security-env vespa-security-env bin) + diff --git a/security-tools/OWNERS b/security-tools/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/security-tools/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/security-tools/README.md b/security-tools/README.md new file mode 100644 index 00000000000..077edd04a58 --- /dev/null +++ b/security-tools/README.md @@ -0,0 +1,3 @@ +# security-tools + +Contains the "vespa-security-env" command line tool for Vespa.
\ No newline at end of file diff --git a/security-tools/pom.xml b/security-tools/pom.xml new file mode 100644 index 00000000000..aa07e96d628 --- /dev/null +++ b/security-tools/pom.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>security-tools</artifactId> + <packaging>jar</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <!-- compile scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + + <!-- test scope --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <finalName>${project.artifactId}-jar-with-dependencies</finalName> + <filters> + <filter> + <!-- Don't include signature files from bouncycastle in uber jar. --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java new file mode 100644 index 00000000000..f57575b406a --- /dev/null +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java @@ -0,0 +1,11 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.security.tool.securityenv; + +/** + * @author bjorncs + */ +public class Main { + public static void main(String[] args) { + System.out.println("TODO implementation"); + } +} diff --git a/security-tools/src/main/sh/vespa-security-env b/security-tools/src/main/sh/vespa-security-env new file mode 100644 index 00000000000..63d71920f99 --- /dev/null +++ b/security-tools/src/main/sh/vespa-security-env @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findhost () { + if [ "${VESPA_HOSTNAME}" = "" ]; then + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 + fi + export VESPA_HOSTNAME +} + +findroot +findhost + +# END environment bootstrap section + +exec java -Djava.awt.headless=true -cp ${VESPA_HOME}/lib/jars/security-tools-jar-with-dependencies.jar com.yahoo.vespa.security.tool.securityenv.Main "$@"
\ No newline at end of file diff --git a/security-utils/README.md b/security-utils/README.md new file mode 100644 index 00000000000..241d2669ddf --- /dev/null +++ b/security-utils/README.md @@ -0,0 +1,3 @@ +# security-utils + +Contains various security utility classes for Java.
\ No newline at end of file diff --git a/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java index e5e56f7a181..a63ca28c793 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/MutableX509KeyManager.java @@ -85,12 +85,14 @@ public class MutableX509KeyManager extends X509ExtendedKeyManager { @Override public X509Certificate[] getCertificateChain(String alias) { + if (alias == null) return null; // this method can be called with 'null' alias prior to any alias getter methods. return getThreadLocalManager() .getCertificateChain(alias); } @Override public PrivateKey getPrivateKey(String alias) { + if (alias == null) return null; // this method can be called with 'null' alias prior to any alias getter methods. return getThreadLocalManager() .getPrivateKey(alias); } |