diff options
52 files changed, 668 insertions, 218 deletions
diff --git a/README.md b/README.md index c07b8d5e45a..1ade07c40ad 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,6 @@ Vespa is an engine for low-latency computation over large data sets. It stores and indexes your data and executes distributed queries including evaluation of machine-learned models over many data points in real time. -Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. - Travis-CI build status: [](https://travis-ci.org/vespa-engine/vespa) ## Table of contents @@ -18,6 +16,7 @@ Travis-CI build status: [ - [Contribute](#contribute) - [Building](#building) +- [License](#license) ## Background @@ -95,3 +94,7 @@ Replace `<source-dir>` with the directory in which you've cloned/unpacked the so ### Create RPM packages sh dist.sh VERSION && rpmbuild -ba ~/rpmbuild/SPECS/vespa-VERSION.spec + +## License + +Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index 88140873b7b..64ced48edd3 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -20,6 +20,7 @@ import com.yahoo.search.rendering.Renderer; import com.yahoo.text.StringUtilities; import com.yahoo.text.Utf8; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; import java.io.File; @@ -315,8 +316,9 @@ public final class Application implements AutoCloseable { break; } catch (Error e) { // the container thinks this is really serious, in this case is it not in the cause is a BindException // catch bind error and reset container - if (e.getCause() != null && e.getCause().getCause() != null && e.getCause().getCause() instanceof BindException) { - exception = (Exception) e.getCause().getCause(); + Optional<BindException> bindException = Exceptions.findCause(e, BindException.class); + if (bindException.isPresent()) { + exception = bindException.get(); com.yahoo.container.Container.resetInstance(); // this is needed to be able to recreate the container from config again } else { throw new Exception(e.getCause()); diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index af9eba92a98..aede68a1dd1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -19,6 +19,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -212,6 +213,14 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override + public void getConfig(AllClustersBucketSpacesConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + for (ContentCluster cluster : model.getContentClusters().values()) { + builder.cluster(cluster.getName(), cluster.clusterBucketSpaceConfigBuilder()); + } + } + + @Override public void getConfig(ModelConfig.Builder builder) { builder.vespaVersion(vespaVersion.toFullString()); for (HostResource modelHost : getHostSystem().getHosts()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index 2a30a89dd8a..9dcd94bf455 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -12,6 +12,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; /** @@ -29,6 +30,7 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, ZookeepersConfig.Producer, LoadTypeConfig.Producer, ClusterListConfig.Producer, + AllClustersBucketSpacesConfig.Producer, ModelConfig.Producer, ApplicationIdConfig.Producer { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 27da41c2bfa..1ec56fe529a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.documentmodel.NewDocumentType; @@ -736,13 +737,26 @@ public class ContentCluster extends AbstractConfigProducer implements private static final String DEFAULT_BUCKET_SPACE = "default"; private static final String GLOBAL_BUCKET_SPACE = "global"; + private String bucketSpaceOfDocumentType(NewDocumentType docType) { + return (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + } + + public AllClustersBucketSpacesConfig.Cluster.Builder clusterBucketSpaceConfigBuilder() { + AllClustersBucketSpacesConfig.Cluster.Builder builder = new AllClustersBucketSpacesConfig.Cluster.Builder(); + for (NewDocumentType docType : getDocumentDefinitions().values()) { + AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder typeBuilder = new AllClustersBucketSpacesConfig.Cluster.DocumentType.Builder(); + typeBuilder.bucketSpace(bucketSpaceOfDocumentType(docType)); + builder.documentType(docType.getName(), typeBuilder); + } + return builder; + } + @Override public void getConfig(BucketspacesConfig.Builder builder) { for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); - docTypeBuilder.bucketspace(bucketSpace); + docTypeBuilder.bucketspace(bucketSpaceOfDocumentType(docType)); builder.documenttype(docTypeBuilder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 53456c627a4..a003e0122a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.metrics.MetricsmanagerConfig; @@ -29,6 +30,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -823,6 +825,63 @@ public class ContentClusterTest extends ContentBaseTest { new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } + private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, + List<String> defaultSpaceTypes, List<String> globalSpaceTypes) { + AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); + assertNotNull(cluster); + assertEquals(defaultSpaceTypes.size() + globalSpaceTypes.size(), cluster.documentType().size()); + assertClusterHasTypesInBucketSpace(cluster, "default", defaultSpaceTypes); + assertClusterHasTypesInBucketSpace(cluster, "global", globalSpaceTypes); + } + + private void assertClusterHasTypesInBucketSpace(AllClustersBucketSpacesConfig.Cluster cluster, + String bucketSpace, List<String> expectedTypes) { + for (String type : expectedTypes) { + assertNotNull(cluster.documentType(type)); + assertEquals(bucketSpace, cluster.documentType(type).bucketSpace()); + } + } + + @Test + public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + String xml = + "<services>" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\"/>" + + "</admin>" + + "<content version=\"1.0\" id=\"foocluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"bunnies\" mode=\"index\"/>" + + " <document type=\"hares\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "<content version=\"1.0\" id=\"barcluster\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "</services>"; + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + + AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); + model.getConfig(builder, "client"); + AllClustersBucketSpacesConfig config = builder.build(); + + assertEquals(2, config.cluster().size()); + + assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); + assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); + } + private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 83b4cfebca5..98fa179b219 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -19,6 +20,7 @@ import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClus import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -168,6 +170,19 @@ public class ContentSearchClusterTest { } @Test + public void bucket_space_config_builder_returns_correct_mappings() throws Exception { + ContentCluster cluster = createClusterWithGlobalType(); + BucketspacesConfig expected = getBucketspacesConfig(cluster); + AllClustersBucketSpacesConfig.Cluster actual = cluster.clusterBucketSpaceConfigBuilder().build(); + assertEquals(2, expected.documenttype().size()); + assertEquals(expected.documenttype().size(), actual.documentType().size()); + assertNotNull(actual.documentType("global")); + assertEquals("global", actual.documentType().get("global").bucketSpace()); + assertNotNull(actual.documentType("regular")); + assertEquals("default", actual.documentType().get("regular").bucketSpace()); + } + + @Test public void cluster_with_global_document_types_sets_cluster_controller_global_docs_config_option() throws Exception { ContentCluster cluster = createClusterWithGlobalType(); assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 0a7d4ef4381..120c403eb74 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -72,5 +72,7 @@ vespa_generate_config(configdefinitions zookeepers.def) install_config_definition(zookeepers.def cloud.config.zookeepers.def) vespa_generate_config(configdefinitions bucketspaces.def) install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) +vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) +install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.all-clusters-bucket-spaces.def) vespa_generate_config(configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) diff --git a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def new file mode 100644 index 00000000000..7676bc2a03f --- /dev/null +++ b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## This config contains the document types handled by all content clusters +## and the bucket spaces they belong to. + +## The bucket space a document type in a particular cluster belongs to. +cluster{}.documentType{}.bucketSpace string + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 6d97b64c441..1bb0cf3fb10 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -127,13 +127,15 @@ public class ConfigServerApiImpl implements ConfigServerApi { if (!e.isRetryable()) throw e; lastException = e; } catch (Exception e) { + lastException = e; + if (configServers.size() == 1) break; + // Failure to communicate with a config server is not abnormal during upgrades if (e.getMessage().contains("(Connection refused)")) { NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); } else { NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); } - lastException = e; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 910c94c54b1..89ab2e60b63 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -265,9 +265,6 @@ public class DockerOperationsImpl implements DockerOperations { return docker.getAllContainersManagedBy(MANAGER_NAME); } - /** - * Returns map of directories to mount and whether they should be writable by everyone - */ private static void addMounts(NodeAgentContext context, Docker.CreateContainerCommand command) { final Path varLibSia = Paths.get("/var/lib/sia"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index 8a23b1d18b1..9a35188a372 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.yolean.Exceptions; -import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.util.Optional; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 67fcfc69393..e2b19e73fa3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -28,7 +28,6 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; import static com.yahoo.yolean.Exceptions.uncheck; -import static com.yahoo.yolean.Exceptions.uncheckAndIgnore; /** * Thin wrapper around java.nio.file.Path, especially nice for UNIX-specific features. diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java index 3a60061344f..09f7d6117f9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java @@ -18,7 +18,7 @@ public class FileDeleterTest { private final TaskContext context = mock(TaskContext.class); @Test - public void deleteExisting() throws IOException { + public void deleteExisting() { assertFalse(deleter.converge(context)); path.createParents().writeUtf8File("bar"); assertTrue(deleter.converge(context)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a3c7759779a..54f5403ca73 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -312,7 +312,7 @@ public class ProvisioningTester { nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION; + ConfigServerApplication application = new ConfigServerApplication(); List<HostSpec> hosts = prepare( application.getApplicationId(), application.getClusterSpecWithVersion(configServersVersion), diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java index f93f25a4eb3..88c2c0d4469 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java @@ -3,76 +3,30 @@ package com.yahoo.vespa.service.duper; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.PortInfo; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; -import com.yahoo.vespa.service.model.ModelGenerator; -import com.yahoo.vespa.service.health.ApplicationHealthMonitor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.stream.Collectors; /** * A service/application model of the config server with health status. */ -public class ConfigServerApplication extends InfraApplication { +public class ConfigServerApplication extends ConfigServerLikeApplication { public static final ConfigServerApplication CONFIG_SERVER_APPLICATION = new ConfigServerApplication(); - public static final TenantId TENANT_ID = new TenantId(CONFIG_SERVER_APPLICATION.getApplicationId().tenant().value()); - public static final ApplicationInstanceId APPLICATION_INSTANCE_ID = - new ApplicationInstanceId(CONFIG_SERVER_APPLICATION.getApplicationId().application().value()); - public static final ClusterId CLUSTER_ID = new ClusterId(CONFIG_SERVER_APPLICATION.getClusterId().value()); - public static final ServiceType SERVICE_TYPE = new ServiceType("configserver"); - public static final String CONFIG_ID_PREFIX = "configid."; - - public static ConfigId configIdFrom(int index) { - return new ConfigId(CONFIG_ID_PREFIX + index); - } public ConfigServerApplication() { - super("zone-config-servers", NodeType.config, - ClusterSpec.Type.admin, ClusterSpec.Id.from("zone-config-servers"), ServiceType.CONFIG_SERVER); + super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ServiceType.CONFIG_SERVER); } public ApplicationInfo makeApplicationInfoFromConfig(ConfigserverConfig config) { - List<HostInfo> hostInfos = new ArrayList<>(); - List<ConfigserverConfig.Zookeeperserver> zooKeeperServers = config.zookeeperserver(); - for (int index = 0; index < zooKeeperServers.size(); ++index) { - String hostname = zooKeeperServers.get(index).hostname(); - hostInfos.add(makeHostInfo(hostname, config.httpport(), index)); - } - - return new ApplicationInfo( - CONFIG_SERVER_APPLICATION.getApplicationId(), - 0, - new HostsModel(hostInfos)); - } - - private static HostInfo makeHostInfo(String hostname, int port, int configIndex) { - PortInfo portInfo = new PortInfo(port, ApplicationHealthMonitor.PORT_TAGS_HEALTH); - - Map<String, String> properties = new HashMap<>(); - properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, CLUSTER_ID.s()); - - ServiceInfo serviceInfo = new ServiceInfo( - // service name == service type for the first service of each type on each host - SERVICE_TYPE.s(), - SERVICE_TYPE.s(), - Collections.singletonList(portInfo), - properties, - configIdFrom(configIndex).s(), - hostname); - - return new HostInfo(hostname, Collections.singletonList(serviceInfo)); + List<HostName> hostnames = config.zookeeperserver().stream() + .map(ConfigserverConfig.Zookeeperserver::hostname) + .map(HostName::from) + .collect(Collectors.toList()); + return makeApplicationInfo(hostnames); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java index f29cc89e987..4273bccbf45 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java @@ -1,14 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; -public class ConfigServerHostApplication extends InfraApplication { +public class ConfigServerHostApplication extends HostAdminApplication { public ConfigServerHostApplication() { - super("configserver-host", NodeType.confighost, - ClusterSpec.Type.container, ClusterSpec.Id.from("configserver-host"), - ServiceType.HOST_ADMIN); + super("configserver-host", NodeType.confighost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java new file mode 100644 index 00000000000..ac5f13e2394 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.applicationmodel.ServiceType; + +/** + * Base class for config server and controller infrastructure applications. + * + * @author hakonhall + */ +public abstract class ConfigServerLikeApplication extends InfraApplication { + protected ConfigServerLikeApplication(String applicationName, NodeType nodeType, ClusterSpec.Type clusterType, ServiceType serviceType) { + super(applicationName, nodeType, clusterType, ClusterSpec.Id.from(applicationName), serviceType, 19071); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java index 1fb01ce0897..3d69b1c9925 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java @@ -8,10 +8,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType; /** * @author mpolden */ -public class ControllerApplication extends InfraApplication { +public class ControllerApplication extends ConfigServerLikeApplication { public ControllerApplication() { - super("controller", NodeType.controller, ClusterSpec.Type.container, - ClusterSpec.Id.from("controller"), ServiceType.CONTROLLER); + super("controller", NodeType.controller, ClusterSpec.Type.container, ServiceType.CONTROLLER); } - } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java index b8e8ead5b7e..7a0c952780d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java @@ -1,16 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; /** * @author mpolden */ -public class ControllerHostApplication extends InfraApplication { +public class ControllerHostApplication extends HostAdminApplication { public ControllerHostApplication() { - super("controller-host", NodeType.controllerhost, ClusterSpec.Type.container, - ClusterSpec.Id.from("controller-host"), ServiceType.HOST_ADMIN); + super("controller-host", NodeType.controllerhost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index ed99d2e3166..e0e2b64bdae 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -32,13 +32,14 @@ public class DuperModelManager implements DuperModelInfraApi { private static Logger logger = Logger.getLogger(DuperModelManager.class.getName()); // Infrastructure applications + private final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication(); + private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + private final ControllerApplication controllerApplication = new ControllerApplication(); + private final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); + // this must be static to be referenced in this(). Remove static once legacy config server from config is gone. private static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); - private static final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication(); - private static final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); - private static final ControllerApplication controllerApplication = new ControllerApplication(); - private static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); - private static final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of( + private final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of( configServerApplication, configServerHostApplication, proxyHostApplication, @@ -52,19 +53,9 @@ public class DuperModelManager implements DuperModelInfraApi { private final Object monitor = new Object(); private final DuperModel duperModel; - // The set of active infrastructure ApplicationInfo. Not all is necessarily in the DuperModel for historical reasons. + // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons. private final Set<ApplicationId> activeInfraInfos = new HashSet<>(2 * supportedInfraApplications.size()); - /** - * Returns true if application is considered an infrastructure application by the DuperModel. - * - * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a - * cluster in the {@link ZoneApplication zone application}. - */ - public static boolean isInfrastructureApplication(ApplicationId applicationId) { - return supportedInfraApplications.containsKey(applicationId); - } - @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { this( @@ -148,6 +139,16 @@ public class DuperModelManager implements DuperModelInfraApi { return new ArrayList<>(supportedInfraApplications.values()); } + /** + * Returns true if application is considered an infrastructure application by the DuperModel. + * + * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a + * cluster in the {@link ZoneApplication zone application}. + */ + public boolean isSupportedInfraApplication(ApplicationId applicationId) { + return supportedInfraApplications.containsKey(applicationId); + } + @Override public boolean infraApplicationIsActive(ApplicationId applicationId) { synchronized (monitor) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java new file mode 100644 index 00000000000..5e6cb23e9c1 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.applicationmodel.ServiceType; + +/** + * @author hakonhall + */ +public abstract class HostAdminApplication extends InfraApplication { + protected HostAdminApplication(String applicationName, NodeType nodeType) { + super(applicationName, nodeType, ClusterSpec.Type.container, ClusterSpec.Id.from(applicationName), + ServiceType.HOST_ADMIN, 8080); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 89aabbfe2b0..8c74fe0396e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import com.yahoo.component.Version; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; @@ -11,34 +12,34 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.monitor.InfraApplicationApi; -import com.yahoo.vespa.service.model.ModelGenerator; +import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.health.ApplicationHealthMonitor; +import com.yahoo.vespa.service.model.ModelGenerator; +import com.yahoo.vespa.service.monitor.InfraApplicationApi; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * @author freva */ public abstract class InfraApplication implements InfraApplicationApi { - static final int HEALTH_PORT = 8080; - private static final TenantName TENANT_NAME = TenantName.from("hosted-vespa"); - private static final String CONFIG_ID_PREFIX = "configid."; private final ApplicationId applicationId; private final Capacity capacity; - private final ClusterSpec.Type clusterType; - private final ClusterSpec.Id clusterId; + private final ClusterSpec.Type clusterSpecType; + private final ClusterSpec.Id clusterSpecId; private final ServiceType serviceType; + private final int healthPort; public static ApplicationId createHostedVespaApplicationId(String applicationName) { return new ApplicationId.Builder() @@ -49,14 +50,16 @@ public abstract class InfraApplication implements InfraApplicationApi { protected InfraApplication(String applicationName, NodeType nodeType, - ClusterSpec.Type clusterType, - ClusterSpec.Id clusterId, - ServiceType serviceType) { + ClusterSpec.Type clusterSpecType, + ClusterSpec.Id clusterSpecId, + ServiceType serviceType, + int healthPort) { this.applicationId = createHostedVespaApplicationId(applicationName); this.capacity = Capacity.fromRequiredNodeType(nodeType); - this.clusterType = clusterType; - this.clusterId = clusterId; + this.clusterSpecType = clusterSpecType; + this.clusterSpecId = clusterSpecId; this.serviceType = serviceType; + this.healthPort = healthPort; } @Override @@ -71,31 +74,43 @@ public abstract class InfraApplication implements InfraApplicationApi { @Override public ClusterSpec getClusterSpecWithVersion(Version version) { - return ClusterSpec.request(clusterType, clusterId, version, true); + return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true); } - public ClusterSpec.Type getClusterType() { - return clusterType; + public ClusterSpec.Type getClusterSpecType() { + return clusterSpecType; } - public ClusterSpec.Id getClusterId() { - return clusterId; + public ClusterSpec.Id getClusterSpecId() { + return clusterSpecId; } - public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { - List<HostInfo> hostInfos = new ArrayList<>(); - for (int index = 0; index < hostnames.size(); ++index) { - hostInfos.add(makeHostInfo(hostnames.get(index), HEALTH_PORT, index, serviceType, clusterId)); - } + public ClusterId getClusterId() { + return new ClusterId(clusterSpecId.value()); + } + public ServiceType getServiceType() { + return serviceType; + } + + public ApplicationInstanceId getApplicationInstanceId() { + return new ApplicationInstanceId(applicationId.application().value()); + } + + public TenantId getTenantId() { + return new TenantId(applicationId.tenant().value()); + } + + public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { + List<HostInfo> hostInfos = hostnames.stream().map(this::makeHostInfo).collect(Collectors.toList()); return new ApplicationInfo(applicationId, 0, new HostsModel(hostInfos)); } - private static HostInfo makeHostInfo(HostName hostname, int port, int configIndex, ServiceType serviceType, ClusterSpec.Id clusterId) { - PortInfo portInfo = new PortInfo(port, ApplicationHealthMonitor.PORT_TAGS_HEALTH); + private HostInfo makeHostInfo(HostName hostname) { + PortInfo portInfo = new PortInfo(healthPort, ApplicationHealthMonitor.PORT_TAGS_HEALTH); Map<String, String> properties = new HashMap<>(); - properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, clusterId.value()); + properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, getClusterId().s()); ServiceInfo serviceInfo = new ServiceInfo( // service name == service type for the first service of each type on each host @@ -103,14 +118,24 @@ public abstract class InfraApplication implements InfraApplicationApi { serviceType.s(), Collections.singletonList(portInfo), properties, - configIdFrom(configIndex).s(), + configIdFor(hostname).s(), hostname.value()); return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo)); } - private static ConfigId configIdFrom(int index) { - return new ConfigId(CONFIG_ID_PREFIX + index); + public ConfigId configIdFor(HostName hostname) { + // Not necessarily unique, but service monitor doesn't require it to be unique. + return new ConfigId(String.format("%s/%s", clusterSpecId.value(), prefixTo(hostname.value(), '.'))); + } + + private static String prefixTo(String string, char sentinel) { + int offset = string.indexOf(sentinel); + if (offset == -1) { + return string; + } else { + return string.substring(0, offset); + } } @Override @@ -118,16 +143,17 @@ public abstract class InfraApplication implements InfraApplicationApi { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; InfraApplication that = (InfraApplication) o; - return Objects.equals(applicationId, that.applicationId) && + return healthPort == that.healthPort && + Objects.equals(applicationId, that.applicationId) && Objects.equals(capacity, that.capacity) && - clusterType == that.clusterType && - Objects.equals(clusterId, that.clusterId) && + clusterSpecType == that.clusterSpecType && + Objects.equals(clusterSpecId, that.clusterSpecId) && Objects.equals(serviceType, that.serviceType); } @Override public int hashCode() { - return Objects.hash(applicationId, capacity, clusterType, clusterId, serviceType); + return Objects.hash(applicationId, capacity, clusterSpecType, clusterSpecId, serviceType, healthPort); } @Override @@ -135,9 +161,10 @@ public abstract class InfraApplication implements InfraApplicationApi { return "InfraApplication{" + "applicationId=" + applicationId + ", capacity=" + capacity + - ", clusterType=" + clusterType + - ", clusterId=" + clusterId + + ", clusterSpecType=" + clusterSpecType + + ", clusterSpecId=" + clusterSpecId + ", serviceType=" + serviceType + + ", healthPort=" + healthPort + '}'; } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java index b6b0964a82d..24230609e60 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java @@ -1,13 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; -public class ProxyHostApplication extends InfraApplication { +public class ProxyHostApplication extends HostAdminApplication { public ProxyHostApplication() { - super("proxy-host", NodeType.proxyhost, ClusterSpec.Type.container, - ClusterSpec.Id.from("proxy-host"), ServiceType.HOST_ADMIN); + super("proxy-host", NodeType.proxyhost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java new file mode 100644 index 00000000000..43be236268c --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.config.model.api.ApplicationInfo; + +/** + * @author hakonhall + */ +@FunctionalInterface +interface ApplicationHealthMonitorFactory { + ApplicationHealthMonitor create(ApplicationInfo applicationInfo); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index daf9fd1931b..e9a5ec314f6 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -8,7 +8,9 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.flags.FileFlagSource; +import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.manager.MonitorManager; @@ -21,14 +23,29 @@ import java.util.concurrent.ConcurrentHashMap; */ public class HealthMonitorManager implements MonitorManager { private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>(); + private final DuperModelManager duperModel; + private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory; + private final FeatureFlag monitorInfra; @Inject - public HealthMonitorManager() { } + public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) { + this(duperModel, + new FeatureFlag("healthmonitor-monitorinfra", true, flagSource), + ApplicationHealthMonitor::startMonitoring); + } + + HealthMonitorManager(DuperModelManager duperModel, + FeatureFlag monitorInfra, + ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { + this.duperModel = duperModel; + this.applicationHealthMonitorFactory = applicationHealthMonitorFactory; + this.monitorInfra = monitorInfra; + } @Override public void applicationActivated(ApplicationInfo application) { if (wouldMonitor(application.getApplicationId())) { - ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(application); + ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application); healthMonitors.put(application.getApplicationId(), monitor); } } @@ -62,7 +79,11 @@ public class HealthMonitorManager implements MonitorManager { @Override public boolean wouldMonitor(ApplicationId id) { - if (id.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId())) { + if (duperModel.isSupportedInfraApplication(id) && monitorInfra.value()) { + return true; + } + + if (id.equals(duperModel.getConfigServerApplication().getApplicationId())) { return true; } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 9bcb3f07cf6..f15101d4439 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; @@ -25,8 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.vespa.service.duper.ConfigServerApplication.CONFIG_SERVER_APPLICATION; - /** * Class to generate an ApplicationInstance given service status for a standard (deployed) application. * @@ -37,10 +36,15 @@ public class ApplicationInstanceGenerator { private final ApplicationInfo applicationInfo; private final Zone zone; + private ApplicationId configServerApplicationId; public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) { this.applicationInfo = applicationInfo; this.zone = zone; + + // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. + // We do this to avoid passing through the ID through multiple levels. + this.configServerApplicationId = new ConfigServerApplication().getApplicationId(); } public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { @@ -105,7 +109,7 @@ public class ApplicationInstanceGenerator { } private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - if (applicationInfo.getApplicationId().equals(CONFIG_SERVER_APPLICATION.getApplicationId())) { + if (applicationInfo.getApplicationId().equals(configServerApplicationId)) { // Removing this historical discrepancy would break orchestration during rollout. // An alternative may be to use a feature flag and flip it between releases, // once that's available. diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java index b695b0b19f8..c0bcff7284c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java @@ -22,9 +22,9 @@ public class ServiceId { private final ConfigId configId; public ServiceId(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { this.applicationId = applicationId; this.clusterId = clusterId; this.serviceType = serviceType; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 39da5040ca0..13f24bc3694 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -28,14 +28,16 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { private final Object monitor = new Object(); private final HashMap<ApplicationId, SlobrokMonitor> slobrokMonitors = new HashMap<>(); + private final DuperModelManager duperModel; @Inject - public SlobrokMonitorManagerImpl() { - this(SlobrokMonitor::new); + public SlobrokMonitorManagerImpl(DuperModelManager duperModel) { + this(SlobrokMonitor::new, duperModel); } - SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory) { + SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory, DuperModelManager duperModel) { this.slobrokMonitorFactory = slobrokMonitorFactory; + this.duperModel = duperModel; } @Override @@ -108,7 +110,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { @Override public boolean wouldMonitor(ApplicationId applicationId) { - if (DuperModelManager.isInfrastructureApplication(applicationId)) { + if (duperModel.isSupportedInfraApplication(applicationId)) { return false; } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index 078f34b84d1..0dfca12099e 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; @@ -16,6 +17,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ApplicationHealthMonitorTest { + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + @Test public void sanityCheck() { MonitorFactory monitorFactory = new MonitorFactory(); @@ -36,28 +39,17 @@ public class ApplicationHealthMonitorTest { ConfigserverUtil.makeExampleConfigServer(), monitorFactory); - ConfigServerApplication configServerApplication = new ConfigServerApplication(); - - ServiceStatus status1 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(0)); - assertEquals(ServiceStatus.UP, status1); - - ServiceStatus status2 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(1)); - assertEquals(ServiceStatus.DOWN, status2); - - ServiceStatus status3 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(2)); - assertEquals(ServiceStatus.NOT_CHECKED, status3); + assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1")); + assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2")); + assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg3")); + } + + private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) { + return monitor.getStatus( + configServerApplication.getApplicationId(), + configServerApplication.getClusterId(), + configServerApplication.getServiceType(), + configServerApplication.configIdFor(HostName.from(hostname))); } private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 0b9d7e123b5..f420f5c1284 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -2,20 +2,51 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.ControllerHostApplication; +import com.yahoo.vespa.service.duper.DuperModelManager; +import com.yahoo.vespa.service.duper.InfraApplication; +import com.yahoo.vespa.service.duper.ProxyHostApplication; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import org.junit.Before; import org.junit.Test; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class HealthMonitorManagerTest { + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + private final DuperModelManager duperModel = mock(DuperModelManager.class); + private final FeatureFlag monitorInfra = mock(FeatureFlag.class); + private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class); + private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class); + private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorInfra, monitorFactory); + + @Before + public void setUp() { + when(duperModel.getConfigServerApplication()).thenReturn(configServerApplication); + when(monitorFactory.create(any())).thenReturn(monitor); + } + @Test public void addRemove() { - HealthMonitorManager manager = new HealthMonitorManager(); + when(monitorInfra.value()).thenReturn(false); ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer(); manager.applicationActivated(applicationInfo); manager.applicationRemoved(applicationInfo.getApplicationId()); @@ -23,7 +54,7 @@ public class HealthMonitorManagerTest { @Test public void withHostAdmin() { - HealthMonitorManager manager = new HealthMonitorManager(); + when(monitorInfra.value()).thenReturn(false); ServiceStatus status = manager.getStatus( ZoneApplication.ZONE_APPLICATION_ID, ClusterId.NODE_ADMIN, @@ -31,4 +62,56 @@ public class HealthMonitorManagerTest { new ConfigId("config-id-1")); assertEquals(ServiceStatus.UP, status); } + + @Test + public void infrastructureApplication() { + when(monitorInfra.value()).thenReturn(false); + + ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); + List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList()); + ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); + + manager.applicationActivated(proxyHostApplicationInfo); + verify(monitorFactory, never()).create(proxyHostApplicationInfo); + + assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1"); + } + + @Test + public void infrastructureApplicationWithMonitoring() { + when(monitorInfra.value()).thenReturn(true); + + ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); + List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList()); + ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); + + manager.applicationActivated(proxyHostApplicationInfo); + verify(monitorFactory, times(1)).create(proxyHostApplicationInfo); + + when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP); + assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1"); + + ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); + when(duperModel.isSupportedInfraApplication(controllerHostApplication.getApplicationId())).thenReturn(true); + assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1"); + } + + private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) { + ServiceStatus actual = manager.getStatus( + infraApplication.getApplicationId(), + infraApplication.getClusterId(), + infraApplication.getServiceType(), + infraApplication.configIdFor(HostName.from(hostname))); + + assertEquals(expected, actual); + + verify(monitor, times(verifyTimes)).getStatus( + infraApplication.getApplicationId(), + infraApplication.getClusterId(), + infraApplication.getServiceType(), + infraApplication.configIdFor(HostName.from(hostname))); + + } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index a59206d14e2..bf3f7017b01 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -6,16 +6,15 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.monitor.ConfigserverUtil; import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.service.duper.ConfigServerApplication.CONFIG_SERVER_APPLICATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -30,6 +29,7 @@ public class ApplicationInstanceGeneratorTest { configServer1, configServer2, configServer3).collect(Collectors.toList()); + private static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); private final ServiceStatusProvider statusProvider = mock(ServiceStatusProvider.class); @@ -43,24 +43,24 @@ public class ApplicationInstanceGeneratorTest { configServer2, configServer3); Zone zone = mock(Zone.class); - ApplicationInfo configServer = CONFIG_SERVER_APPLICATION.makeApplicationInfoFromConfig(config); + ApplicationInfo configServer = configServerApplication.makeApplicationInfoFromConfig(config); ApplicationInstance applicationInstance = new ApplicationInstanceGenerator(configServer, zone) .makeApplicationInstance(statusProvider); assertEquals( - ConfigServerApplication.APPLICATION_INSTANCE_ID, + configServerApplication.getApplicationInstanceId(), applicationInstance.applicationInstanceId()); assertEquals( - ConfigServerApplication.TENANT_ID, + configServerApplication.getTenantId(), applicationInstance.tenantId()); assertEquals( - ConfigServerApplication.TENANT_ID.toString() + - ":" + ConfigServerApplication.APPLICATION_INSTANCE_ID, + configServerApplication.getTenantId().toString() + + ":" + configServerApplication.getApplicationInstanceId(), applicationInstance.reference().toString()); assertEquals( - ConfigServerApplication.CLUSTER_ID, + configServerApplication.getClusterId(), applicationInstance.serviceClusters().iterator().next().clusterId()); assertEquals( diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index bcc136f80e3..30a49835f03 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -33,6 +33,7 @@ public class ModelGeneratorTest { private final String REGION = "us-west-1"; private final String HOSTNAME = "hostname"; private final int PORT = 2; + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); @Test public void toApplicationModel() throws Exception { @@ -63,8 +64,7 @@ public class ModelGeneratorTest { ApplicationInstance applicationInstance1 = iterator.next().getValue(); ApplicationInstance applicationInstance2 = iterator.next().getValue(); - if (applicationInstance1.applicationInstanceId().equals( - ConfigServerApplication.APPLICATION_INSTANCE_ID)) { + if (applicationInstance1.applicationInstanceId().equals(configServerApplication.getApplicationInstanceId())) { verifyConfigServerApplication(applicationInstance1); verifyOtherApplication(applicationInstance2); } else { @@ -108,7 +108,6 @@ public class ModelGeneratorTest { private void verifyConfigServerApplication( ApplicationInstance applicationInstance) { - assertEquals(ConfigServerApplication.APPLICATION_INSTANCE_ID, - applicationInstance.applicationInstanceId()); + assertEquals(configServerApplication.getApplicationInstanceId(), applicationInstance.applicationInstanceId()); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java index c334fad2334..7f817a0f1e6 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java @@ -33,7 +33,7 @@ public class ConfigserverUtil { String configServerHostname1, String configServerHostname2, String configServerHostname3) { - return ConfigServerApplication.CONFIG_SERVER_APPLICATION.makeApplicationInfoFromConfig(create( + return new ConfigServerApplication().makeApplicationInfoFromConfig(create( true, true, configServerHostname1, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java index c3ee1d078bf..bc6bff6fba8 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.DuperModelManager; import org.junit.Before; import org.junit.Test; @@ -25,8 +27,10 @@ public class SlobrokMonitorManagerImplTest { @SuppressWarnings("unchecked") private final Supplier<SlobrokMonitor> slobrokMonitorFactory = mock(Supplier.class); + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + private final DuperModelManager duperModelManager = mock(DuperModelManager.class); private final SlobrokMonitorManagerImpl slobrokMonitorManager = - new SlobrokMonitorManagerImpl(slobrokMonitorFactory); + new SlobrokMonitorManagerImpl(slobrokMonitorFactory, duperModelManager); private final SlobrokMonitor slobrokMonitor = mock(SlobrokMonitor.class); private final ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance"); private final ApplicationInfo application = mock(ApplicationInfo.class); @@ -34,6 +38,7 @@ public class SlobrokMonitorManagerImplTest { @Before public void setup() { + when(duperModelManager.getConfigServerApplication()).thenReturn(configServerApplication); when(slobrokMonitorFactory.get()).thenReturn(slobrokMonitor); when(application.getApplicationId()).thenReturn(applicationId); } diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 095caf17a56..b95b4fda01f 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_define_module( TESTS src/tests/array + src/tests/assert src/tests/benchmark src/tests/bits src/tests/clock diff --git a/staging_vespalib/src/tests/assert/.gitignore b/staging_vespalib/src/tests/assert/.gitignore new file mode 100644 index 00000000000..d0e23fcd846 --- /dev/null +++ b/staging_vespalib/src/tests/assert/.gitignore @@ -0,0 +1 @@ +staging_vespalib_asserter_app diff --git a/staging_vespalib/src/tests/assert/CMakeLists.txt b/staging_vespalib/src/tests/assert/CMakeLists.txt new file mode 100644 index 00000000000..7403a488645 --- /dev/null +++ b/staging_vespalib/src/tests/assert/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(staging_vespalib_assert_test_app TEST + SOURCES + assert_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_assert_test_app COMMAND staging_vespalib_assert_test_app) + +vespa_add_executable(staging_vespalib_asserter_app TEST + SOURCES + asserter.cpp + DEPENDS + staging_vespalib +) diff --git a/staging_vespalib/src/tests/assert/assert_test.cpp b/staging_vespalib/src/tests/assert/assert_test.cpp new file mode 100644 index 00000000000..454c0957974 --- /dev/null +++ b/staging_vespalib/src/tests/assert/assert_test.cpp @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/assert.h> +#include <vespa/vespalib/io/fileutil.h> +#include <sys/stat.h> +#include <unistd.h> +#include <vespa/defaults.h> + +using namespace vespalib; + +TEST("that it borks the first time.") { + std::string assertName; + const char * assertDir = "var/db/vespa/tmp"; + vespalib::rmdir("var", true); + ASSERT_TRUE(vespalib::mkdir(assertDir, true)); + { + SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./staging_vespalib_asserter_app myassert 10000"); + proc.wait(); + ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); + } + { + SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./staging_vespalib_asserter_app myassert 10000"); + proc.readLine(assertName); + proc.wait(); + ASSERT_EQUAL(proc.getExitCode() & 0x7f, 0); + } + ASSERT_EQUAL(0, unlink(assertName.c_str())); + { + SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./staging_vespalib_asserter_app myassert 10000"); + proc.wait(); + ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); + } + ASSERT_EQUAL(0, unlink(assertName.c_str())); + ASSERT_TRUE(vespalib::rmdir("var", true)); +} + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/assert/asserter.cpp b/staging_vespalib/src/tests/assert/asserter.cpp new file mode 100644 index 00000000000..640464889c0 --- /dev/null +++ b/staging_vespalib/src/tests/assert/asserter.cpp @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/util/assert.h> +#include <cassert> +#include <cstdlib> +#include <fstream> +#include <string> + +int main(int argc, char *argv[]) { + assert(argc == 3); + const char * assertKey = argv[1]; + size_t assertCount = strtoul(argv[2], nullptr, 0); + for (size_t i(0); i < assertCount; i++) { + ASSERT_ONCE_OR_LOG(true, assertKey, 100); + ASSERT_ONCE_OR_LOG(false, assertKey, 100); + } + std::string filename = vespalib::assert::getAssertLogFileName(assertKey); + std::ifstream is(filename.c_str()); + assert(is); + std::string line; + std::getline(is, line); + printf("%s\n", filename.c_str()); + assert(line.find(assertKey) != std::string::npos); + assert(assertCount == vespalib::assert::getNumAsserts(assertKey)); + return 0; +} diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index 20d47c90453..c104c047cd0 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib_vespalib_util OBJECT SOURCES + assert.cpp bits.cpp clock.cpp crc.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/assert.cpp b/staging_vespalib/src/vespa/vespalib/util/assert.cpp new file mode 100644 index 00000000000..0482d873f7f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/assert.cpp @@ -0,0 +1,77 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "assert.h" +#include <vespa/defaults.h> +#include <vespa/vespalib/util/backtrace.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/component/vtag.h> +#include <fstream> +#include <map> +#include <mutex> +#include <chrono> +#include <iomanip> + +#include <vespa/log/log.h> +LOG_SETUP(".vespa.assert"); + +namespace vespalib::assert { + +namespace { + +std::mutex _G_lock; +std::map<std::string, size_t> _G_assertMap; + +} + +size_t +getNumAsserts(const char *key) +{ + std::lock_guard guard(_G_lock); + return _G_assertMap[key]; +} + +vespalib::string +getAssertLogFileName(const char *key) +{ + vespalib::string relative = make_string("var/db/vespa/tmp/%s.%s.assert", key, Vtag::currentVersion.toString().c_str()); + return vespa::Defaults::underVespaHome(relative.c_str()); +} + +void +assertOnceOrLog(const char *expr, const char *key, size_t freq) +{ + size_t count(0); + { + std::lock_guard guard(_G_lock); + count = _G_assertMap[key]++; + } + if (count) { + if ((count % freq) == 0) { + LOG(error, "assert(%s) named '%s' has failed %zu times. Stacktrace = %s", + expr, key, count+1, vespalib::getStackTrace(0).c_str()); + } + } else { + std::string rememberAssert = getAssertLogFileName(key); + std::ifstream prevAssertFile(rememberAssert.c_str()); + if (prevAssertFile) { + if ((count % freq) == 0) { + LOG(error, "assert(%s) named '%s' has failed %zu times. Stacktrace = %s", + expr, key, count + 1, vespalib::getStackTrace(0).c_str()); + } + } else { + { + LOG(error, "assert(%s) named '%s' failed first time. Stacktrace = %s", + expr, key, vespalib::getStackTrace(0).c_str()); + std::ofstream assertStream(rememberAssert.c_str()); + std::chrono::time_point now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + assertStream << std::put_time(std::gmtime(&now_c), "%F %T") << " assert(" << expr + << ") named " << key << " failed" << std::endl; + assertStream.close(); + } + abort(); + } + } +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/assert.h b/staging_vespalib/src/vespa/vespalib/util/assert.h new file mode 100644 index 00000000000..698fa6774c1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/assert.h @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib::assert { + +/** + * How many times has asserts against this key failed. + * @param key + * @return + */ +size_t getNumAsserts(const char *key); + +/** + * Get the filename that will be used for remembering asserts. + * @param key + * @return + */ +vespalib::string getAssertLogFileName(const char *key); + +/** + * If there is no record on file that this assert has failed, it will be recorded and aborted. + * However if there is a record of it, it will merely be logged the first and then every #freq time. + * @param expr that failed the assert + * @param key unique name of assert + * @param logFreq how often will a failing assert be logged. + */ +void assertOnceOrLog(const char *expr, const char *key, size_t logFreq); + +} + +#define ASSERT_ONCE_OR_LOG(expr, key, freq) { \ + if ( ! (expr) ) { \ + vespalib::assert::assertOnceOrLog(#expr, key, freq); \ + } \ +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 65e7191e164..2b12c7cd78c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -50,7 +50,7 @@ public class OperationHandlerImpl implements OperationHandler { } public interface BucketSpaceResolver { - Optional<String> clusterBucketSpaceFromDocumentType(String clusterConfigId, String docType); + Optional<String> clusterBucketSpaceFromDocumentType(String clusterId, String docType); } public static class BucketSpaceRoute { @@ -91,20 +91,6 @@ public class OperationHandlerImpl implements OperationHandler { private final ConcurrentResourcePool<SyncSession> syncSessions; - private static ClusterEnumerator defaultClusterEnumerator() { - return () -> new ClusterList("client").getStorageClusters(); - } - - private static BucketSpaceResolver defaultBucketResolver() { - return (clusterConfigId, docType) -> Optional.ofNullable(BucketSpaceEnumerator - .fromConfig(clusterConfigId).getDoctypeToSpaceMapping() - .get(docType)); - } - - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, MetricReceiver metricReceiver) { - this(documentAccess, clusterEnumerator, defaultBucketResolver(), metricReceiver); - } - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, BucketSpaceResolver bucketSpaceResolver, MetricReceiver metricReceiver) { this.documentAccess = documentAccess; @@ -335,7 +321,7 @@ public class OperationHandlerImpl implements OperationHandler { String targetBucketSpace; if (!restUri.isRootOnly()) { String docType = restUri.getDocumentType(); - Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getConfigId(), docType); + Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getName(), docType); if (!resolvedSpace.isPresent()) { throw new RestApiException(Response.createErrorResponse(400, String.format( "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index f088db31c23..60f9101cdc8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -30,6 +30,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -75,6 +76,7 @@ public class RestApi extends LoggingRequestHandler { @Inject public RestApi(LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, + AllClustersBucketSpacesConfig bucketSpacesConfig, ClusterListConfig clusterListConfig, MetricReceiver metricReceiver) { super(parentCtx); @@ -83,6 +85,7 @@ public class RestApi extends LoggingRequestHandler { this.operationHandler = new OperationHandlerImpl( new MessageBusDocumentAccess(params), fixedClusterEnumeratorFromConfig(clusterListConfig), + fixedBucketSpaceResolverFromConfig(bucketSpacesConfig), metricReceiver); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); // 40% of the threads can be blocked before we deny requests. @@ -120,6 +123,14 @@ public class RestApi extends LoggingRequestHandler { return () -> clusters; } + private static OperationHandlerImpl.BucketSpaceResolver fixedBucketSpaceResolverFromConfig( + AllClustersBucketSpacesConfig bucketSpacesConfig) { + return (clusterId, docType) -> + Optional.ofNullable(bucketSpacesConfig.cluster(clusterId)) + .map(cluster -> cluster.documentType(docType)) + .map(type -> type.bucketSpace()); + } + private static Optional<String> requestProperty(String parameter, HttpRequest request) { final String property = request.getProperty(parameter); if (property != null && ! property.isEmpty()) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 4c04070cec4..30f039c31fb 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -132,7 +132,7 @@ public class OperationHandlerImplTest { }); when(documentAccess.createSyncSession(any(SyncParameters.class))).thenReturn(mockSyncSession); OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (configId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); + OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (clusterId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); return new OperationHandlerImpl(documentAccess, clusterEnumerator, bucketSpaceResolver, MetricReceiver.nullImplementation); } } diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp index fc7ee703e7f..b0f14bc18f3 100644 --- a/vespalib/src/vespa/vespalib/component/version.cpp +++ b/vespalib/src/vespa/vespalib/component/version.cpp @@ -19,7 +19,7 @@ Version::Version(int major, int minor, int micro, const string & qualifier) Version::Version(const Version &) = default; Version & Version::operator = (const Version &) = default; -Version::~Version() { } +Version::~Version() = default; void Version::initialize() @@ -151,5 +151,4 @@ Version::compareTo(const Version& other) const return getQualifier().compare(other.getQualifier()); } - } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/component/versionspecification.cpp b/vespalib/src/vespa/vespalib/component/versionspecification.cpp index 3351219804f..84cf446e8c5 100644 --- a/vespalib/src/vespa/vespalib/component/versionspecification.cpp +++ b/vespalib/src/vespa/vespalib/component/versionspecification.cpp @@ -22,7 +22,7 @@ VersionSpecification::VersionSpecification(int major, int minor, int micro, cons VersionSpecification::VersionSpecification(const VersionSpecification &) = default; -VersionSpecification::~VersionSpecification() { } +VersionSpecification::~VersionSpecification() = default; void VersionSpecification::initialize() diff --git a/vespalib/src/vespa/vespalib/component/versionspecification.h b/vespalib/src/vespa/vespalib/component/versionspecification.h index 5ba49106915..399db123a7a 100644 --- a/vespalib/src/vespa/vespalib/component/versionspecification.h +++ b/vespalib/src/vespa/vespalib/component/versionspecification.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/component/version.h> +#include "version.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/component/vtag.cpp b/vespalib/src/vespa/vespalib/component/vtag.cpp index b8ab2c5a920..c9cfade6e28 100644 --- a/vespalib/src/vespa/vespalib/component/vtag.cpp +++ b/vespalib/src/vespa/vespalib/component/vtag.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cstdio> #include "vtag.h" +#include <cstdio> #ifndef V_TAG #define V_TAG "NOTAG" diff --git a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java index fa3eb412016..063ba70c75d 100644 --- a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java @@ -3,9 +3,7 @@ package com.yahoo.yolean; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; import java.util.Optional; -import java.util.function.Supplier; /** * Helper methods for handling exceptions @@ -51,6 +49,17 @@ public class Exceptions { } /** + * Returns the first cause or the given throwable that is an instance of {@code clazz} + */ + public static <T extends Throwable> Optional<T> findCause(Throwable t, Class<T> clazz) { + for (; t != null; t = t.getCause()) { + if (clazz.isInstance(t)) + return Optional.of(clazz.cast(t)); + } + return Optional.empty(); + } + + /** * Wraps any IOException thrown from a runnable in an UncheckedIOException. */ public static void uncheck(RunnableThrowingIOException runnable) { diff --git a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java index 390018efdf7..cd06d14bb55 100644 --- a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java +++ b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -17,6 +18,19 @@ import static org.junit.Assert.assertNull; public class ExceptionsTestCase { @Test + public void testFindCause() { + IllegalArgumentException e1 = new IllegalArgumentException(); + IllegalStateException e2 = new IllegalStateException(e1); + RuntimeException e3 = new RuntimeException(e2); + + assertEquals(Optional.of(e3), Exceptions.findCause(e3, RuntimeException.class)); + assertEquals(Optional.of(e1), Exceptions.findCause(e3, IllegalArgumentException.class)); + assertEquals(Optional.empty(), Exceptions.findCause(e3, NumberFormatException.class)); + + assertEquals(Optional.of(e2), Exceptions.findCause(e2, RuntimeException.class)); + } + + @Test public void testToMessageStrings() { assertEquals("Blah",Exceptions.toMessageString(new Exception("Blah"))); assertEquals("Blah", Exceptions.toMessageString(new Exception(new Exception("Blah")))); |