diff options
189 files changed, 1929 insertions, 1276 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 32129a464c7..61fadc92b4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,7 @@ add_subdirectory(node-repository) add_subdirectory(orchestrator) add_subdirectory(persistence) add_subdirectory(persistencetypes) +add_subdirectory(predicate-search) add_subdirectory(searchcommon) add_subdirectory(searchcore) add_subdirectory(searchcorespi) diff --git a/Code-map.md b/Code-map.md index fba22bdf8e5..6eae723f31d 100644 --- a/Code-map.md +++ b/Code-map.md @@ -1,3 +1,5 @@ +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + # A map to the Vespa code base You want to get familiar with the Vespa code base but don't know where to start? diff --git a/README.md b/README.md index 8d62f415bfa..09a3f9a7124 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Some suggested improvements with pointers to code are in [TODO.md](TODO.md). ### Set up the build environment -C++ and Java building is supported on CentOS 7. The Java source can also be built on any platform having Java 8 and Maven installed. +C++ and Java building is supported on CentOS 7. The Java source can also be built on any platform having Java 11 and Maven installed. We recommend using the following environment: [Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md). You can also setup CentOS 7 natively and install the following build dependencies: diff --git a/athenz-identity-provider-service/README.md b/athenz-identity-provider-service/README.md new file mode 100644 index 00000000000..389af0b8042 --- /dev/null +++ b/athenz-identity-provider-service/README.md @@ -0,0 +1,4 @@ +# Athenz Identity Provider Service + +An [Athenz Copper Argos](https://github.com/yahoo/athenz/blob/master/docs/copper_argos.md) provider implementation for configserver. + diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index f93688abddc..378c262cd72 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -73,7 +73,7 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setConfigServerSpecs(List<Configserver.Spec> configServerSpecs) { + public TestProperties setConfigServerSpecs(List<Spec> configServerSpecs) { this.configServerSpecs = ImmutableList.copyOf(configServerSpecs); return this; } @@ -82,4 +82,55 @@ public class TestProperties implements ModelContext.Properties { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; return this; } + + public static class Spec implements ConfigServerSpec { + + private final String hostName; + private final int configServerPort; + private final int httpPort; + private final int zooKeeperPort; + + public String getHostName() { + return hostName; + } + + public int getConfigServerPort() { + return configServerPort; + } + + public int getHttpPort() { + return httpPort; + } + + public int getZooKeeperPort() { + return zooKeeperPort; + } + + @Override + public boolean equals(Object o) { + if (o instanceof ConfigServerSpec) { + ConfigServerSpec other = (ConfigServerSpec)o; + + return hostName.equals(other.getHostName()) && + configServerPort == other.getConfigServerPort() && + httpPort == other.getHttpPort() && + zooKeeperPort == other.getZooKeeperPort(); + } else { + return false; + } + } + + @Override + public int hashCode() { + return hostName.hashCode(); + } + + public Spec(String hostName, int configServerPort, int httpPort, int zooKeeperPort) { + this.hostName = hostName; + this.configServerPort = configServerPort; + this.httpPort = httpPort; + this.zooKeeperPort = zooKeeperPort; + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 50e9e29a6d9..c2f2fb61f4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -103,14 +103,6 @@ public class Admin extends AbstractConfigProducer implements Serializable { return configservers; } - public List<ConfigServerSpec> getConfigServerSpecs() { - List<ConfigServerSpec> serverSpecs = new ArrayList<>(); - for (Configserver server : getConfigservers()) { - serverSpecs.add(server.getConfigServerSpec()); - } - return serverSpecs; - } - public void removeSlobroks() { slobroks.clear(); } /** Returns an immutable list of the slobroks in this */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index a2839ec0fb6..2645b6cd59d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; @@ -74,64 +73,10 @@ public class Configserver extends AbstractService { return getRelativePort(1); } - ConfigServerSpec getConfigServerSpec() { - return new Spec(getHostName(), getConfigServerRpcPort(), getConfigServerHttpPort(), ZooKeepersConfigProvider.zkPort); - } - @Override public int getHealthPort() { return getRelativePort(1); } - // TODO: Remove this implementation when we are on Hosted Vespa. - public static class Spec implements ConfigServerSpec { - - private final String hostName; - private final int configServerPort; - private final int httpPort; - private final int zooKeeperPort; - - public String getHostName() { - return hostName; - } - - public int getConfigServerPort() { - return configServerPort; - } - - public int getHttpPort() { - return httpPort; - } - - public int getZooKeeperPort() { - return zooKeeperPort; - } - - @Override - public boolean equals(Object o) { - if (o instanceof ConfigServerSpec) { - ConfigServerSpec other = (ConfigServerSpec)o; - - return hostName.equals(other.getHostName()) && - configServerPort == other.getConfigServerPort() && - httpPort == other.getHttpPort() && - zooKeeperPort == other.getZooKeeperPort(); - } else { - return false; - } - } - - @Override - public int hashCode() { - return hostName.hashCode(); - } - - public Spec(String hostName, int configServerPort, int httpPort, int zooKeeperPort) { - this.hostName = hostName; - this.configServerPort = configServerPort; - this.httpPort = httpPort; - this.zooKeeperPort = zooKeeperPort; - } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java index 143df20d557..be3ec8a6a14 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java @@ -28,9 +28,7 @@ public class ZooKeepersConfigProvider implements ZookeepersConfig.Producer { public List<String> getZooKeepers() { List<String> servers = new ArrayList<>(); for (Configserver server : configServers) { - ConfigServerSpec serverSpec = server.getConfigServerSpec(); - servers.add(serverSpec.getHostName() + ":" + serverSpec.getZooKeeperPort()); - + servers.add(server.getHostName() + ":" + zkPort); } return servers; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 73fc9c0cf41..72db12dbbd8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -6,7 +6,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; -import com.yahoo.vespa.model.container.http.ssl.DummySslProvider; +import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import static com.yahoo.component.ComponentSpecification.fromString; @@ -22,7 +22,7 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig private final SimpleComponent sslProviderComponent; public ConnectorFactory(String name, int listenPort) { - this(name, listenPort, new DummySslProvider(name)); + this(name, listenPort, new DefaultSslProvider(name)); } public ConnectorFactory(String name, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredSslProvider.java new file mode 100644 index 00000000000..3c36933c030 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredSslProvider.java @@ -0,0 +1,63 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.ssl; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.util.Optional; + +import static com.yahoo.component.ComponentSpecification.fromString; + +/** + * @author mortent + */ +public class ConfiguredSslProvider extends SimpleComponent implements ConnectorConfig.Producer { + public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; + public static final String COMPONENT_CLASS = ConfiguredSslContextFactoryProvider.class.getName(); + public static final String COMPONENT_BUNDLE = "jdisc_http_service"; + + private final String privateKeyPath; + private final String certificatePath; + private final String caCertificatePath; + private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + + public ConfiguredSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + super(new ComponentModel( + new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), + fromString(COMPONENT_CLASS), + fromString(COMPONENT_BUNDLE)))); + this.privateKeyPath = privateKeyPath; + this.certificatePath = certificatePath; + this.caCertificatePath = caCertificatePath; + this.clientAuthentication = mapToConfigEnum(clientAuthentication); + } + + @Override + public void getConfig(ConnectorConfig.Builder builder) { + builder.ssl.enabled(true); + builder.ssl.privateKeyFile(privateKeyPath); + builder.ssl.certificateFile(certificatePath); + builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); + builder.ssl.clientAuth(clientAuthentication); + } + + public SimpleComponent getComponent() { + return new SimpleComponent(new ComponentModel(getComponentId().stringValue(), COMPONENT_CLASS, COMPONENT_BUNDLE)); + } + + private static ConnectorConfig.Ssl.ClientAuth.Enum mapToConfigEnum(String clientAuthValue) { + if ("disabled".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; + } else if ("want".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH; + } else if ("need".equals(clientAuthValue)) { + return ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH; + } else { + return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java index 3554ce95faf..1a5ce615a9d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DefaultSslProvider.java @@ -8,56 +8,24 @@ import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; -import java.util.Optional; - import static com.yahoo.component.ComponentSpecification.fromString; /** - * @author mortent + * @author bjorncs */ public class DefaultSslProvider extends SimpleComponent implements ConnectorConfig.Producer { + public static final String COMPONENT_ID_PREFIX = "default-ssl-provider@"; public static final String COMPONENT_CLASS = DefaultSslContextFactoryProvider.class.getName(); public static final String COMPONENT_BUNDLE = "jdisc_http_service"; - private final String privateKeyPath; - private final String certificatePath; - private final String caCertificatePath; - private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; - - public DefaultSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + public DefaultSslProvider(String serverName) { super(new ComponentModel( - new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), + new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX + serverName), fromString(COMPONENT_CLASS), fromString(COMPONENT_BUNDLE)))); - this.privateKeyPath = privateKeyPath; - this.certificatePath = certificatePath; - this.caCertificatePath = caCertificatePath; - this.clientAuthentication = mapToConfigEnum(clientAuthentication); } @Override - public void getConfig(ConnectorConfig.Builder builder) { - builder.ssl.enabled(true); - builder.ssl.privateKeyFile(privateKeyPath); - builder.ssl.certificateFile(certificatePath); - builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); - builder.ssl.clientAuth(clientAuthentication); - } - - public SimpleComponent getComponent() { - return new SimpleComponent(new ComponentModel(getComponentId().stringValue(), COMPONENT_CLASS, COMPONENT_BUNDLE)); - } - - private static ConnectorConfig.Ssl.ClientAuth.Enum mapToConfigEnum(String clientAuthValue) { - if ("disabled".equals(clientAuthValue)) { - return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; - } else if ("want".equals(clientAuthValue)) { - return ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH; - } else if ("need".equals(clientAuthValue)) { - return ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH; - } else { - return ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED; - } - } -} + public void getConfig(ConnectorConfig.Builder builder) {} +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DummySslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DummySslProvider.java deleted file mode 100644 index 4e2ee61f33f..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DummySslProvider.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.http.ssl; - -import com.yahoo.component.ComponentId; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.ThrowingSslContextFactoryProvider; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -import static com.yahoo.component.ComponentSpecification.fromString; - -/** - * @author bjorncs - */ -public class DummySslProvider extends SimpleComponent implements ConnectorConfig.Producer { - - public static final String COMPONENT_ID_PREFIX = "dummy-ssl-provider@"; - public static final String COMPONENT_CLASS = ThrowingSslContextFactoryProvider.class.getName(); - public static final String COMPONENT_BUNDLE = "jdisc_http_service"; - - public DummySslProvider(String serverName) { - super(new ComponentModel( - new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX + serverName), - fromString(COMPONENT_CLASS), - fromString(COMPONENT_BUNDLE)))); - } - - @Override - public void getConfig(ConnectorConfig.Builder builder) {} -}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index 4c41aaa504c..23865eb9bdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; +import com.yahoo.vespa.model.container.http.ssl.ConfiguredSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; -import com.yahoo.vespa.model.container.http.ssl.DummySslProvider; import org.w3c.dom.Element; import java.util.Optional; @@ -39,7 +39,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); - return new DefaultSslProvider( + return new ConfiguredSslProvider( serverName, privateKeyFile, certificateFile, @@ -50,7 +50,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String bundle = sslProviderConfigurator.getAttribute("bundle"); return new CustomSslProvider(serverName, className, bundle); } else { - return new DummySslProvider(serverName); + return new DefaultSslProvider(serverName); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index d7861a6f284..f4fb3ac5247 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; import com.yahoo.text.XML; import com.yahoo.vespa.model.admin.*; @@ -115,9 +116,9 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { @Test public void multitenant() { List<ConfigServerSpec> configServerSpecs = Arrays.asList( - new Configserver.Spec("test1", 19070, 19071, 2181), - new Configserver.Spec("test2", 19070, 19071, 2181), - new Configserver.Spec("test3", 19070, 19071, 2181)); + new TestProperties.Spec("test1", 19070, 19071, 2181), + new TestProperties.Spec("test2", 19070, 19071, 2181), + new TestProperties.Spec("test3", 19070, 19071, 2181)); Admin admin = buildAdmin(servicesMultitenantAdminOnly(), true, configServerSpecs); assertThat(admin.getConfigservers().size(), is(3)); assertThat(admin.getSlobroks().size(), is(1)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 5525b75373d..424f55c67cb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.JettyHttpServer; -import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; +import com.yahoo.vespa.model.container.http.ssl.ConfiguredSslProvider; import org.junit.Test; import org.w3c.dom.Element; @@ -150,21 +150,21 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas "</jdisc>"); createModel(root, clusterElem); - ConnectorConfig minimalCfg = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/minimal/default-ssl-provider@minimal"); + ConnectorConfig minimalCfg = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/minimal/configured-ssl-provider@minimal"); assertTrue(minimalCfg.ssl().enabled()); assertThat(minimalCfg.ssl().privateKeyFile(), is(equalTo("/foo/key"))); assertThat(minimalCfg.ssl().certificateFile(), is(equalTo("/foo/cert"))); assertThat(minimalCfg.ssl().caCertificateFile(), is(equalTo(""))); assertThat(minimalCfg.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED))); - ConnectorConfig withCaCerts = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-cacerts/default-ssl-provider@with-cacerts"); + ConnectorConfig withCaCerts = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-cacerts/configured-ssl-provider@with-cacerts"); assertTrue(withCaCerts.ssl().enabled()); assertThat(withCaCerts.ssl().privateKeyFile(), is(equalTo("/foo/key"))); assertThat(withCaCerts.ssl().certificateFile(), is(equalTo("/foo/cert"))); assertThat(withCaCerts.ssl().caCertificateFile(), is(equalTo("/foo/cacerts"))); assertThat(withCaCerts.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.DISABLED))); - ConnectorConfig needClientAuth = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/need-client-auth/default-ssl-provider@need-client-auth"); + ConnectorConfig needClientAuth = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/need-client-auth/configured-ssl-provider@need-client-auth"); assertTrue(needClientAuth.ssl().enabled()); assertThat(needClientAuth.ssl().privateKeyFile(), is(equalTo("/foo/key"))); assertThat(needClientAuth.ssl().certificateFile(), is(equalTo("/foo/cert"))); @@ -173,7 +173,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); - connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, DefaultSslProvider.COMPONENT_CLASS)); + connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredSslProvider.COMPONENT_CLASS)); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index 5d842c6651c..14659494780 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -280,7 +280,7 @@ public class VespaModelTestCase { .applicationPackage(applicationPackage) .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com")) .properties(new TestProperties() - .setConfigServerSpecs(Arrays.asList(new Configserver.Spec("cfghost", 1234, 1235, 1236))) + .setConfigServerSpecs(Arrays.asList(new TestProperties.Spec("cfghost", 1234, 1235, 1236))) .setMultitenant(true)) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); diff --git a/config/src/vespa/config/common/vespa_version.cpp b/config/src/vespa/config/common/vespa_version.cpp index 9cc15388acd..7cbbe696dba 100644 --- a/config/src/vespa/config/common/vespa_version.cpp +++ b/config/src/vespa/config/common/vespa_version.cpp @@ -8,10 +8,9 @@ namespace config { const VespaVersion currentVersion(VespaVersion::fromString(vespalib::string(vespalib::VersionTagComponent))); -VespaVersion::VespaVersion(const VespaVersion & vespaVersion) - : _versionString(vespaVersion._versionString) -{ -} +VespaVersion::VespaVersion(const VespaVersion & vespaVersion) = default; + +VespaVersion & VespaVersion::operator=(const VespaVersion &rhs) = default; VespaVersion::VespaVersion(const vespalib::string & versionString) : _versionString(versionString) diff --git a/config/src/vespa/config/common/vespa_version.h b/config/src/vespa/config/common/vespa_version.h index 2ca2ddc858e..0ee1970d447 100644 --- a/config/src/vespa/config/common/vespa_version.h +++ b/config/src/vespa/config/common/vespa_version.h @@ -14,6 +14,7 @@ public: static VespaVersion fromString(const vespalib::string & versionString); static const VespaVersion & getCurrentVersion(); VespaVersion(const VespaVersion & version); + VespaVersion &operator=(const VespaVersion &rhs); const vespalib::string & toString() const; private: VespaVersion(const vespalib::string & versionString); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index e8a5f1708a1..d631cc18d75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -40,7 +40,7 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp * applications. If that is done successfully the RPC server will start and the health status code will change from * 'initializing' to 'up'. If VIP status mode is VIP_STATUS_PROGRAMMATICALLY the config server * will be put into rotation (start serving status.html with 200 OK), if the mode is VIP_STATUS_FILE a VIP status - * file is created or removed ny some external pgrogram based on the health status code. + * file is created or removed ny some external program based on the health status code. * * @author Ulf Lilleengen * @author hmusum @@ -176,12 +176,10 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void up() { - stateMonitor.status(StateMonitor.Status.up); vipStatus.setInRotation(true); } private void down() { - stateMonitor.status(StateMonitor.Status.down); vipStatus.setInRotation(false); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 281933e8943..2e7b5a1f4d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -72,9 +72,10 @@ public class ConfigServerBootstrapTest { RpcServer rpcServer = createRpcServer(configserverConfig); // Take a host away so that there are too few for the application, to verify we can still bootstrap provisioner.allocations().values().iterator().next().remove(0); - VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, - versionState, createStateMonitor(), + versionState, stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY); assertFalse(vipStatus.isInRotation()); bootstrap.start(); @@ -102,9 +103,10 @@ public class ConfigServerBootstrapTest { assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(VIP_STATUS_FILE); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, - versionState, createStateMonitor(), + versionState, stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_FILE); assertTrue(vipStatus.isInRotation()); // default is in rotation when using status file @@ -132,16 +134,17 @@ public class ConfigServerBootstrapTest { .resolve("sessions/2/services.xml")); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - createStateMonitor(), + stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY); assertFalse(vipStatus.isInRotation()); // Call method directly, to be sure that it is finished redeploying all applications and we can check status bootstrap.start(); // App is invalid, bootstrapping was unsuccessful. Status should be 'initializing', // rpc server should not be running and it should be out of rotation - assertEquals(StateMonitor.Status.initializing, bootstrap.status()); + assertEquals(StateMonitor.Status.initializing, stateMonitor.status()); assertFalse(rpcServer.isRunning()); assertFalse(vipStatus.isInRotation()); @@ -174,9 +177,10 @@ public class ConfigServerBootstrapTest { curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY); + StateMonitor stateMonitor = createStateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - createStateMonitor(), vipStatus, + stateMonitor, vipStatus, BOOTSTRAP_IN_SEPARATE_THREAD, VIP_STATUS_PROGRAMMATICALLY); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); @@ -229,14 +233,10 @@ public class ConfigServerBootstrapTest { return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); } - private VipStatus createVipStatus(ConfigServerBootstrap.VipStatusMode vipStatusMode) throws IOException { + private VipStatus createVipStatus(StateMonitor stateMonitor) { return new VipStatus(new QrSearchersConfig.Builder().build(), - new VipStatusConfig.Builder() - .initiallyInRotation(vipStatusMode == VIP_STATUS_FILE) - .statusfile(temporaryFolder.newFile().getAbsolutePath()) - .accessdisk(vipStatusMode == VIP_STATUS_FILE) - .build(), - new ClustersStatus()); + new ClustersStatus(), + stateMonitor); } public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc { diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index dc13bd1807d..12d804e86c3 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -241,6 +241,8 @@ "public void <init>()", "public void <init>(com.yahoo.container.QrSearchersConfig)", "public void <init>(com.yahoo.container.handler.ClustersStatus)", + "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus)", + "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus)", "public void setInRotation(java.lang.Boolean)", "public void addToRotation(java.lang.String)", diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index 4f4d4635933..aab13a1cc7b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -9,12 +9,16 @@ import java.util.Map; /** * A component which tracks the up/down status of any clusters which should influence - * the up down status of this container itself, as well as the separate fact that such clusters are present. + * the up down status of this container itself, as well as the separate fact (from config) + * that such clusters are present. This is a separate fact because we might know we have clusters configured + * but we don't have positive information that they are up yet, and in this case we should be down. * * This is a separate component which has <b>no dependencies</b> such that the status tracked in this * will survive reconfiguration events and inform other components even immediately after a reconfiguration * (where the true statue of clusters may not yet be available). * + * This is multithread safe. + * * @author bratseth */ public class ClustersStatus extends AbstractComponent { @@ -26,9 +30,6 @@ public class ClustersStatus extends AbstractComponent { /** Are there any (in-service influencing) clusters in this container? */ private boolean containerHasClusters; - /** If we have no clusters, what should we answer? */ - private boolean receiveTrafficByDefault; - private final Object mutex = new Object(); /** The status of clusters, when known. Note that clusters may exist for which there is no knowledge yet. */ @@ -42,10 +43,9 @@ public class ClustersStatus extends AbstractComponent { } } + /** @deprecated this is ignored */ + @Deprecated // TODO: remove on Vespa 8 public void setReceiveTrafficByDefault(boolean receiveTrafficByDefault) { - synchronized (mutex) { - this.receiveTrafficByDefault = receiveTrafficByDefault; - } } void setUp(String clusterIdentifier) { @@ -80,7 +80,7 @@ public class ClustersStatus extends AbstractComponent { return clusterStatus.values().stream().anyMatch(status -> status==true); } else { - return receiveTrafficByDefault; + return true; } } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index 3b71ffff616..6f827041ffb 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -4,38 +4,63 @@ package com.yahoo.container.handler; import com.google.inject.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.jdisc.state.StateMonitor; /** - * API for programmatically removing the container from VIP rotation. + * A component which keeps track of whether or not this container instance should receive traffic + * and respond that it is in good health. + * + * This is multithread safe. * * @author Steinar Knutsen + * @author bratseth */ public class VipStatus { private final ClustersStatus clustersStatus; + private final StateMonitor healthState; + /** If this is non-null, its value decides whether this container is in rotation */ - private Boolean inRotationOverride; + private Boolean rotationOverride = null; + + /** The current state of this */ + private boolean currentlyInRotation; + + private final Object mutex = new Object(); + /** For testing */ public VipStatus() { - this(new QrSearchersConfig(new QrSearchersConfig.Builder()), - new VipStatusConfig(new VipStatusConfig.Builder()), - new ClustersStatus()); + this(new ClustersStatus()); } + /** For testing */ public VipStatus(QrSearchersConfig dispatchers) { - this(dispatchers, new VipStatusConfig(new VipStatusConfig.Builder()), new ClustersStatus()); + this(dispatchers, new ClustersStatus()); } + /** For testing */ public VipStatus(ClustersStatus clustersStatus) { - this.clustersStatus = clustersStatus; + this(new QrSearchersConfig.Builder().build(), clustersStatus); + } + + public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) { + this(dispatchers, clustersStatus, new StateMonitor()); } @Inject - public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig vipStatusConfig, ClustersStatus clustersStatus) { + public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { this.clustersStatus = clustersStatus; - clustersStatus.setReceiveTrafficByDefault(vipStatusConfig.initiallyInRotation()); + this.healthState = healthState; + healthState.status(StateMonitor.Status.initializing); clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty()); + updateCurrentlyInRotation(); + } + + /** @deprecated don't pass VipStatusConfig */ + @Deprecated // TODO: Remove on Vespa 8 + public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig ignored, ClustersStatus clustersStatus) { + this(dispatchers, clustersStatus); } /** @@ -45,17 +70,22 @@ public class VipStatus { * false to set it out, and null to make this decision using the usual cluster-dependent logic */ public void setInRotation(Boolean inRotation) { - this.inRotationOverride = inRotation; + synchronized (mutex) { + rotationOverride = inRotation; + updateCurrentlyInRotation(); + } } /** Note that a cluster (which influences up/down state) is up */ public void addToRotation(String clusterIdentifier) { clustersStatus.setUp(clusterIdentifier); + updateCurrentlyInRotation(); } /** Note that a cluster (which influences up/down state) is down */ public void removeFromRotation(String clusterIdentifier) { clustersStatus.setDown(clusterIdentifier); + updateCurrentlyInRotation(); } /** @deprecated use addToRotation(String) instead */ @@ -70,10 +100,24 @@ public class VipStatus { removeFromRotation((String) clusterIdentifier); } + private void updateCurrentlyInRotation() { + synchronized (mutex) { + if (rotationOverride != null) + currentlyInRotation = rotationOverride; + else + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + + // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' + if (currentlyInRotation) + healthState.status(StateMonitor.Status.up); + else if (healthState.status() == StateMonitor.Status.up) + healthState.status(StateMonitor.Status.down); + } + } + /** Returns whether this container should receive traffic at this time */ public boolean isInRotation() { - if (inRotationOverride != null) return inRotationOverride; - return clustersStatus.containerShouldReceiveTraffic(); + return currentlyInRotation; } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index c8cf575dff3..a37255436ca 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -57,7 +57,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { private StatusResponse() { super(com.yahoo.jdisc.http.HttpResponse.Status.OK); // status may be overwritten below if (vipStatus != null && ! vipStatus.isInRotation()) { - searchContainerOutOfService(); + setOutOfServiceStatus(); } else if (accessDisk) { preSlurpFile(); } else { @@ -140,7 +140,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { /** * Behaves like a VIP status response file has been deleted. */ - private void searchContainerOutOfService() { + private void setOutOfServiceStatus() { contentType = "text/plain"; data = Utf8.toBytes(NO_SEARCH_BACKENDS); setStatus(com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND); @@ -185,23 +185,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { public HttpResponse handle(HttpRequest request) { if (metric != null) metric.add(NUM_REQUESTS_METRIC, 1, null); - if (vipStatus != null) - updateAndLogRotationState(); return new StatusResponse(); } - private void updateAndLogRotationState() { - final boolean currentlyInRotation = vipStatus.isInRotation(); - final boolean previousRotationAnswer = previouslyInRotation; - previouslyInRotation = currentlyInRotation; - - if (previousRotationAnswer != currentlyInRotation) { - if (currentlyInRotation) { - log.log(LogLevel.INFO, "Putting container back into rotation by serving status.html again."); - } else { - log.log(LogLevel.WARNING, "Removing container from rotation by no longer serving status.html."); - } - } - } - } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 9d558a9c2a2..f690c240537 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.log.LogLevel; import java.util.Map; @@ -37,6 +38,11 @@ public class StateMonitor extends AbstractComponent { private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); + /** For testing */ + public StateMonitor() { + this(new HealthMonitorConfig.Builder().build(), new SystemTimer()); + } + @Inject public StateMonitor(HealthMonitorConfig config, Timer timer) { this(config, timer, runnable -> { diff --git a/container-core/src/main/resources/configdefinitions/vip-status.def b/container-core/src/main/resources/configdefinitions/vip-status.def index ed7ab3e4802..a042f5915c2 100644 --- a/container-core/src/main/resources/configdefinitions/vip-status.def +++ b/container-core/src/main/resources/configdefinitions/vip-status.def @@ -8,5 +8,5 @@ accessdisk bool default=false ## If the path is relative vespa home is prepended statusfile string default="share/qrsdocs/status.html" -## The default rotation state when there are no configured clusters to decide rotation state +## Not used TODO: Remove on Vespa 8 initiallyInRotation bool default=true diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 4c1c1622140..52679c15957 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; +import com.yahoo.container.QrSearchersConfig; import org.junit.Test; /** @@ -14,13 +15,17 @@ public class VipStatusTestCase { @Test public void testVipStatusWorksWithClusters() { - ClustersStatus clustersStatus = new ClustersStatus(); - clustersStatus.setContainerHasClusters(true); - VipStatus v = new VipStatus(clustersStatus); - - String cluster1 = new String("a"); - String cluster2 = new String("b"); - String cluster3 = new String("c"); + var b = new QrSearchersConfig.Builder(); + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + searchClusterB.name("cluster1"); + searchClusterB.name("cluster2"); + searchClusterB.name("cluster3"); + b.searchcluster(searchClusterB); + VipStatus v = new VipStatus(b.build()); + + String cluster1 = "cluster1"; + String cluster2 = "cluster2"; + String cluster3 = "cluster3"; // initial state assertFalse(v.isInRotation()); diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index e3c10c1f8ce..77fb740043e 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -100,7 +100,7 @@ public class QueryPacket extends Packet { byte[] stripped = new byte[encodedBody.length - (ignoreableSize + getSessionKeySkipLength()) + utf8Summary.length + 1]; System.arraycopy(encodedBody, 0, stripped, 0, ignoreableOffset); - stripped[1] = (byte)(stripped[1] & 0x7f); // Ignor sessionKey feature flag + stripped[1] = (byte)(stripped[1] & 0x7f); // Ignore sessionKey feature flag System.arraycopy(utf8Summary, 0, stripped, ignoreableOffset, utf8Summary.length); stripped[ignoreableOffset + utf8Summary.length] = 0; diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index af89220e504..ca776aef011 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -836,7 +836,10 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< /** Returns the set of summaries for which all concrete hits recursively below this is filled. */ @Override public Set<String> getFilled() { - Set<String> filled = null; + /* + This is an optimisation to avoid creating many temporary hash sets in the happy path. + The simple naive implementation is + Set<String> filled = null; for (Hit hit : hits) { if (hit.getFilled() == null) continue; if (filled == null) @@ -845,6 +848,54 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< filled.retainAll(hit.getFilled()); } return filled; + */ + Iterator<Hit> iterator = hits.iterator(); + Set<String> firstSummaryNames = getSummaryNamesNextFilledHit(iterator); + if (firstSummaryNames == null || firstSummaryNames.isEmpty()) + return firstSummaryNames; + + Set<String> intersection = firstSummaryNames; + while (iterator.hasNext()) { + Set<String> summaryNames = getSummaryNamesNextFilledHit(iterator); + if (summaryNames == null) continue; + + if (intersection.size() == 1) + return getFilledSingle(intersection.iterator().next(), summaryNames, iterator); + + boolean identical = false; + if (intersection == firstSummaryNames) { + // Here we have detected the first inequality and we must create a modifiable set for later use of retainAll + identical = intersection.equals(summaryNames); + if (!identical) { + intersection = new HashSet<>(firstSummaryNames); + } + } + if ( ! identical ) { + intersection.retainAll(summaryNames); + } + } + return intersection; + } + + + private Set<String> getSummaryNamesNextFilledHit(Iterator<Hit> hitIterator) { + while (hitIterator.hasNext()) { + Set<String> filled = hitIterator.next().getFilled(); + if (filled != null) + return filled; + } + return null; + } + + private Set<String> getFilledSingle(String summaryName, Set<String> summaryNames, Iterator<Hit> iterator) { + while (true) { + if (summaryNames == null) { + return Collections.singleton(summaryName); + } else if (!summaryNames.contains(summaryName)) { + return Collections.emptySet(); + } + summaryNames = getSummaryNamesNextFilledHit(iterator); + } } private Iterable<Hit> fillableHits() { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java index 5377eb670c4..0caef371d0c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.fastsearch.test; import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; @@ -45,11 +46,16 @@ class FastSearcherTester { } public FastSearcherTester(int containerClusterSize, List<Node> searchNodes) { - ClustersStatus clustersStatus = new ClustersStatus(); - clustersStatus.setContainerHasClusters(true); - vipStatus = new VipStatus(clustersStatus); + String clusterId = "a"; + + var b = new QrSearchersConfig.Builder(); + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + searchClusterB.name(clusterId); + b.searchcluster(searchClusterB); + vipStatus = new VipStatus(b.build()); + mockFS4ResourcePool = new MockFS4ResourcePool(); - mockDispatcher = new MockDispatcher("a", searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); + mockDispatcher = new MockDispatcher(clusterId, searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); fastSearcher = new FastSearcher(new MockBackend(selfHostname, 0L, true), mockFS4ResourcePool, mockDispatcher, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 46a6d1c35ad..e8b0c8a66ef 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -2,15 +2,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.UUID; +import java.util.TreeSet; import java.util.stream.Collectors; /** @@ -20,67 +16,64 @@ import java.util.stream.Collectors; */ public class MemoryNameService implements NameService { - private final Map<RecordId, Set<Record>> records = new HashMap<>(); + private final Set<Record> records = new TreeSet<>(); - public Map<RecordId, Set<Record>> records() { - return Collections.unmodifiableMap(records); + public Set<Record> records() { + return Collections.unmodifiableSet(records); } @Override - public RecordId createCname(RecordName name, RecordData canonicalName) { - RecordId id = new RecordId(UUID.randomUUID().toString()); - records.put(id, Set.of(new Record(id, Record.Type.CNAME, name, canonicalName))); - return id; + public Record createCname(RecordName name, RecordData canonicalName) { + Record record = new Record(Record.Type.CNAME, name, canonicalName); + records.add(record); + return record; } @Override - public RecordId createAlias(RecordName name, Set<AliasTarget> targets) { - RecordId id = new RecordId(UUID.randomUUID().toString()); - Set<Record> records = targets.stream() + public List<Record> createAlias(RecordName name, Set<AliasTarget> targets) { + List<Record> records = targets.stream() .sorted((a, b) -> Comparator.comparing(AliasTarget::name).compare(a, b)) - .map(target -> new Record(id, Record.Type.ALIAS, name, + .map(target -> new Record(Record.Type.ALIAS, name, RecordData.fqdn(target.name().value()))) - .collect(Collectors.toCollection(LinkedHashSet::new)); + .collect(Collectors.toList()); // Satisfy idempotency contract of interface - findRecords(Record.Type.ALIAS, name).stream().map(Record::id).forEach(this::removeRecord); - this.records.put(id, records); - return id; + removeRecords(findRecords(Record.Type.ALIAS, name)); + this.records.addAll(records); + return records; } @Override public List<Record> findRecords(Record.Type type, RecordName name) { - return records.values().stream() - .flatMap(Collection::stream) + return records.stream() .filter(record -> record.type() == type && record.name().equals(name)) .collect(Collectors.toUnmodifiableList()); } @Override public List<Record> findRecords(Record.Type type, RecordData data) { - return records.values().stream() - .flatMap(Collection::stream) + return records.stream() .filter(record -> record.type() == type && record.data().equals(data)) .collect(Collectors.toUnmodifiableList()); } @Override - public void updateRecord(RecordId id, RecordData newData) { - records.computeIfPresent(id, (k, records) -> { - if (records.isEmpty()) { - throw new IllegalArgumentException("No record with data '" + newData.asString() + "' exists"); - } - if (records.size() > 1) { - throw new IllegalArgumentException("Cannot update multi-value record '" + id.asString() + "' with '" + - newData.asString() + "'"); - } - Record existing = records.iterator().next(); - return Set.of(new Record(id, existing.type(), existing.name(), newData)); - }); + public void updateRecord(Record record, RecordData newData) { + List<Record> records = findRecords(record.type(), record.name()); + if (records.isEmpty()) { + throw new IllegalArgumentException("No record with data '" + newData.asString() + "' exists"); + } + if (records.size() > 1) { + throw new IllegalArgumentException("Cannot update multi-value record '" + record.name().asString() + + "' with '" + newData.asString() + "'"); + } + Record existing = records.get(0); + this.records.remove(existing); + this.records.add(new Record(existing.type(), existing.name(), newData)); } @Override - public void removeRecord(RecordId id) { - records.remove(id); + public void removeRecords(List<Record> records) { + this.records.removeAll(records); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java index 537460c8b1e..444c8dda8d3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java @@ -14,24 +14,31 @@ public interface NameService { /** * Create a new CNAME record * - * @param alias The alias to create - * @param canonicalName The canonical name which the alias should point to. This must be a FQDN. + * @param name The alias to create (lhs of the record) + * @param canonicalName The canonical name which the alias should point to (rhs of the record). This must be a FQDN. + * @return The created record */ - RecordId createCname(RecordName alias, RecordData canonicalName); + Record createCname(RecordName name, RecordData canonicalName); - /** Create a non-standard ALIAS record pointing to given targets. Implementations of this are expected to be idempotent */ - RecordId createAlias(RecordName name, Set<AliasTarget> targets); + /** + * Create a non-standard ALIAS record pointing to given targets. Implementations of this can be expected to be + * idempotent + * + * @param targets Targets that should be resolved by this alias. pointing to given targets. + * @return The created records. One for each target. + */ + List<Record> createAlias(RecordName name, Set<AliasTarget> targets); - /** Find records matching type and name */ + /** Find all records matching given type and name */ List<Record> findRecords(Record.Type type, RecordName name); - /** Find records matching type and data */ + /** Find all records matching given type and data */ List<Record> findRecords(Record.Type type, RecordData data); /** Update existing record */ - void updateRecord(RecordId id, RecordData newData); + void updateRecord(Record record, RecordData newData); - /** Remove record by ID */ - void removeRecord(RecordId id); + /** Remove given record(s) */ + void removeRecords(List<Record> record); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java index 218fc9f5266..f47f2061000 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java @@ -1,43 +1,41 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.dns; +import java.util.Comparator; import java.util.Objects; /** - * A basic representation of a DNS resource record, containing the record id, type, name and value. + * A basic representation of a DNS resource record, containing the record type, name and data. * * @author mpolden */ -public class Record { +public class Record implements Comparable<Record> { + + private static final Comparator<Record> comparator = Comparator.comparing(Record::type) + .thenComparing(Record::name) + .thenComparing(Record::data); - private final RecordId id; private final Type type; private final RecordName name; private final RecordData data; - public Record(RecordId id, Type type, RecordName name, RecordData data) { - this.id = Objects.requireNonNull(id, "id cannot be null"); + public Record(Type type, RecordName name, RecordData data) { this.type = Objects.requireNonNull(type, "type cannot be null"); this.name = Objects.requireNonNull(name, "name cannot be null"); this.data = Objects.requireNonNull(data, "data cannot be null"); } - /** Unique identifier for this */ - public RecordId id() { - return id; - } - /** DNS type of this */ public Type type() { return type; } - /** Data in this, e.g. IP address for "A" record */ + /** Data in this, e.g. IP address for records of type A */ public RecordData data() { return data; } - /** Name of this, e.g. a FQDN for "A" record */ + /** Name of this, e.g. a FQDN for records of type A */ public RecordName name() { return name; } @@ -57,7 +55,7 @@ public class Record { @Override public String toString() { - return String.format("%s: %s %s -> %s", id, type, name, data); + return String.format("%s %s -> %s", type, name, data); } @Override @@ -65,14 +63,19 @@ public class Record { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Record record = (Record) o; - return Objects.equals(id, record.id) && - type == record.type && - Objects.equals(name, record.name) && - Objects.equals(data, record.data); + return type == record.type && + name.equals(record.name) && + data.equals(record.data); } @Override public int hashCode() { - return Objects.hash(id, type, name, data); + return Objects.hash(type, name, data); } + + @Override + public int compareTo(Record that) { + return comparator.compare(this, that); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordData.java index 6c765efd35a..fddcd85e8af 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordData.java @@ -10,7 +10,7 @@ import java.util.Objects; * * @author mpolden */ -public class RecordData { +public class RecordData implements Comparable<RecordData> { private final String data; @@ -40,7 +40,7 @@ public class RecordData { return data; } - /** Create a new record containing the given data */ + /** Create data containing the given data */ public static RecordData from(String data) { return new RecordData(data); } @@ -50,4 +50,9 @@ public class RecordData { return from(data.endsWith(".") ? data : data + "."); } + @Override + public int compareTo(RecordData that) { + return this.data.compareTo(that.data); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java deleted file mode 100644 index ed628f0c827..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.dns; - -import java.util.Objects; - -/** - * Unique identifier for a resource record. - * - * @author mpolden - */ -public class RecordId { - - private final String id; - - public RecordId(String id) { - this.id = id; - } - - public String asString() { - return id; - } - - @Override - public String toString() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RecordId recordId = (RecordId) o; - return id.equals(recordId.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordName.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordName.java index d3abad9fb62..f092209c1d8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordName.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordName.java @@ -4,11 +4,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import java.util.Objects; /** - * Represents the name field of a DNS record (NAME). This is typically a FQDN. + * Represents the name field of a DNS record (NAME). * * @author mpolden */ -public class RecordName { +public class RecordName implements Comparable<RecordName> { private final String name; @@ -20,6 +20,16 @@ public class RecordName { return name; } + /** Returns whether this is a fully qualified domain name (ends in trailing dot) */ + public boolean isFqdn() { + return name.endsWith("."); + } + + /** Returns this as a fully qualified domain name (ends in trailing dot) */ + public RecordName asFqdn() { + return isFqdn() ? this : new RecordName(name + "."); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -42,4 +52,13 @@ public class RecordName { return new RecordName(name); } + public static RecordName fqdn(String name) { + return from(name).asFqdn(); + } + + @Override + public int compareTo(RecordName that) { + return this.name.compareTo(that.name); + } + } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 23d2bfc33d4..034f96b9445 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -124,12 +124,6 @@ <version>${project.version}</version> </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.6</version> - </dependency> - <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 30c99288d51..a5e952b76c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -37,7 +37,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; @@ -498,16 +497,15 @@ public class ApplicationController { records.forEach(record -> { // Ensure that the existing record points to the correct rotation if ( ! record.data().equals(rotationName)) { - nameService.updateRecord(record.id(), rotationName); - log.info("Updated mapping for record ID " + record.id().asString() + ": '" + dnsName - + "' -> '" + rotation.name() + "'"); + nameService.updateRecord(record, rotationName); + log.info("Updated mapping for record '" + record + "': '" + dnsName + + "' -> '" + rotation.name() + "'"); } }); if (records.isEmpty()) { - RecordId id = nameService.createCname(RecordName.from(dnsName), rotationName); - log.info("Registered mapping with record ID " + id.asString() + ": '" + dnsName + "' -> '" - + rotation.name() + "'"); + Record record = nameService.createCname(RecordName.from(dnsName), rotationName); + log.info("Registered mapping as record '" + record + "'"); } } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to register CNAME", e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index a0b4a888727..131164c5c5e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.google.common.hash.Hashing; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import org.apache.commons.codec.digest.DigestUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -42,7 +42,7 @@ public class ApplicationPackage { */ public ApplicationPackage(byte[] zippedContent) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.contentHash = DigestUtils.shaHex(zippedContent); + this.contentHash = Hashing.sha1().hashBytes(zippedContent).toString(); this.deploymentSpec = extractFile("deployment.xml", zippedContent).map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty); this.validationOverrides = extractFile("validation-overrides.xml", zippedContent).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); Optional<Inspector> buildMetaObject = extractFileBytes("build-meta.json", zippedContent) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index 9924727edbc..0d1aff82bf2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -28,17 +28,15 @@ public class RoutingPolicy { private final ApplicationId owner; private final ZoneId zone; - private final String recordId; private final HostName alias; private final HostName canonicalName; private final Optional<String> dnsZone; private final Set<RotationName> rotations; - public RoutingPolicy(ApplicationId owner, ZoneId zone, String recordId, HostName alias, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ZoneId zone, HostName alias, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { this.owner = Objects.requireNonNull(owner, "owner must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.recordId = Objects.requireNonNull(recordId, "recordId must be non-null"); this.alias = Objects.requireNonNull(alias, "alias must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); @@ -55,11 +53,6 @@ public class RoutingPolicy { return zone; } - /** The ID of the DNS record identifying this */ - public String recordId() { - return recordId; - } - /** This alias (lhs of a CNAME or ALIAS record) */ public HostName alias() { return alias; @@ -84,24 +77,20 @@ public class RoutingPolicy { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - RoutingPolicy that = (RoutingPolicy) o; - return owner.equals(that.owner) && - zone.equals(that.zone) && - recordId.equals(that.recordId) && - alias.equals(that.alias) && - canonicalName.equals(that.canonicalName) && - dnsZone.equals(that.dnsZone) && - rotations.equals(that.rotations); + RoutingPolicy policy = (RoutingPolicy) o; + return owner.equals(policy.owner) && + zone.equals(policy.zone) && + canonicalName.equals(policy.canonicalName); } @Override public int hashCode() { - return Objects.hash(owner, zone, recordId, alias, canonicalName, dnsZone, rotations); + return Objects.hash(owner, zone, canonicalName); } @Override public String toString() { - return String.format("%s: %s -> %s [rotations: %s%s], owned by %s, in %s", recordId, alias, canonicalName, rotations, + return String.format("%s -> %s [rotations: %s%s], owned by %s, in %s", alias, canonicalName, rotations, dnsZone.map(z -> ", DNS zone: " + z).orElse(""), owner.toShortString(), zone.value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java index 36d83b8ee6a..2fe6af02480 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Performs DNS maintenance tasks such as removing DNS aliases for unassigned rotations. @@ -46,21 +47,22 @@ public class DnsMaintainer extends Maintainer { protected void maintain() { try (RotationLock lock = rotationRepository().lock()) { Map<RotationId, Rotation> unassignedRotations = rotationRepository().availableRotations(lock); - rotationToCheckOf(unassignedRotations.values()).ifPresent(this::removeDnsAlias); + rotationToCheckOf(unassignedRotations.values()).ifPresent(this::removeCname); } } - /** Remove DNS alias for unassigned rotation */ - private void removeDnsAlias(Rotation rotation) { + /** Remove CNAME(s) for unassigned rotation */ + private void removeCname(Rotation rotation) { // When looking up CNAME by data, the data must be a FQDN - nameService.findRecords(Record.Type.CNAME, RecordData.fqdn(rotation.name())).stream() - .filter(DnsMaintainer::canUpdate) - .forEach(record -> { - log.info(String.format("Removing DNS record %s (%s) because it points to the unassigned " + - "rotation %s (%s)", record.id().asString(), - record.name().asString(), rotation.id().asString(), rotation.name())); - nameService.removeRecord(record.id()); - }); + List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordData.fqdn(rotation.name())).stream() + .filter(DnsMaintainer::canUpdate) + .collect(Collectors.toList()); + if (records.isEmpty()) { + return; + } + log.info(String.format("Removing DNS records %s because they point to the unassigned " + + "rotation %s (%s)", records, rotation.id().asString(), rotation.name())); + nameService.removeRecords(records); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index c778b103700..80b03f50ebd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; @@ -60,10 +59,10 @@ public class RoutingPolicyMaintainer extends Maintainer { @Override protected void maintain() { Map<DeploymentId, List<LoadBalancer>> loadBalancers = findLoadBalancers(); + removeObsoleteAliases(loadBalancers); registerCnames(loadBalancers); removeObsoleteCnames(loadBalancers); registerAliases(); - removeObsoleteAliases(loadBalancers); } /** Find all exclusive load balancers in this system, grouped by deployment */ @@ -96,7 +95,7 @@ public class RoutingPolicyMaintainer extends Maintainer { Set<AliasTarget> targets = route.getValue() .stream() .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.alias(), + .map(policy -> new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.zone())) .collect(Collectors.toSet()); @@ -119,7 +118,11 @@ public class RoutingPolicyMaintainer extends Maintainer { Set<RoutingPolicy> policies = new LinkedHashSet<>(db.readRoutingPolicies(application)); for (LoadBalancer loadBalancer : entry.getValue()) { try { - policies.add(registerCname(application, zone, loadBalancer)); + RoutingPolicy policy = registerCname(application, zone, loadBalancer); + if (!policies.add(policy)) { + policies.remove(policy); + policies.add(policy); + } } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " + loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(), @@ -141,17 +144,15 @@ public class RoutingPolicyMaintainer extends Maintainer { throw new IllegalStateException("Found more than 1 CNAME record for " + name.asString() + ": " + existingRecords); } Optional<Record> record = existingRecords.stream().findFirst(); - RecordId id; if (record.isPresent()) { - id = record.get().id(); if (!record.get().data().equals(data)) { - nameService.updateRecord(id, data); + nameService.updateRecord(record.get(), data); } } else { - id = nameService.createCname(name, data); + nameService.createCname(name, data); } - return new RoutingPolicy(application, zone, id.asString(), alias, loadBalancer.hostname(), - loadBalancer.dnsZone(), loadBalancer.rotations()); + return new RoutingPolicy(application, zone, alias, loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancer.rotations()); } /** Remove all DNS records that point to non-existing load balancers */ @@ -167,9 +168,10 @@ public class RoutingPolicyMaintainer extends Maintainer { removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName())); for (RoutingPolicy policy : removalCandidates) { try { - nameService.removeRecord(new RecordId(policy.recordId())); + List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordName.from(policy.alias().value())); + nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove CNAME record with ID '" + policy.recordId() + + log.log(LogLevel.WARNING, "Failed to remove record '" + policy.alias() + "'. Retrying in " + maintenanceInterval()); } } @@ -186,7 +188,7 @@ public class RoutingPolicyMaintainer extends Maintainer { GlobalDnsName dnsName = dnsName(id); try { List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(dnsName.oathDnsName())); - records.stream().map(Record::id).forEach(nameService::removeRecord); + nameService.removeRecords(records); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + dnsName.oathDnsName() + "'. Retrying in " + maintenanceInterval()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 0f43aaece7f..0c79793e6a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -74,7 +74,6 @@ public class CuratorDb { private static final Path applicationRoot = root.append("applications"); private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); - private static final Path legacyRoutingPoliciesRoot = root.append("loadBalancerAliases"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -186,15 +185,7 @@ public class CuratorDb { } public Lock lockRoutingPolicies() { - Set<Version> clusterVersions = cluster().stream() - .map(this::readControllerVersion) - .collect(Collectors.toSet()); - // TODO: Remove this once cluster has completely upgraded once - Path newPath = lockRoot.append("routingPolicies"); - if (clusterVersions.size() > 1 && !curator.exists(newPath)) { - return lock(lockRoot.append("loadBalancerAliases"), defaultLockTimeout); - } - return lock(newPath, defaultLockTimeout); + return lock(lockRoot.append("routingPolicies"), defaultLockTimeout); } // -------------- Helpers ------------------------------------------ @@ -482,17 +473,10 @@ public class CuratorDb { } public Set<RoutingPolicy> readRoutingPolicies() { - List<String> children; - if (curator.exists(routingPoliciesRoot)) { - children = curator.getChildren(routingPoliciesRoot); - curator.delete(legacyRoutingPoliciesRoot); - } else { - children = curator.getChildren(legacyRoutingPoliciesRoot); // TODO: Remove after 7.9 has been released - } - return children.stream() - .map(ApplicationId::fromSerializedForm) - .flatMap(application -> readRoutingPolicies(application).stream()) - .collect(Collectors.toUnmodifiableSet()); + return curator.getChildren(routingPoliciesRoot).stream() + .map(ApplicationId::fromSerializedForm) + .flatMap(application -> readRoutingPolicies(application).stream()) + .collect(Collectors.toUnmodifiableSet()); } public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index ccea83caef4..841aedbf5fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.RotationName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -27,9 +25,6 @@ import java.util.function.Function; public class RoutingPolicySerializer { private static final String routingPoliciesField = "routingPolicies"; - private static final String aliasesField = "aliases"; - private static final String idField = "id"; - private static final String recordIdField = "recordId"; private static final String aliasField = "alias"; private static final String canonicalNameField = "canonicalName"; private static final String zoneField = "zone"; @@ -42,7 +37,6 @@ public class RoutingPolicySerializer { Cursor policyArray = root.setArray(routingPoliciesField); routingPolicies.forEach(policy -> { Cursor policyObject = policyArray.addObject(); - policyObject.setString(recordIdField, policy.recordId()); policyObject.setString(aliasField, policy.alias().value()); policyObject.setString(zoneField, policy.zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); @@ -59,20 +53,11 @@ public class RoutingPolicySerializer { Set<RoutingPolicy> policies = new LinkedHashSet<>(); Cursor root = slime.get(); Cursor field = root.field(routingPoliciesField); - if (!field.valid()) { - field = root.field(aliasesField); // TODO: Remove after 7.9 has been released - } field.traverse((ArrayTraverser) (i, inspect) -> { Set<RotationName> rotations = new LinkedHashSet<>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, rotation) -> rotations.add(RotationName.from(rotation.asString()))); - Inspector recordId = inspect.field(recordIdField); - if (!recordId.valid()) { - recordId = inspect.field(idField); // TODO: Remove after 7.9 has been released - } policies.add(new RoutingPolicy(owner, - // TODO: Remove fallback after 7.13 has been released - optionalField(inspect.field(zoneField), ZoneId::from).orElse(ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName())), - recordId.asString(), + ZoneId.from(inspect.field(zoneField).asString()), HostName.from(inspect.field(aliasField).asString()), HostName.from(inspect.field(canonicalNameField).asString()), optionalField(inspect.field(dnsZoneField), Function.identity()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index a4f9dfeb13f..578e7824913 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -23,7 +22,6 @@ import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -131,7 +129,7 @@ public class DnsMaintainerTest { } } - private Map<RecordId, Set<Record>> records() { + private Set<Record> records() { return tester.controllerTester().nameService().records(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index ce691bf4de0..c3e8e63e805 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -21,7 +21,6 @@ import org.junit.Test; import java.time.Duration; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -31,9 +30,11 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author mortent + * @author mpolden */ public class RoutingPolicyMaintainerTest { @@ -62,8 +63,8 @@ public class RoutingPolicyMaintainerTest { maintainer.maintain(); Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app1--tenant1.global.vespa.oath.cloud")); assertEquals(2, records1.get().size()); - assertEquals("c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud.", records1.get().get(0).data().asString()); - assertEquals("c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud.", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records1.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(1).data().asString()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() @@ -72,29 +73,34 @@ public class RoutingPolicyMaintainerTest { .region("us-central-1") .region("us-east-3") .build(); + int numberOfDeployments = 3; tester.deployCompletely(app1, updatedApplicationPackage, BuildJob.defaultBuildNumber + 1); // Cluster in new deployment is added to the rotation provisionLoadBalancers(app1, 2, rotations); maintainer.maintain(); - assertEquals(3, records1.get().size()); - assertEquals("c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud.", records1.get().get(0).data().asString()); - assertEquals("c0--app1--tenant1.prod.us-east-3.vespa.oath.cloud.", records1.get().get(1).data().asString()); - assertEquals("c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud.", records1.get().get(2).data().asString()); + assertEquals(numberOfDeployments, records1.get().size()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records1.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-east-3.", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(2).data().asString()); - // Another appplication is deployed + // Another application is deployed Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app2--tenant1.global.vespa.oath.cloud")); tester.deployCompletely(app2, applicationPackage); provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0")))); maintainer.maintain(); assertEquals(2, records2.get().size()); - assertEquals("c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud.", records2.get().get(0).data().asString()); - assertEquals("c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud.", records2.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records2.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records2.get().get(1).data().asString()); - // Rotation for app1 is removed + // All rotations for app1 are removed provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap()); maintainer.maintain(); - assertEquals(0, records1.get().size()); + assertEquals(List.of(), records1.get()); + Set<RoutingPolicy> policies = tester.controller().curator().readRoutingPolicies(app1.id()); + assertEquals(clustersPerZone * numberOfDeployments, policies.size()); + assertTrue("Rotation membership is removed from all policies", + policies.stream().allMatch(policy -> policy.rotations().isEmpty())); assertEquals("Rotations for " + app2 + " are not removed", 2, records2.get().size()); } @@ -189,13 +195,8 @@ public class RoutingPolicyMaintainerTest { return tester.controller().curator().readRoutingPolicies(application.id()); } - private List<Record> findAlias(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)); - } - private Set<String> recordNames() { - return tester.controllerTester().nameService().records().values().stream() - .flatMap(Collection::stream) + return tester.controllerTester().nameService().records().stream() .map(Record::name) .map(RecordName::asString) .collect(Collectors.toSet()); @@ -224,8 +225,8 @@ public class RoutingPolicyMaintainerTest { new LoadBalancer("LB-" + i + "-Z-" + zone.value(), application, ClusterSpec.Id.from("c" + i), - HostName.from("loadbalancer-" + i + "-" + application.serializedForm() + - "-zone-" + zone.value()), + HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()), Optional.of("dns-zone-1"), rotations)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 99a2f6406ef..e329aee7fc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -3,16 +3,12 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.RotationName; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Test; -import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -30,14 +26,12 @@ public class RoutingPolicySerializerTest { Set<RotationName> rotations = Set.of(RotationName.from("r1"), RotationName.from("r2")); Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner, ZoneId.from("prod", "us-north-1"), - "record-id-1", HostName.from("my-pretty-alias"), HostName.from("long-and-ugly-name"), Optional.of("zone1"), rotations), new RoutingPolicy(owner, ZoneId.from("prod", "us-north-2"), - "record-id-2", HostName.from("my-pretty-alias-2"), HostName.from("long-and-ugly-name-2"), Optional.empty(), @@ -46,40 +40,4 @@ public class RoutingPolicySerializerTest { assertEquals(loadBalancers, serialized); } - // TODO: Remove after 7.10 has been released - @Test - public void test_serialization_old_format() { - String json = "{\n" + - " \"aliases\": [\n" + - " {\n" + - " \"id\": \"record-id-1\",\n" + - " \"alias\": \"my-pretty-alias\",\n" + - " \"canonicalName\": \"long-and-ugly-name\"" + - " },\n" + - " {\n" + - " \"id\": \"record-id-2\",\n" + - " \"alias\": \"my-pretty-alias-2\",\n" + - " \"canonicalName\": \"long-and-ugly-name-2\"" + - " }\n" + - " ]\n" + - "}\n"; - ApplicationId owner = ApplicationId.defaultId(); - Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner, - ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()), - "record-id-1", - HostName.from("my-pretty-alias"), - HostName.from("long-and-ugly-name"), - Optional.empty(), - Collections.emptySet()), - new RoutingPolicy(owner, - ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()), - "record-id-2", - HostName.from("my-pretty-alias-2"), - HostName.from("long-and-ugly-name-2"), - Optional.empty(), - Collections.emptySet())); - RoutingPolicySerializer serializer = new RoutingPolicySerializer(); - assertEquals(loadBalancers, serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json))); - } - } diff --git a/document/src/tests/gtest_runner.cpp b/document/src/tests/gtest_runner.cpp index efe108c2042..fed042acceb 100644 --- a/document/src/tests/gtest_runner.cpp +++ b/document/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("document_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/document/src/vespa/document/base/globalid.h b/document/src/vespa/document/base/globalid.h index dd1d061dbf7..16781145bf0 100644 --- a/document/src/vespa/document/base/globalid.h +++ b/document/src/vespa/document/base/globalid.h @@ -89,6 +89,8 @@ public: */ explicit GlobalId(const void *gid) { set(gid); } + GlobalId(const GlobalId &rhs) = default; + /** * Implements the assignment operator. * diff --git a/document/src/vespa/document/datatype/arraydatatype.h b/document/src/vespa/document/datatype/arraydatatype.h index 55b5af184dd..275878e5dc2 100644 --- a/document/src/vespa/document/datatype/arraydatatype.h +++ b/document/src/vespa/document/datatype/arraydatatype.h @@ -27,6 +27,7 @@ public: void print(std::ostream&, bool verbose, const std::string& indent) const override; bool operator==(const DataType& other) const override; ArrayDataType* clone() const override { return new ArrayDataType(*this); } + ArrayDataType &operator=(const ArrayDataType &rhs) = default; void onBuildFieldPath(FieldPath & path, vespalib::stringref remainFieldName) const override; DECLARE_IDENTIFIABLE(ArrayDataType); diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.h b/document/src/vespa/document/update/arithmeticvalueupdate.h index 1f9e33a3c3a..697798a7275 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.h +++ b/document/src/vespa/document/update/arithmeticvalueupdate.h @@ -55,6 +55,8 @@ public: _operator(update._operator), _operand(update._operand) {} + ArithmeticValueUpdate &operator=(const ArithmeticValueUpdate &rhs) = default; + bool operator==(const ValueUpdate& other) const override; Operator getOperator() const { return _operator; } diff --git a/document/src/vespa/document/update/clearvalueupdate.h b/document/src/vespa/document/update/clearvalueupdate.h index 2ac47deaaf9..49842653e65 100644 --- a/document/src/vespa/document/update/clearvalueupdate.h +++ b/document/src/vespa/document/update/clearvalueupdate.h @@ -17,6 +17,7 @@ public: typedef std::unique_ptr<ClearValueUpdate> UP; ClearValueUpdate() : ValueUpdate() {} ClearValueUpdate(const ClearValueUpdate& update) : ValueUpdate(update) {} + ClearValueUpdate &operator=(const ClearValueUpdate &rhs) = default; bool operator==(const ValueUpdate& other) const override; void checkCompatibility(const Field& field) const override; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index c34291c634f..00c16c16fec 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -567,7 +567,7 @@ RoutableFactories60::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf // Doing an explicit move here to force converting result to an rvalue. // This is done automatically in GCC >= 5. - return std::move(reply); + return reply; } bool diff --git a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt index 6c19b13db7e..e511b23c5ec 100644 --- a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(eval_tensor_add_operation_test_app TEST tensor_add_operation_test.cpp DEPENDS vespaeval + gtest ) vespa_add_test(NAME eval_tensor_add_operation_test_app COMMAND eval_tensor_add_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp b/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp index 5ac1c503b5d..4c92dc717a7 100644 --- a/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp +++ b/eval/src/tests/tensor/tensor_add_operation/tensor_add_operation_test.cpp @@ -4,7 +4,7 @@ #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> using vespalib::eval::Value; using vespalib::eval::TensorSpec; @@ -18,10 +18,19 @@ assertAdd(const TensorSpec &source, const TensorSpec &arg, const TensorSpec &exp auto argTensor = makeTensor<Tensor>(arg); auto resultTensor = sourceTensor->add(*argTensor); auto actual = resultTensor->toSpec(); - EXPECT_EQUAL(actual, expected); + EXPECT_EQ(actual, expected); } -TEST("require that cells can be added to a sparse tensor") +void +assertNullTensor(const TensorSpec &source, const TensorSpec &arg) +{ + auto sourceTensor = makeTensor<Tensor>(source); + auto argTensor = makeTensor<Tensor>(arg); + auto resultTensor = sourceTensor->add(*argTensor); + EXPECT_FALSE(resultTensor); +} + +TEST(TensorAddTest, cells_can_be_added_to_a_sparse_tensor) { assertAdd(TensorSpec("tensor(x{},y{})") .add({{"x","a"},{"y","b"}}, 2) @@ -35,4 +44,42 @@ TEST("require that cells can be added to a sparse tensor") .add({{"x","e"},{"y","f"}}, 7)); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST(TensorAddTest, cells_can_be_added_to_a_mixed_tensor) +{ + assertAdd(TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3) + .add({{"x","b"},{"y",0}}, 4) + .add({{"x","b"},{"y",1}}, 5), + TensorSpec("tensor(x{},y[2])") + .add({{"x","b"},{"y",0}}, 6) + .add({{"x","b"},{"y",1}}, 7) + .add({{"x","c"},{"y",0}}, 8) + .add({{"x","c"},{"y",1}}, 9), + TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3) + .add({{"x","b"},{"y",0}}, 6) + .add({{"x","b"},{"y",1}}, 7) + .add({{"x","c"},{"y",0}}, 8) + .add({{"x","c"},{"y",1}}, 9)); +} + +TEST(TensorAddTest, cells_can_be_added_to_empty_mixed_tensor) +{ + assertAdd(TensorSpec("tensor(x{},y[2])"), + TensorSpec("tensor(x{},y[2])") + .add({{"x","b"},{"y",0}}, 6) + .add({{"x","b"},{"y",1}}, 7), + TensorSpec("tensor(x{},y[2])") + .add({{"x","b"},{"y",0}}, 6) + .add({{"x","b"},{"y",1}}, 7)); +} + +TEST(TensorAddTest, tensors_of_different_types_cannot_be_added_together) +{ + assertNullTensor(TensorSpec("tensor(x{},y[2])"), TensorSpec("tensor(x{},y{})")); + assertNullTensor(TensorSpec("tensor(x{},y[2])"), TensorSpec("tensor(x{},y[3])")); +} + +GTEST_MAIN_RUN_ALL_TESTS diff --git a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt index b0ffa48eb3b..481a0aac7fb 100644 --- a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(eval_tensor_modify_operation_test_app TEST tensor_modify_operation_test.cpp DEPENDS vespaeval + gtest ) vespa_add_test(NAME eval_tensor_modify_operation_test_app COMMAND eval_tensor_modify_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp b/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp index 5cacff1881b..31f17b3eed2 100644 --- a/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp +++ b/eval/src/tests/tensor/tensor_modify_operation/tensor_modify_operation_test.cpp @@ -1,28 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/tensor/cell_values.h> +#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/stringfmt.h> using vespalib::eval::Value; using vespalib::eval::TensorSpec; using vespalib::tensor::test::makeTensor; using namespace vespalib::tensor; -namespace { - -double -replace(double, double b) -{ - return b; -} - -} - void checkUpdate(const TensorSpec &source, const TensorSpec &update, const TensorSpec &expect) { @@ -30,57 +21,91 @@ checkUpdate(const TensorSpec &source, const TensorSpec &update, const TensorSpec auto updateTensor = makeTensor<SparseTensor>(update); const CellValues cellValues(*updateTensor); - auto actualTensor = sourceTensor->modify(replace, cellValues); + auto actualTensor = sourceTensor->modify(vespalib::eval::operation::Add::f, cellValues); auto actual = actualTensor->toSpec(); auto expectTensor = makeTensor<Tensor>(expect); auto expectPadded = expectTensor->toSpec(); - EXPECT_EQUAL(actual, expectPadded); + EXPECT_EQ(actual, expectPadded); } -TEST("require that sparse tensors can be modified") { +TEST(TensorModifyTest, sparse_tensors_can_be_modified) +{ checkUpdate(TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 11) - .add({{"x","9"},{"y","9"}}, 11), + .add({{"x","8"},{"y","9"}}, 11) + .add({{"x","9"},{"y","9"}}, 11), TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2), + .add({{"x","8"},{"y","9"}}, 2), TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2) - .add({{"x","9"},{"y","9"}}, 11)); + .add({{"x","8"},{"y","9"}}, 13) + .add({{"x","9"},{"y","9"}}, 11)); } -TEST("require that dense tensors can be modified") { +TEST(TensorModifyTest, dense_tensors_can_be_modified) +{ checkUpdate(TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 11) - .add({{"x",9},{"y",9}}, 11), + .add({{"x",8},{"y",9}}, 11) + .add({{"x",9},{"y",9}}, 11), TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2), + .add({{"x","8"},{"y","9"}}, 2), TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 2) - .add({{"x",9},{"y",9}}, 11)); + .add({{"x",8},{"y",9}}, 13) + .add({{"x",9},{"y",9}}, 11)); +} + +TEST(TensorModifyTest, mixed_tensors_can_be_modified) +{ + checkUpdate(TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3) + .add({{"x","b"},{"y",0}}, 4) + .add({{"x","b"},{"y",1}}, 5), + TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","0"}}, 6) + .add({{"x","b"},{"y","1"}}, 7), + TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 8) + .add({{"x","a"},{"y",1}}, 3) + .add({{"x","b"},{"y",0}}, 4) + .add({{"x","b"},{"y",1}}, 12)); } -TEST("require that sparse tensors ignore updates to missing cells") { +TEST(TensorModifyTest, sparse_tensors_ignore_updates_to_missing_cells) +{ checkUpdate(TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 11) - .add({{"x","9"},{"y","9"}}, 11), + .add({{"x","8"},{"y","9"}}, 11) + .add({{"x","9"},{"y","9"}}, 11), TensorSpec("tensor(x{},y{})") - .add({{"x","7"},{"y","9"}}, 2) - .add({{"x","8"},{"y","9"}}, 2), + .add({{"x","7"},{"y","9"}}, 2) + .add({{"x","8"},{"y","9"}}, 2), TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2) - .add({{"x","9"},{"y","9"}}, 11)); + .add({{"x","8"},{"y","9"}}, 13) + .add({{"x","9"},{"y","9"}}, 11)); } -TEST("require that dense tensors ignore updates to out of range cells") { +TEST(TensorModifyTest, dense_tensors_ignore_updates_to_out_of_range_cells) +{ checkUpdate(TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 11) - .add({{"x",9},{"y",9}}, 11), + .add({{"x",8},{"y",9}}, 11) + .add({{"x",9},{"y",9}}, 11), TensorSpec("tensor(x{},y{})") - .add({{"x","8"},{"y","9"}}, 2) - .add({{"x","10"},{"y","9"}}, 2), + .add({{"x","8"},{"y","9"}}, 2) + .add({{"x","10"},{"y","9"}}, 2), TensorSpec("tensor(x[10],y[10])") - .add({{"x",8},{"y",9}}, 2) - .add({{"x",9},{"y",9}}, 11)); + .add({{"x",8},{"y",9}}, 13) + .add({{"x",9},{"y",9}}, 11)); +} + +TEST(TensorModifyTest, mixed_tensors_ignore_updates_to_missing_or_out_of_range_cells) +{ + checkUpdate(TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3), + TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","2"}}, 4) + .add({{"x","c"},{"y","0"}}, 5), + TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3)); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS diff --git a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt index 8dfb8181f2b..ffc71563107 100644 --- a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(eval_tensor_remove_operation_test_app TEST tensor_remove_operation_test.cpp DEPENDS vespaeval + gtest ) vespa_add_test(NAME eval_tensor_remove_operation_test_app COMMAND eval_tensor_remove_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp b/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp index 8b0c44a6e06..df21b46691d 100644 --- a/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp +++ b/eval/src/tests/tensor/tensor_remove_operation/tensor_remove_operation_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/sparse/sparse_tensor.h> #include <vespa/eval/tensor/test/test_utils.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> using vespalib::eval::Value; using vespalib::eval::TensorSpec; @@ -19,10 +19,10 @@ assertRemove(const TensorSpec &source, const TensorSpec &arg, const TensorSpec & auto argTensor = makeTensor<SparseTensor>(arg); auto resultTensor = sourceTensor->remove(CellValues(*argTensor)); auto actual = resultTensor->toSpec(); - EXPECT_EQUAL(actual, expected); + EXPECT_EQ(actual, expected); } -TEST("require that cells can be removed from a sparse tensor") +TEST(TensorRemoveTest, cells_can_be_removed_from_a_sparse_tensor) { assertRemove(TensorSpec("tensor(x{},y{})") .add({{"x","a"},{"y","b"}}, 2) @@ -34,7 +34,7 @@ TEST("require that cells can be removed from a sparse tensor") .add({{"x","a"},{"y","b"}}, 2)); } -TEST("require that all cells can be removed from a sparse tensor") +TEST(TensorRemoveTest, all_cells_can_be_removed_from_a_sparse_tensor) { assertRemove(TensorSpec("tensor(x{},y{})") .add({{"x","a"},{"y","b"}}, 2), @@ -43,4 +43,53 @@ TEST("require that all cells can be removed from a sparse tensor") TensorSpec("tensor(x{},y{})")); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST(TensorRemoveTest, cells_can_be_removed_from_a_mixed_tensor) +{ + assertRemove(TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3) + .add({{"x","b"},{"y",0}}, 4) + .add({{"x","b"},{"y",1}}, 5), + TensorSpec("tensor(x{})") + .add({{"x","b"}}, 1) + .add({{"x","c"}}, 1), + TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3)); + + assertRemove(TensorSpec("tensor(x{},y{},z[2])") + .add({{"x","a"},{"y","c"},{"z",0}}, 2) + .add({{"x","a"},{"y","c"},{"z",1}}, 3) + .add({{"x","b"},{"y","c"},{"z",0}}, 4) + .add({{"x","b"},{"y","c"},{"z",1}}, 5), + TensorSpec("tensor(x{},y{})") + .add({{"x","b"},{"y","c"}}, 1) + .add({{"x","c"},{"y","c"}}, 1), + TensorSpec("tensor(x{},y{},z[2])") + .add({{"x","a"},{"y","c"},{"z",0}}, 2) + .add({{"x","a"},{"y","c"},{"z",1}}, 3)); + + assertRemove(TensorSpec("tensor(x{},y[1],z[2])") + .add({{"x","a"},{"y",0},{"z",0}}, 2) + .add({{"x","a"},{"y",0},{"z",1}}, 3) + .add({{"x","b"},{"y",0},{"z",0}}, 4) + .add({{"x","b"},{"y",0},{"z",1}}, 5), + TensorSpec("tensor(x{})") + .add({{"x","b"}}, 1) + .add({{"x","c"}}, 1), + TensorSpec("tensor(x{},y[1],z[2])") + .add({{"x","a"},{"y",0},{"z",0}}, 2) + .add({{"x","a"},{"y",0},{"z",1}}, 3)); +} + +TEST(TensorRemoveTest, all_cells_can_be_removed_from_a_mixed_tensor) +{ + assertRemove(TensorSpec("tensor(x{},y[2])") + .add({{"x","a"},{"y",0}}, 2) + .add({{"x","a"},{"y",1}}, 3), + TensorSpec("tensor(x{})") + .add({{"x","a"}}, 1), + TensorSpec("tensor(x{},y[2])")); +} + +GTEST_MAIN_RUN_ALL_TESTS diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index 2b3c5679488..c8c9a82d267 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -49,7 +49,7 @@ const Value &to_value(std::unique_ptr<SimpleTensor> tensor, Stash &stash) { Value::UP to_value(std::unique_ptr<SimpleTensor> tensor) { if (tensor->type().is_tensor()) { - return std::move(tensor); + return tensor; } if (tensor->type().is_double()) { return std::make_unique<DoubleValue>(tensor->as_double()); diff --git a/eval/src/vespa/eval/tensor/cell_values.h b/eval/src/vespa/eval/tensor/cell_values.h index dabdcde3294..4b8fd33376a 100644 --- a/eval/src/vespa/eval/tensor/cell_values.h +++ b/eval/src/vespa/eval/tensor/cell_values.h @@ -23,6 +23,10 @@ public: void accept(TensorVisitor &visitor) const { _tensor.accept(visitor); } + + eval::TensorSpec toSpec() const { + return _tensor.toSpec(); + } }; } diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 20296ac5032..554953288e1 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -91,7 +91,7 @@ Value::UP to_value(std::unique_ptr<Tensor> tensor) { return std::make_unique<ErrorValue>(); } if (tensor->type().is_tensor()) { - return std::move(tensor); + return tensor; } return std::make_unique<DoubleValue>(tensor->as_double()); } diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index 7eebff1f010..c182c09c6b0 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -2,10 +2,10 @@ #pragma once +#include "sparse_tensor_address_ref.h" #include <vespa/eval/tensor/cell_function.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/tensor_address.h> -#include "sparse_tensor_address_ref.h" #include <vespa/eval/tensor/types.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/string.h> diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 9df59a63873..a982a4b0fe1 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wrapped_simple_tensor.h" +#include "cell_values.h" #include "tensor_address_builder.h" #include "tensor_visitor.h" +#include "wrapped_simple_tensor.h" #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/vespalib/util/stringfmt.h> @@ -12,6 +13,9 @@ LOG_SETUP(".eval.tensor.wrapped_simple_tensor"); namespace vespalib::tensor { +using eval::SimpleTensor; +using eval::TensorSpec; + bool WrappedSimpleTensor::equals(const Tensor &arg) const { @@ -77,22 +81,103 @@ WrappedSimpleTensor::reduce(join_fun_t, const std::vector<vespalib::string> &) c LOG_ABORT("should not be reached"); } +namespace { + +TensorSpec::Address +convertToOnlyMappedDimensions(const TensorSpec::Address &address) +{ + TensorSpec::Address result; + for (const auto &elem : address) { + if (elem.second.is_indexed()) { + result.emplace(std::make_pair(elem.first, + TensorSpec::Label(vespalib::make_string("%zu", elem.second.index)))); + } else { + result.emplace(elem); + } + } + return result; +} + +} + std::unique_ptr<Tensor> -WrappedSimpleTensor::modify(join_fun_t, const CellValues &) const +WrappedSimpleTensor::modify(join_fun_t op, const CellValues &cellValues) const { - LOG_ABORT("should not be reached"); + TensorSpec oldTensor = toSpec(); + TensorSpec toModify = cellValues.toSpec(); + TensorSpec result(type().to_spec()); + + for (const auto &cell : oldTensor.cells()) { + TensorSpec::Address mappedAddress = convertToOnlyMappedDimensions(cell.first); + auto itr = toModify.cells().find(mappedAddress); + if (itr != toModify.cells().end()) { + result.add(cell.first, op(cell.second, itr->second)); + } else { + result.add(cell.first, cell.second); + } + } + return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); } std::unique_ptr<Tensor> -WrappedSimpleTensor::add(const Tensor &) const +WrappedSimpleTensor::add(const Tensor &arg) const { - LOG_ABORT("should not be reached"); + const auto *rhs = dynamic_cast<const WrappedSimpleTensor *>(&arg); + if (!rhs || type() != rhs->type()) { + return Tensor::UP(); + } + + TensorSpec oldTensor = toSpec(); + TensorSpec argTensor = rhs->toSpec(); + TensorSpec result(type().to_spec()); + for (const auto &cell : oldTensor.cells()) { + auto argItr = argTensor.cells().find(cell.first); + if (argItr != argTensor.cells().end()) { + result.add(argItr->first, argItr->second); + } else { + result.add(cell.first, cell.second); + } + } + for (const auto &cell : argTensor.cells()) { + auto resultItr = result.cells().find(cell.first); + if (resultItr == result.cells().end()) { + result.add(cell.first, cell.second); + } + } + return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); +} + +namespace { + +TensorSpec::Address +extractMappedDimensions(const TensorSpec::Address &address) +{ + TensorSpec::Address result; + for (const auto &elem : address) { + if (elem.second.is_mapped()) { + result.emplace(elem); + } + } + return result; +} + } std::unique_ptr<Tensor> -WrappedSimpleTensor::remove(const CellValues &) const +WrappedSimpleTensor::remove(const CellValues &cellAddresses) const { - LOG_ABORT("should not be reached"); + TensorSpec oldTensor = toSpec(); + TensorSpec toRemove = cellAddresses.toSpec(); + TensorSpec result(type().to_spec()); + + for (const auto &cell : oldTensor.cells()) { + TensorSpec::Address mappedAddress = extractMappedDimensions(cell.first); + auto itr = toRemove.cells().find(mappedAddress); + if (itr == toRemove.cells().end()) { + result.add(cell.first, cell.second); + } + } + return std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(result)); } -} // namespace vespalib::tensor +} diff --git a/fastos/src/vespa/fastos/unix_time.h b/fastos/src/vespa/fastos/unix_time.h index adb952f1f68..82254c98f8d 100644 --- a/fastos/src/vespa/fastos/unix_time.h +++ b/fastos/src/vespa/fastos/unix_time.h @@ -57,6 +57,8 @@ public: SetMilliSecs(s * 1000.0); } + FastOS_UNIX_Time(const FastOS_UNIX_Time &rhs) = default; + operator fastos::TimeStamp () { return fastos::TimeStamp(_time); } diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 5e48390a297..b70b3fa8b01 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -3,11 +3,14 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fnet/fnet.h> #include <vespa/vespalib/net/server_socket.h> +#include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/stringfmt.h> using namespace vespalib; +int short_time = 20; // ms + struct BlockingHostResolver : public AsyncResolver::HostResolver { AsyncResolver::SimpleHostResolver resolver; Gate caller; @@ -33,6 +36,43 @@ AsyncResolver::SP make_resolver(AsyncResolver::HostResolver::SP host_resolver) { //----------------------------------------------------------------------------- +struct BlockingCryptoSocket : public CryptoSocket { + SocketHandle socket; + Gate &handshake_work_enter; + Gate &handshake_work_exit; + Gate &handshake_socket_deleted; + BlockingCryptoSocket(SocketHandle s, Gate &hs_work_enter, Gate &hs_work_exit, Gate &hs_socket_deleted) + : socket(std::move(s)), handshake_work_enter(hs_work_enter), handshake_work_exit(hs_work_exit), + handshake_socket_deleted(hs_socket_deleted) {} + ~BlockingCryptoSocket() override { + handshake_socket_deleted.countDown(); + } + int get_fd() const override { return socket.get(); } + HandshakeResult handshake() override { return HandshakeResult::NEED_WORK; } + void do_handshake_work() override { + handshake_work_enter.countDown(); + handshake_work_exit.await(); + } + size_t min_read_buffer_size() const override { return 1; } + ssize_t read(char *buf, size_t len) override { return socket.read(buf, len); } + ssize_t drain(char *, size_t) override { return 0; } + ssize_t write(const char *buf, size_t len) override { return socket.write(buf, len); } + ssize_t flush() override { return 0; } + ssize_t half_close() override { return socket.half_close(); } +}; + +struct BlockingCryptoEngine : public CryptoEngine { + Gate handshake_work_enter; + Gate handshake_work_exit; + Gate handshake_socket_deleted; + CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool) override { + return std::make_unique<BlockingCryptoSocket>(std::move(socket), + handshake_work_enter, handshake_work_exit, handshake_socket_deleted); + } +}; + +//----------------------------------------------------------------------------- + struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { FNET_SimplePacketStreamer streamer; FastOS_ThreadPool pool; @@ -50,6 +90,12 @@ struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { { transport.Start(&pool); } + TransportFixture(CryptoEngine::SP crypto) + : streamer(nullptr), pool(128 * 1024), transport(crypto, 1), + conn_lost(), conn_deleted() + { + transport.Start(&pool); + } HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context) override { ASSERT_TRUE(packet->GetCommand() == FNET_ControlPacket::FNET_CMD_CHANNEL_LOST); conn_lost.countDown(); @@ -83,19 +129,19 @@ TEST_MT_FFF("require that normal connect works", 2, FNET_Connection *conn = f2.connect(spec); TEST_BARRIER(); conn->Owner()->Close(conn); - EXPECT_TRUE(f2.conn_lost.await(60000)); - EXPECT_TRUE(!f2.conn_deleted.await(20)); + f2.conn_lost.await(); + EXPECT_TRUE(!f2.conn_deleted.await(short_time)); conn->SubRef(); - EXPECT_TRUE(f2.conn_deleted.await(60000)); + f2.conn_deleted.await(); } } TEST_FF("require that bogus connect fail asynchronously", TransportFixture(), TimeBomb(60)) { FNET_Connection *conn = f1.connect("invalid"); - EXPECT_TRUE(f1.conn_lost.await(60000)); - EXPECT_TRUE(!f1.conn_deleted.await(20)); + f1.conn_lost.await(); + EXPECT_TRUE(!f1.conn_deleted.await(short_time)); conn->SubRef(); - EXPECT_TRUE(f1.conn_deleted.await(60000)); + f1.conn_deleted.await(); } TEST_MT_FFFF("require that async close can be called before async resolve completes", 2, @@ -110,13 +156,38 @@ TEST_MT_FFFF("require that async close can be called before async resolve comple FNET_Connection *conn = f3.connect(spec); f2->wait_for_caller(); conn->Owner()->Close(conn); - EXPECT_TRUE(f3.conn_lost.await(60000)); + f3.conn_lost.await(); f2->release_caller(); - EXPECT_TRUE(!f3.conn_deleted.await(20)); + EXPECT_TRUE(!f3.conn_deleted.await(short_time)); conn->SubRef(); - EXPECT_TRUE(f3.conn_deleted.await(60000)); + f3.conn_deleted.await(); f1.shutdown(); } } +TEST_MT_FFFF("require that async close during async do_handshake_work works", 2, + ServerSocket("tcp/0"), std::shared_ptr<BlockingCryptoEngine>(new BlockingCryptoEngine()), + TransportFixture(f2), TimeBomb(60)) +{ + if (thread_id == 0) { + SocketHandle socket = f1.accept(); + EXPECT_TRUE(socket.valid()); + TEST_BARRIER(); // #1 + } else { + vespalib::string spec = make_string("tcp/localhost:%d", f1.address().port()); + FNET_Connection *conn = f3.connect(spec); + f2->handshake_work_enter.await(); + conn->Owner()->Close(conn, false); + conn = nullptr; // ref given away + f3.conn_lost.await(); + TEST_BARRIER(); // #1 + // verify that pending work keeps relevant objects alive + EXPECT_TRUE(!f3.conn_deleted.await(short_time)); + EXPECT_TRUE(!f2->handshake_socket_deleted.await(short_time)); + f2->handshake_work_exit.countDown(); + f3.conn_deleted.await(); + f2->handshake_socket_deleted.await(); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp index 31aec84afd5..2441ea6eaa0 100644 --- a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp +++ b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp @@ -89,13 +89,13 @@ void perform_test(size_t thread_id, Client &client, Result &result) { req = client.orb.AllocRPCRequest(req); req->SetMethodName("inc"); req->GetParams()->AddInt64(seq); - target->InvokeSync(req, 60.0); + target->InvokeSync(req, 300.0); ASSERT_TRUE(req->CheckReturnTypes("l")); uint64_t ret = req->GetReturn()->GetValue(0)._intval64; EXPECT_EQUAL(ret, seq + 1); seq = ret; }; - size_t loop_cnt = 128; + size_t loop_cnt = 8; BenchmarkTimer::benchmark(invoke, invoke, 0.5); BenchmarkTimer timer(1.5); while (timer.has_budget()) { diff --git a/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp b/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp index b3e613de93f..0e5f4712e61 100644 --- a/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp +++ b/fnet/src/tests/frt/parallel_rpc/tls_rpc_bench.cpp @@ -59,6 +59,22 @@ struct StartCmp { } }; +vespalib::string get_prefix(const std::vector<TimeTracer::Record> &stats, size_t idx) { + vespalib::string prefix; + TimeTracer::Record self = stats[idx]; + while (idx-- > 0) { + if (stats[idx].thread_id == self.thread_id) { + if (stats[idx].stop > self.start) { + prefix.append("..."); + } + } + } + if (!prefix.empty()) { + prefix.append(" "); + } + return prefix; +} + void benchmark_rpc(Fixture &fixture, bool reconnect) { uint64_t seq = 0; FRT_Target *target = fixture.connect(); @@ -95,24 +111,25 @@ void benchmark_rpc(Fixture &fixture, bool reconnect) { ASSERT_TRUE(stats.size() > 0); std::sort(stats.begin(), stats.end(), StartCmp()); fprintf(stderr, "===== time line BEGIN =====\n"); - for (const auto &entry: stats) { + for (size_t i = 0; i < stats.size(); ++i) { + const auto &entry = stats[i]; double abs_start = std::chrono::duration<double, std::milli>(entry.start - med_sample.start).count(); double abs_stop = std::chrono::duration<double, std::milli>(entry.stop - med_sample.start).count(); - fprintf(stderr, "[%g, %g] [%u:%s] %g ms\n", abs_start, abs_stop, entry.thread_id, entry.tag_name().c_str(), entry.ms_duration()); + fprintf(stderr, "%s[%g, %g] [%u:%s] %g ms\n", get_prefix(stats, i).c_str(), abs_start, abs_stop, + entry.thread_id, entry.tag_name().c_str(), entry.ms_duration()); } fprintf(stderr, "===== time line END =====\n"); - std::sort(stats.begin(), stats.end(), DurationCmp()); - ASSERT_TRUE(stats.back().tag_id == req_tag.id()); - double rest_ms = stats.back().ms_duration(); - while (!stats.empty() && stats.back().ms_duration() > 1.0) { - const auto &entry = stats.back(); + std::vector<TimeTracer::Record> high_duration; + for (const auto &entry: stats) { + if (entry.ms_duration() > 1.0) { + high_duration.push_back(entry); + } + } + for (const auto &entry: high_duration) { if (entry.tag_id != req_tag.id()) { fprintf(stderr, "WARNING: high duration: [%u:%s] %g ms\n", entry.thread_id, entry.tag_name().c_str(), entry.ms_duration()); - rest_ms -= entry.ms_duration(); } - stats.pop_back(); } - fprintf(stderr, "INFO: total non-critical overhead: %g ms\n", rest_ms); } TEST_F("^^^-- rpc with null encryption", Fixture(null_crypto)) { diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 1a5834d6e96..74a7d19387c 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -51,6 +51,26 @@ SyncPacket::Free() _cond.notify_one(); } } + + +struct DoHandshakeWork : vespalib::Executor::Task { + FNET_Connection *conn; + vespalib::CryptoSocket *socket; + DoHandshakeWork(FNET_Connection *conn_in, vespalib::CryptoSocket *socket_in) + : conn(conn_in), socket(socket_in) + { + conn->AddRef(); + } + void run() override { + socket->do_handshake_work(); + conn->Owner()->handshake_act(conn, false); + conn = nullptr; // ref given away above + } + ~DoHandshakeWork() { + assert(conn == nullptr); + } +}; + } @@ -216,6 +236,9 @@ bool FNET_Connection::handshake() { bool broken = false; + if (_flags._handshake_work_pending) { + return !broken; + } switch (_socket->handshake()) { case vespalib::CryptoSocket::HandshakeResult::FAIL: LOG(debug, "Connection(%s): handshake failed with peer %s", GetSpec(), GetPeerSpec().c_str()); @@ -247,6 +270,12 @@ FNET_Connection::handshake() EnableReadEvent(false); EnableWriteEvent(true); break; + case vespalib::CryptoSocket::HandshakeResult::NEED_WORK: + EnableReadEvent(false); + EnableWriteEvent(false); + assert(!_flags._handshake_work_pending); + _flags._handshake_work_pending = true; + Owner()->owner().post_or_perform(std::make_unique<DoHandshakeWork>(this, _socket.get())); } return !broken; } @@ -552,6 +581,13 @@ FNET_Connection::handle_add_event() return (_socket && (_socket->get_fd() >= 0)); } +bool +FNET_Connection::handle_handshake_act() +{ + assert(_flags._handshake_work_pending); + _flags._handshake_work_pending = false; + return ((_state == FNET_CONNECTING) && handshake()); +} void FNET_Connection::SetCleanupHandler(FNET_IConnectionCleanupHandler *handler) @@ -695,7 +731,9 @@ FNET_Connection::Close() detach_selector(); SetState(FNET_CLOSED); _ioc_socket_fd = -1; - _socket.reset(); + if (!_flags._handshake_work_pending) { + _socket.reset(); + } } diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 40f826243bf..b52d0147ce1 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -71,13 +71,15 @@ private: _inCallback(false), _callbackWait(false), _discarding(false), - _framed(false) + _framed(false), + _handshake_work_pending(false) { } bool _gotheader; bool _inCallback; bool _callbackWait; bool _discarding; bool _framed; + bool _handshake_work_pending; }; struct ResolveHandler : public vespalib::AsyncResolver::ResultHandler { FNET_Connection *connection; @@ -357,6 +359,18 @@ public: bool handle_add_event() override; /** + * This function is called by the transport thread to handle the + * completion of an asynchronous invocation of + * 'do_handshake_work'. This function will try to progress + * connection handshaking further, based on the work performed by + * 'do_handshake_work'. If this function returns false, the + * component is broken and should be closed immediately. + * + * @return false if broken, true otherwise. + **/ + bool handle_handshake_act() override; + + /** * Register a cleanup handler to be invoked when this connection is * about to be destructed. * diff --git a/fnet/src/vespa/fnet/controlpacket.cpp b/fnet/src/vespa/fnet/controlpacket.cpp index f4d10e48353..e2ca5fdcf8b 100644 --- a/fnet/src/vespa/fnet/controlpacket.cpp +++ b/fnet/src/vespa/fnet/controlpacket.cpp @@ -101,6 +101,9 @@ FNET_ControlPacket FNET_ControlPacket::IOCDisableWrite(FNET_CMD_IOC_DISABLE_WRITE); FNET_ControlPacket +FNET_ControlPacket::IOCHandshakeACT(FNET_CMD_IOC_HANDSHAKE_ACT); + +FNET_ControlPacket FNET_ControlPacket::IOCClose(FNET_CMD_IOC_CLOSE); FNET_ControlPacket diff --git a/fnet/src/vespa/fnet/controlpacket.h b/fnet/src/vespa/fnet/controlpacket.h index 5e6eb8a564b..c69cb8e086b 100644 --- a/fnet/src/vespa/fnet/controlpacket.h +++ b/fnet/src/vespa/fnet/controlpacket.h @@ -39,6 +39,7 @@ public: FNET_CMD_IOC_DISABLE_READ, FNET_CMD_IOC_ENABLE_WRITE, FNET_CMD_IOC_DISABLE_WRITE, + FNET_CMD_IOC_HANDSHAKE_ACT, FNET_CMD_IOC_CLOSE, FNET_CMD_EXECUTE, FNET_CMD_TIMEOUT, @@ -54,6 +55,7 @@ public: static FNET_ControlPacket IOCDisableRead; static FNET_ControlPacket IOCEnableWrite; static FNET_ControlPacket IOCDisableWrite; + static FNET_ControlPacket IOCHandshakeACT; static FNET_ControlPacket IOCClose; static FNET_ControlPacket Execute; static FNET_ControlPacket Timeout; diff --git a/fnet/src/vespa/fnet/iocomponent.cpp b/fnet/src/vespa/fnet/iocomponent.cpp index d4244cbf204..0ecf63ea642 100644 --- a/fnet/src/vespa/fnet/iocomponent.cpp +++ b/fnet/src/vespa/fnet/iocomponent.cpp @@ -139,6 +139,11 @@ FNET_IOComponent::handle_add_event() return true; } +bool +FNET_IOComponent::handle_handshake_act() +{ + return true; +} void FNET_IOComponent::CleanupHook() diff --git a/fnet/src/vespa/fnet/iocomponent.h b/fnet/src/vespa/fnet/iocomponent.h index 901c3d1a5d0..1a50ccbca73 100644 --- a/fnet/src/vespa/fnet/iocomponent.h +++ b/fnet/src/vespa/fnet/iocomponent.h @@ -197,6 +197,18 @@ public: virtual bool handle_add_event(); /** + * This function is called by the transport thread to handle the + * completion of an asynchronous invocation of + * 'do_handshake_work'. This functionality is used by TLS + * connections in order to move expensive cpu work out of the + * transport thread. If this function returns false, the component + * is broken and should be closed immediately. + * + * @return false if broken, true otherwise. + **/ + virtual bool handle_handshake_act(); + + /** * This method is called by the SubRef methods just before the * object is deleted. It may be used to perform cleanup tasks that * must be done before the destructor is invoked. diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index fdc3576db2f..4db2a33c78c 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -20,11 +20,14 @@ struct HashState { key_hash(XXH64(key, key_len, 0)) {} }; +VESPA_THREAD_STACK_TAG(fnet_work_pool); + } // namespace <unnamed> FNET_Transport::FNET_Transport(vespalib::AsyncResolver::SP resolver, vespalib::CryptoEngine::SP crypto, size_t num_threads) : _async_resolver(std::move(resolver)), _crypto_engine(std::move(crypto)), + _work_pool(1, 128 * 1024, fnet_work_pool, 1024), _threads() { assert(num_threads >= 1); @@ -36,6 +39,15 @@ FNET_Transport::FNET_Transport(vespalib::AsyncResolver::SP resolver, vespalib::C FNET_Transport::~FNET_Transport() { _async_resolver->wait_for_pending_resolves(); + _work_pool.shutdown().sync(); +} + +void +FNET_Transport::post_or_perform(vespalib::Executor::Task::UP task) +{ + if (auto rejected = _work_pool.execute(std::move(task))) { + rejected->run(); + } } void diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 4624c183b01..27264aa1965 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -7,6 +7,7 @@ #include <vector> #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/util/threadstackexecutor.h> class FastOS_TimeInterface; class FNET_TransportThread; @@ -30,6 +31,7 @@ private: vespalib::AsyncResolver::SP _async_resolver; vespalib::CryptoEngine::SP _crypto_engine; + vespalib::ThreadStackExecutor _work_pool; Threads _threads; public: @@ -53,6 +55,18 @@ public: ~FNET_Transport(); /** + * Try to execute the given task on the internal work pool + * executor (post). If the executor has been closed or there is + * too much pending work the task is run in the context of the + * current thread instead (perform). The idea is to move work away + * from the transport threads as long as pending work is not + * piling up. + * + * @param task work to be done + **/ + void post_or_perform(vespalib::Executor::Task::UP task); + + /** * Resolve a connect spec into a socket address to be used to * connect to a remote socket. This operation will be performed * asynchronously and the result will be given to the result diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 8ee33ad7960..a5cf15c3939 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -144,6 +144,7 @@ FNET_TransportThread::DiscardEvent(FNET_ControlPacket *cpacket, case FNET_ControlPacket::FNET_CMD_IOC_DISABLE_READ: case FNET_ControlPacket::FNET_CMD_IOC_ENABLE_WRITE: case FNET_ControlPacket::FNET_CMD_IOC_DISABLE_WRITE: + case FNET_ControlPacket::FNET_CMD_IOC_HANDSHAKE_ACT: case FNET_ControlPacket::FNET_CMD_IOC_CLOSE: context._value.IOC->SubRef(); break; @@ -344,6 +345,15 @@ FNET_TransportThread::DisableWrite(FNET_IOComponent *comp, bool needRef) FNET_Context(comp)); } +void +FNET_TransportThread::handshake_act(FNET_IOComponent *comp, bool needRef) +{ + if (needRef) { + comp->AddRef(); + } + PostEvent(&FNET_ControlPacket::IOCHandshakeACT, + FNET_Context(comp)); +} void FNET_TransportThread::Close(FNET_IOComponent *comp, bool needRef) @@ -481,6 +491,13 @@ FNET_TransportThread::handle_wakeup() context._value.IOC->EnableWriteEvent(false); context._value.IOC->SubRef(); break; + case FNET_ControlPacket::FNET_CMD_IOC_HANDSHAKE_ACT: + if (context._value.IOC->handle_handshake_act()) { + context._value.IOC->SubRef(); + } else { + handle_close_cmd(context._value.IOC); + } + break; case FNET_ControlPacket::FNET_CMD_IOC_CLOSE: handle_close_cmd(context._value.IOC); break; diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index ffbbb7acc0f..dd135502afe 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -414,6 +414,24 @@ public: /** + * Signal the completion of an asyncronous handshake operation for + * the given io component. Note that the actual work is performed + * by the transport thread. This method simply posts an event on + * the transport thread event queue. NOTE: in order to post async + * events regarding I/O components, an extra reference to the + * component needs to be allocated. The needRef flag indicates + * wether the caller already has done this. + * + * @param comp the component to signal about operation completion + * @param needRef should be set to false if the caller of this + * method already has obtained an extra reference to the + * component. If this flag is true, this method will call the + * AddRef method on the component. + **/ + void handshake_act(FNET_IOComponent *comp, bool needRef = true); + + + /** * Close an I/O component and remove it from the working set of this * transport object. Note that the actual work is performed by the * transport thread. This method simply posts an event on the diff --git a/fsa/src/vespa/fsa/fsa.h b/fsa/src/vespa/fsa/fsa.h index 072dc1de4bd..e4d3246d924 100644 --- a/fsa/src/vespa/fsa/fsa.h +++ b/fsa/src/vespa/fsa/fsa.h @@ -124,6 +124,8 @@ public: _symbol(it._symbol), _state(it._state), _fsa(it._fsa) {} + iteratorItem &operator=(const iteratorItem &rhs) = default; + /** * @brief Destructor. */ diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java index a050fbf6496..7de4a9273bd 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java @@ -11,7 +11,6 @@ import org.osgi.framework.BundleContext; import org.osgi.framework.BundleException; import org.osgi.framework.Constants; import org.osgi.framework.FrameworkEvent; -import org.osgi.framework.FrameworkListener; import org.osgi.framework.wiring.FrameworkWiring; import java.io.File; @@ -99,6 +98,16 @@ public class FelixFramework implements OsgiFramework { } bundle.start(); } + log.info(startedBundlesMessage(bundles)); + } + + private String startedBundlesMessage(List<Bundle> bundles) { + StringBuilder sb = new StringBuilder("Started bundles: {" ); + for (Bundle b : bundles) + sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ":" + b.getVersion() + ", "); + sb.setLength(sb.length() - 2); + sb.append("}"); + return sb.toString(); } @Override diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 099b009fba5..62093a8cf3c 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -1128,19 +1128,5 @@ "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" ], "fields": [] - }, - "com.yahoo.jdisc.http.ssl.ThrowingSslContextFactoryProvider": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.jdisc.http.ssl.SslContextFactoryProvider" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index f9892759fbd..5d3550db9d2 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -5,9 +5,12 @@ import com.google.inject.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.OptionalSslConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -15,6 +18,7 @@ import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.channels.ServerSocketChannel; +import java.util.List; /** * @author Einar M R Rosenvinge @@ -37,15 +41,8 @@ public class ConnectorFactory { } public ServerConnector createConnector(final Metric metric, final Server server, final ServerSocketChannel ch) { - ServerConnector connector; - if (connectorConfig.ssl().enabled()) { - connector = new JDiscServerConnector(connectorConfig, metric, server, ch, - newSslConnectionFactory(), - newHttpConnectionFactory()); - } else { - connector = new JDiscServerConnector(connectorConfig, metric, server, ch, - newHttpConnectionFactory()); - } + ServerConnector connector = new JDiscServerConnector( + connectorConfig, metric, server, ch, createConnectionFactories().toArray(ConnectionFactory[]::new)); connector.setPort(connectorConfig.listenPort()); connector.setName(connectorConfig.name()); connector.setAcceptQueueSize(connectorConfig.acceptQueueSize()); @@ -55,6 +52,26 @@ public class ConnectorFactory { return connector; } + private List<ConnectionFactory> createConnectionFactories() { + HttpConnectionFactory httpConnectionFactory = newHttpConnectionFactory(); + if (connectorConfig.ssl().enabled()) { + return List.of(newSslConnectionFactory(), httpConnectionFactory); + } else if (TransportSecurityUtils.isTransportSecurityEnabled()) { + SslConnectionFactory sslConnectionsFactory = newSslConnectionFactory(); + switch (TransportSecurityUtils.getInsecureMixedMode()) { + case TLS_CLIENT_MIXED_SERVER: + case PLAINTEXT_CLIENT_MIXED_SERVER: + return List.of(newOptionalSslConnectionFactory(sslConnectionsFactory), sslConnectionsFactory, httpConnectionFactory); + case DISABLED: + return List.of(sslConnectionsFactory, httpConnectionFactory); + default: + throw new IllegalStateException(); + } + } else { + return List.of(httpConnectionFactory); + } + } + private HttpConnectionFactory newHttpConnectionFactory() { HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setSendDateHeader(true); @@ -64,7 +81,7 @@ public class ConnectorFactory { httpConfig.setOutputBufferSize(connectorConfig.outputBufferSize()); httpConfig.setRequestHeaderSize(connectorConfig.requestHeaderSize()); httpConfig.setResponseHeaderSize(connectorConfig.responseHeaderSize()); - if (connectorConfig.ssl().enabled()) { + if (connectorConfig.ssl().enabled() || TransportSecurityUtils.isTransportSecurityEnabled()) { // TODO Cleanup once mixed mode is gone httpConfig.addCustomizer(new SecureRequestCustomizer()); } return new HttpConnectionFactory(httpConfig); @@ -75,4 +92,8 @@ public class ConnectorFactory { return new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()); } + private OptionalSslConnectionFactory newOptionalSslConnectionFactory(SslConnectionFactory sslConnectionsFactory) { + return new OptionalSslConnectionFactory(sslConnectionsFactory, HttpVersion.HTTP_1_1.asString()); + } + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java deleted file mode 100644 index b28ef54e822..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import org.eclipse.jetty.util.ssl.SslContextFactory; - -/** - * A dummy implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors - * - * @author bjorncs - */ -public class ThrowingSslContextFactoryProvider implements SslContextFactoryProvider { - @Override - public SslContextFactory getInstance(String containerId, int port) { - throw new UnsupportedOperationException(); - } -}
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java new file mode 100644 index 00000000000..188fabc4841 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -0,0 +1,95 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. + * + * @author bjorncs + */ +public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider { + + private final ConnectorConfig connectorConfig; + + public ConfiguredSslContextFactoryProvider(ConnectorConfig connectorConfig) { + validateConfig(connectorConfig.ssl()); + this.connectorConfig = connectorConfig; + } + + @Override + public SslContextFactory getInstance(String containerId, int port) { + ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); + if (!sslConfig.enabled()) throw new IllegalStateException(); + SslContextFactory factory = new JDiscSslContextFactory(); + + switch (sslConfig.clientAuth()) { + case NEED_AUTH: + factory.setNeedClientAuth(true); + break; + case WANT_AUTH: + factory.setWantClientAuth(true); + break; + } + + // Check if using new ssl syntax from services.xml + factory.setKeyStore(createKeystore(sslConfig)); + factory.setKeyStorePassword(""); + if (!sslConfig.caCertificateFile().isEmpty()) { + factory.setTrustStore(createTruststore(sslConfig)); + } + factory.setProtocol("TLS"); + return factory; + } + + private static void validateConfig(ConnectorConfig.Ssl config) { + if (!config.enabled()) return; + if (config.certificateFile().isEmpty()) { + throw new IllegalArgumentException("Missing certificate file."); + } + if (config.privateKeyFile().isEmpty()) { + throw new IllegalArgumentException("Missing private key file."); + } + + } + + private static KeyStore createTruststore(ConnectorConfig.Ssl sslConfig) { + List<X509Certificate> caCertificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.caCertificateFile())); + KeyStoreBuilder truststoreBuilder = KeyStoreBuilder.withType(KeyStoreType.JKS); + for (int i = 0; i < caCertificates.size(); i++) { + truststoreBuilder.withCertificateEntry("entry-" + i, caCertificates.get(i)); + } + return truststoreBuilder.build(); + } + + private static KeyStore createKeystore(ConnectorConfig.Ssl sslConfig) { + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(readToString(sslConfig.privateKeyFile())); + List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.certificateFile())); + return KeyStoreBuilder.withType(KeyStoreType.JKS).withKeyEntry("default", privateKey, certificates).build(); + } + + private static String readToString(String filename) { + try { + return Files.readString(Paths.get(filename), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java index f9cdefeb5e8..03796c551e5 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -1,95 +1,35 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.ssl.impl; -import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.ReloadingTlsContext; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.util.ssl.SslContextFactory; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.util.List; - /** - * JDisc's default implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. + * The default implementation of {@link SslContextFactoryProvider} to be injected into connectors without explicit ssl configuration * * @author bjorncs */ -public class DefaultSslContextFactoryProvider implements SslContextFactoryProvider { - - private final ConnectorConfig connectorConfig; +public class DefaultSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { - public DefaultSslContextFactoryProvider(ConnectorConfig connectorConfig) { - validateConfig(connectorConfig.ssl()); - this.connectorConfig = connectorConfig; - } + private final TlsContext tlsContext = TransportSecurityUtils.getConfigFile() + .map(configFile -> new ReloadingTlsContext(configFile, TransportSecurityUtils.getInsecureAuthorizationMode())) + .orElse(null); @Override public SslContextFactory getInstance(String containerId, int port) { - ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); - if (!sslConfig.enabled()) throw new IllegalStateException(); - SslContextFactory factory = new JDiscSslContextFactory(); - - switch (sslConfig.clientAuth()) { - case NEED_AUTH: - factory.setNeedClientAuth(true); - break; - case WANT_AUTH: - factory.setWantClientAuth(true); - break; - } - - // Check if using new ssl syntax from services.xml - factory.setKeyStore(createKeystore(sslConfig)); - factory.setKeyStorePassword(""); - if (!sslConfig.caCertificateFile().isEmpty()) { - factory.setTrustStore(createTruststore(sslConfig)); + if (tlsContext != null) { + return new TlsContextManagedSslContextFactory(tlsContext); + } else { + throw new UnsupportedOperationException(); } - factory.setProtocol("TLS"); - factory.setEndpointIdentificationAlgorithm(null); // disable hostname verification of client certs - return factory; } - private static void validateConfig(ConnectorConfig.Ssl config) { - if (!config.enabled()) return; - if (config.certificateFile().isEmpty()) { - throw new IllegalArgumentException("Missing certificate file."); - } - if (config.privateKeyFile().isEmpty()) { - throw new IllegalArgumentException("Missing private key file."); - } - - } - - private static KeyStore createTruststore(ConnectorConfig.Ssl sslConfig) { - List<X509Certificate> caCertificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.caCertificateFile())); - KeyStoreBuilder truststoreBuilder = KeyStoreBuilder.withType(KeyStoreType.JKS); - for (int i = 0; i < caCertificates.size(); i++) { - truststoreBuilder.withCertificateEntry("entry-" + i, caCertificates.get(i)); - } - return truststoreBuilder.build(); - } - - private static KeyStore createKeystore(ConnectorConfig.Ssl sslConfig) { - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(readToString(sslConfig.privateKeyFile())); - List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(readToString(sslConfig.certificateFile())); - return KeyStoreBuilder.withType(KeyStoreType.JKS).withKeyEntry("default", privateKey, certificates).build(); - } - - private static String readToString(String filename) { - try { - return new String(Files.readAllBytes(Paths.get(filename))); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + @Override + public void deconstruct() { + tlsContext.close(); } - -} +}
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java new file mode 100644 index 00000000000..d7d28e7d242 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java @@ -0,0 +1,42 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.security.tls.TlsContext; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLEngine; +import java.net.InetSocketAddress; + +/** + * A Jetty {@link SslContextFactory} backed by {@link TlsContext}. + * Overrides methods that are used by Jetty to construct ssl sockets and ssl engines. + * + * @author bjorncs + */ +class TlsContextManagedSslContextFactory extends SslContextFactory { + + private final TlsContext tlsContext; + + TlsContextManagedSslContextFactory(TlsContext tlsContext) { + this.tlsContext = tlsContext; + } + + @Override protected void doStart() { } // Override default behaviour + @Override protected void doStop() { } // Override default behaviour + + @Override + public SSLEngine newSSLEngine() { + return tlsContext.createSslEngine(); + } + + @Override + public SSLEngine newSSLEngine(InetSocketAddress address) { + return tlsContext.createSslEngine(address.getHostString(), address.getPort()); + } + + @Override + public SSLEngine newSSLEngine(String host, int port) { + return tlsContext.createSslEngine(host, port); + } + +} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java index a4baccb86c9..cc2a00c08c6 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java @@ -10,7 +10,7 @@ import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Builder; import com.yahoo.jdisc.http.server.jetty.ConnectorFactory; -import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; /** * Guice module for test ConnectorFactories @@ -46,7 +46,7 @@ public class ConnectorFactoryRegistryModule implements Module { private static class StaticKeyDbConnectorFactory extends ConnectorFactory { public StaticKeyDbConnectorFactory(ConnectorConfig connectorConfig) { - super(connectorConfig, new DefaultSslContextFactoryProvider(connectorConfig)); + super(connectorConfig, new ConfiguredSslContextFactoryProvider(connectorConfig)); } } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java index cf32801ce88..3a932ba6113 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java @@ -3,7 +3,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.impl.DefaultSslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -78,7 +78,7 @@ public class ConnectorFactoryTest { } private static ConnectorFactory createConnectorFactory(ConnectorConfig config) { - return new ConnectorFactory(config, new DefaultSslContextFactoryProvider(config)); + return new ConnectorFactory(config, new ConfiguredSslContextFactoryProvider(config)); } private static class HelloWorldHandler extends AbstractHandler { diff --git a/messagebus/src/vespa/messagebus/message.h b/messagebus/src/vespa/messagebus/message.h index ebb2e7b6027..e7e7d74033e 100644 --- a/messagebus/src/vespa/messagebus/message.h +++ b/messagebus/src/vespa/messagebus/message.h @@ -29,8 +29,8 @@ public: * Constructs a new instance of this class. */ Message(); - Message(Message &&) noexcept = default; - Message & operator = (Message &&) noexcept = default; + Message(Message &&) = default; + Message & operator = (Message &&) = default; /** * If a message is deleted with elements on the callstack, this destructor diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index 97fa2641504..2d7a36a28ef 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -51,7 +51,7 @@ Sequencer::filter(Message::UP msg) } _seqMap[seqId] = nullptr; // insert empty queue } - return std::move(msg); + return msg; } void diff --git a/metrics/src/tests/gtest_runner.cpp b/metrics/src/tests/gtest_runner.cpp index 35475ba19a9..2f0e0705792 100644 --- a/metrics/src/tests/gtest_runner.cpp +++ b/metrics/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("metrics_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/node-admin/pom.xml b/node-admin/pom.xml index d550ceb7c9d..a61a55fcfd9 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -33,12 +33,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>defaults</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -74,12 +68,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>node-repository</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> <version>4.4.1</version> @@ -135,6 +123,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>node-repository</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>service-monitor</artifactId> <version>${project.version}</version> <scope>test</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java deleted file mode 100644 index cbe9b32cc47..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.chain.ChainedComponent; - -public abstract class TaskComponent extends ChainedComponent implements IdempotentTask<TaskContext> { - protected TaskComponent(ComponentId id) { - super(id); - } - - public String name() { - return getIdString(); - } - - public abstract boolean converge(TaskContext context); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java index 8407d42131b..078b745bac4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver.flags; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; -import java.util.List; import java.util.Map; /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 8c8adf5c190..ad052f28d7d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -30,7 +30,6 @@ public class NodeAttributes { private Optional<String> hardwareDivergence = Optional.empty(); private Optional<String> hardwareFailureDescription = Optional.empty(); private Optional<Boolean> wantToDeprovision = Optional.empty(); - private Optional<String> modelName = Optional.empty(); /** The list of reports to patch. A null value is used to remove the report. */ private Map<String, JsonNode> reports = new TreeMap<>(); @@ -80,11 +79,6 @@ public class NodeAttributes { return this; } - public NodeAttributes withModelName(String modelName) { - this.modelName = Optional.of(modelName); - return this; - } - public NodeAttributes withReports(Map<String, JsonNode> nodeReports) { this.reports = new TreeMap<>(nodeReports); return this; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java new file mode 100644 index 00000000000..bb16e2bae63 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java @@ -0,0 +1,77 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +/** + * @author freva + */ +public class NodeMembership { + private final String clusterType; + private final String clusterId; + private final String group; + private final int index; + private final boolean retired; + + public NodeMembership(String clusterType, String clusterId, String group, int index, boolean retired) { + this.clusterType = clusterType; + this.clusterId = clusterId; + this.group = group; + this.index = index; + this.retired = retired; + } + + public String getClusterType() { + return clusterType; + } + + public String getClusterId() { + return clusterId; + } + + public String getGroup() { + return group; + } + + public int getIndex() { + return index; + } + + public boolean isRetired() { + return retired; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NodeMembership that = (NodeMembership) o; + + if (index != that.index) return false; + if (retired != that.retired) return false; + if (!clusterType.equals(that.clusterType)) return false; + if (!clusterId.equals(that.clusterId)) return false; + return group.equals(that.group); + + } + + @Override + public int hashCode() { + int result = clusterType.hashCode(); + result = 31 * result + clusterId.hashCode(); + result = 31 * result + group.hashCode(); + result = 31 * result + index; + result = 31 * result + (retired ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "Membership {" + + " clusterType = " + clusterType + + " clusterId = " + clusterId + + " group = " + group + + " index = " + index + + " retired = " + retired + + " }"; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java new file mode 100644 index 00000000000..c1900316bb9 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author freva + */ +public class NodeOwner { + private final String tenant; + private final String application; + private final String instance; + + public NodeOwner(String tenant, String application, String instance) { + this.tenant = tenant; + this.application = application; + this.instance = instance; + } + + public String getTenant() { + return tenant; + } + + public String getApplication() { + return application; + } + + public String getInstance() { + return instance; + } + + public ApplicationId asApplicationId() { + return ApplicationId.from(tenant, application, instance); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NodeOwner owner = (NodeOwner) o; + + if (!tenant.equals(owner.tenant)) return false; + if (!application.equals(owner.application)) return false; + return instance.equals(owner.instance); + + } + + @Override + public int hashCode() { + int result = tenant.hashCode(); + result = 31 * result + application.hashCode(); + result = 31 * result + instance.hashCode(); + return result; + } + + public String toString() { + return "Owner {" + + " tenant = " + tenant + + " application = " + application + + " instance = " + instance + + " }"; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java index 5fc82a70e80..4cce6ce08d6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; -import com.yahoo.vespa.hosted.provision.Node; - import java.util.List; import java.util.Map; import java.util.Optional; @@ -26,7 +24,5 @@ public interface NodeRepository { void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes); - void setNodeState(String hostName, Node.State nodeState); - - void scheduleReboot(String hostname); + void setNodeState(String hostName, NodeState nodeState); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index ac65ac8af35..35bcf2f2114 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.provision.Node; import java.time.Instant; import java.util.Collections; @@ -18,7 +16,7 @@ import java.util.Set; */ public class NodeSpec { private final String hostname; - private final Node.State state; + private final NodeState state; private final NodeType nodeType; private final String flavor; private final String canonicalFlavor; @@ -45,8 +43,8 @@ public class NodeSpec { private final Optional<Boolean> allowedToBeDown; private final Optional<Boolean> wantToDeprovision; - private final Optional<Owner> owner; - private final Optional<Membership> membership; + private final Optional<NodeOwner> owner; + private final Optional<NodeMembership> membership; private final double minCpuCores; private final double minMainMemoryAvailableGb; @@ -65,7 +63,7 @@ public class NodeSpec { String hostname, Optional<DockerImage> wantedDockerImage, Optional<DockerImage> currentDockerImage, - Node.State state, + NodeState state, NodeType nodeType, String flavor, String canonicalFlavor, @@ -75,8 +73,8 @@ public class NodeSpec { Optional<String> currentOsVersion, Optional<Boolean> allowedToBeDown, Optional<Boolean> wantToDeprovision, - Optional<Owner> owner, - Optional<Membership> membership, + Optional<NodeOwner> owner, + Optional<NodeMembership> membership, Optional<Long> wantedRestartGeneration, Optional<Long> currentRestartGeneration, long wantedRebootGeneration, @@ -130,7 +128,7 @@ public class NodeSpec { return hostname; } - public Node.State getState() { + public NodeState getState() { return state; } @@ -206,11 +204,11 @@ public class NodeSpec { return wantToDeprovision; } - public Optional<Owner> getOwner() { + public Optional<NodeOwner> getOwner() { return owner; } - public Optional<Membership> getMembership() { + public Optional<NodeMembership> getMembership() { return membership; } @@ -358,140 +356,11 @@ public class NodeSpec { + " }"; } - public static class Owner { - private final String tenant; - private final String application; - private final String instance; - - public Owner(String tenant, String application, String instance) { - this.tenant = tenant; - this.application = application; - this.instance = instance; - } - - public String getTenant() { - return tenant; - } - - public String getApplication() { - return application; - } - - public String getInstance() { - return instance; - } - - public ApplicationId asApplicationId() { - return ApplicationId.from(tenant, application, instance); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Owner owner = (Owner) o; - - if (!tenant.equals(owner.tenant)) return false; - if (!application.equals(owner.application)) return false; - return instance.equals(owner.instance); - - } - - @Override - public int hashCode() { - int result = tenant.hashCode(); - result = 31 * result + application.hashCode(); - result = 31 * result + instance.hashCode(); - return result; - } - - public String toString() { - return "Owner {" + - " tenant = " + tenant + - " application = " + application + - " instance = " + instance + - " }"; - } - } - - public static class Membership { - private final String clusterType; - private final String clusterId; - private final String group; - private final int index; - private final boolean retired; - - public Membership(String clusterType, String clusterId, String group, int index, boolean retired) { - this.clusterType = clusterType; - this.clusterId = clusterId; - this.group = group; - this.index = index; - this.retired = retired; - } - - public String getClusterType() { - return clusterType; - } - - public String getClusterId() { - return clusterId; - } - - public String getGroup() { - return group; - } - - public int getIndex() { - return index; - } - - public boolean isRetired() { - return retired; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Membership that = (Membership) o; - - if (index != that.index) return false; - if (retired != that.retired) return false; - if (!clusterType.equals(that.clusterType)) return false; - if (!clusterId.equals(that.clusterId)) return false; - return group.equals(that.group); - - } - - @Override - public int hashCode() { - int result = clusterType.hashCode(); - result = 31 * result + clusterId.hashCode(); - result = 31 * result + group.hashCode(); - result = 31 * result + index; - result = 31 * result + (retired ? 1 : 0); - return result; - } - - @Override - public String toString() { - return "Membership {" + - " clusterType = " + clusterType + - " clusterId = " + clusterId + - " group = " + group + - " index = " + index + - " retired = " + retired + - " }"; - } - } - public static class Builder { private String hostname; private Optional<DockerImage> wantedDockerImage = Optional.empty(); private Optional<DockerImage> currentDockerImage = Optional.empty(); - private Node.State state; + private NodeState state; private NodeType nodeType; private String flavor; private String canonicalFlavor; @@ -501,8 +370,8 @@ public class NodeSpec { private Optional<String> currentOsVersion = Optional.empty(); private Optional<Boolean> allowedToBeDown = Optional.empty(); private Optional<Boolean> wantToDeprovision = Optional.empty(); - private Optional<Owner> owner = Optional.empty(); - private Optional<Membership> membership = Optional.empty(); + private Optional<NodeOwner> owner = Optional.empty(); + private Optional<NodeMembership> membership = Optional.empty(); private Optional<Long> wantedRestartGeneration = Optional.empty(); private Optional<Long> currentRestartGeneration = Optional.empty(); private long wantedRebootGeneration; @@ -572,7 +441,7 @@ public class NodeSpec { return this; } - public Builder state(Node.State state) { + public Builder state(NodeState state) { this.state = state; return this; } @@ -622,12 +491,12 @@ public class NodeSpec { return this; } - public Builder owner(Owner owner) { + public Builder owner(NodeOwner owner) { this.owner = Optional.of(owner); return this; } - public Builder membership(Membership membership) { + public Builder membership(NodeMembership membership) { this.membership = Optional.of(membership); return this; } @@ -747,7 +616,7 @@ public class NodeSpec { return currentDockerImage; } - public Node.State getState() { + public NodeState getState() { return state; } @@ -787,11 +656,11 @@ public class NodeSpec { return wantToDeprovision; } - public Optional<Owner> getOwner() { + public Optional<NodeOwner> getOwner() { return owner; } - public Optional<Membership> getMembership() { + public Optional<NodeMembership> getMembership() { return membership; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java new file mode 100644 index 00000000000..e1765efef31 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +/** + * All the states a node can be in the node-repository. + * + * See com.yahoo.vespa.hosted.provision.NodeState + * + * @author freva + */ +public enum NodeState { + provisioned, ready, reserved, active, inactive, dirty, failed, parked +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 8e63024842d..586dec3399c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.DockerImage; @@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Ge import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeMessageResponse; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; -import com.yahoo.vespa.hosted.provision.Node; import java.time.Instant; import java.util.Collections; @@ -32,7 +30,6 @@ import java.util.stream.Stream; */ public class RealNodeRepository implements NodeRepository { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(RealNodeRepository.class); - private static final ObjectMapper mapper = new ObjectMapper(); private final ConfigServerApi configServerApi; @@ -133,7 +130,7 @@ public class RealNodeRepository implements NodeRepository { } @Override - public void setNodeState(String hostName, Node.State nodeState) { + public void setNodeState(String hostName, NodeState nodeState) { String state = nodeState.name(); NodeMessageResponse response = configServerApi.put( "/nodes/v2/state/" + state + "/" + hostName, @@ -145,25 +142,13 @@ public class RealNodeRepository implements NodeRepository { throw new NodeRepositoryException("Unexpected message " + response.message + " " + response.errorCode); } - @Override - public void scheduleReboot(String hostName) { - NodeMessageResponse response = configServerApi.post( - "/nodes/v2/command/reboot?hostname=" + hostName, - Optional.empty(), /* body */ - NodeMessageResponse.class); - NODE_ADMIN_LOGGER.info(response.message); - - if (Strings.isNullOrEmpty(response.errorCode)) return; - throw new NodeRepositoryException("Unexpected message " + response.message + " " + response.errorCode); - } - private static NodeSpec createNodeSpec(NodeRepositoryNode node) { Objects.requireNonNull(node.type, "Unknown node type"); NodeType nodeType = NodeType.valueOf(node.type); Objects.requireNonNull(node.state, "Unknown node state"); - Node.State nodeState = Node.State.valueOf(node.state); - if (nodeState == Node.State.active) { + NodeState nodeState = NodeState.valueOf(node.state); + if (nodeState == NodeState.active) { Objects.requireNonNull(node.wantedVespaVersion, "Unknown vespa version for active node"); Objects.requireNonNull(node.wantedDockerImage, "Unknown docker image for active node"); Objects.requireNonNull(node.restartGeneration, "Unknown restartGeneration for active node"); @@ -172,14 +157,14 @@ public class RealNodeRepository implements NodeRepository { String hostName = Objects.requireNonNull(node.hostname, "hostname is null"); - NodeSpec.Owner owner = null; + NodeOwner owner = null; if (node.owner != null) { - owner = new NodeSpec.Owner(node.owner.tenant, node.owner.application, node.owner.instance); + owner = new NodeOwner(node.owner.tenant, node.owner.application, node.owner.instance); } - NodeSpec.Membership membership = null; + NodeMembership membership = null; if (node.membership != null) { - membership = new NodeSpec.Membership(node.membership.clusterType, node.membership.clusterId, + membership = new NodeMembership(node.membership.clusterType, node.membership.clusterId, node.membership.group, node.membership.index, node.membership.retired); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index eb306036416..41c4544c533 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -5,12 +5,12 @@ import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import com.yahoo.vespa.hosted.provision.Node; import java.time.Clock; import java.time.Duration; @@ -125,7 +125,7 @@ public class NodeAdminStateUpdater { throw new ConvergenceException("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen")); } - boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).getState() == Node.State.active; + boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).getState() == NodeState.active; switch (wantedState) { case RESUMED: if (hostIsActiveInNR) orchestrator.resume(hostHostname); @@ -183,7 +183,7 @@ public class NodeAdminStateUpdater { private List<String> getNodesInActiveState() { return nodeRepository.getNodes(hostHostname) .stream() - .filter(node -> node.getState() == Node.State.active) + .filter(node -> node.getState() == NodeState.active) .map(NodeSpec::getHostname) .collect(Collectors.toList()); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 804450f05ff..9ca19a76706 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.provision.Node; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -177,7 +177,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { public Builder(String hostname) { this.nodeSpecBuilder .hostname(hostname) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("d-2-8-50"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index d31a888ea44..dc968a8717e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.fasterxml.jackson.core.JsonProcessingException; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.log.LogLevel; import com.yahoo.vespa.flags.DoubleFlag; import com.yahoo.vespa.flags.FetchVector; @@ -11,38 +10,33 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; +import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; -import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; -import com.yahoo.vespa.hosted.provision.Node; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; import java.util.function.Function; import java.util.logging.Logger; @@ -82,10 +76,6 @@ public class NodeAgentImpl implements NodeAgent { private Optional<Long> currentRestartGeneration = Optional.empty(); private final Thread loopThread; - private final ScheduledExecutorService filebeatRestarter = - Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("filebeatrestarter")); - private final Consumer<String> serviceRestarter; - private Optional<Future<?>> currentFilebeatRestarter = Optional.empty(); /** @@ -139,20 +129,6 @@ public class NodeAgentImpl implements NodeAgent { } }); this.loopThread.setName("tick-" + contextSupplier.currentContext().hostname()); - - this.serviceRestarter = service -> { - NodeAgentContext context = contextSupplier.currentContext(); - try { - ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot( - context, "service", service, "restart"); - - if (!processResult.isSuccess()) { - context.log(logger, LogLevel.ERROR, "Failed to restart service " + service + ": " + processResult); - } - } catch (Exception e) { - context.log(logger, LogLevel.ERROR, "Failed to restart service " + service, e); - } - }; } @Override @@ -165,15 +141,13 @@ public class NodeAgentImpl implements NodeAgent { if (!terminated.compareAndSet(false, true)) { throw new RuntimeException("Can not re-stop a node agent."); } - filebeatRestarter.shutdown(); contextSupplier.interrupt(); do { try { loopThread.join(); - filebeatRestarter.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException ignored) { } - } while (loopThread.isAlive() || !filebeatRestarter.isTerminated()); + } while (loopThread.isAlive()); contextSupplier.currentContext().log(logger, "Stopped"); } @@ -188,12 +162,6 @@ public class NodeAgentImpl implements NodeAgent { void resumeNodeIfNeeded(NodeAgentContext context) { if (!hasResumedNode) { - if (!currentFilebeatRestarter.isPresent()) { - storageMaintainer.writeMetricsConfig(context); - currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( - () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); - } - context.log(logger, LogLevel.DEBUG, "Starting optional node program resume command"); dockerOperations.resumeNode(context); hasResumedNode = true; @@ -269,7 +237,7 @@ public class NodeAgentImpl implements NodeAgent { } private void restartServices(NodeAgentContext context, Container existingContainer) { - if (existingContainer.state.isRunning() && context.node().getState() == Node.State.active) { + if (existingContainer.state.isRunning() && context.node().getState() == NodeState.active) { context.log(logger, "Restarting services"); // Since we are restarting the services we need to suspend the node. orchestratorSuspendNode(context); @@ -308,8 +276,8 @@ public class NodeAgentImpl implements NodeAgent { } private Optional<String> shouldRemoveContainer(NodeSpec node, Container existingContainer) { - final Node.State nodeState = node.getState(); - if (nodeState == Node.State.dirty || nodeState == Node.State.provisioned) { + final NodeState nodeState = node.getState(); + if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) { return Optional.of("Node in state " + nodeState + ", container should no longer be running"); } if (node.getWantedDockerImage().isPresent() && !node.getWantedDockerImage().get().equals(existingContainer.image)) { @@ -347,7 +315,7 @@ public class NodeAgentImpl implements NodeAgent { orchestratorSuspendNode(context); try { - if (context.node().getState() != Node.State.dirty) { + if (context.node().getState() != NodeState.dirty) { suspend(); } stopServices(); @@ -355,7 +323,6 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, LogLevel.WARNING, "Failed stopping services, ignoring", e); } } - stopFilebeatSchedulerIfNeeded(); storageMaintainer.handleCoreDumpsForContainer(context, Optional.of(existingContainer)); dockerOperations.removeContainer(context, existingContainer); currentRebootGeneration = context.node().getWantedRebootGeneration(); @@ -379,7 +346,7 @@ public class NodeAgentImpl implements NodeAgent { private ContainerResources getContainerResources(NodeSpec node) { double cpuCap = node.getOwner() - .map(NodeSpec.Owner::asApplicationId) + .map(NodeOwner::asApplicationId) .map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())) .orElse(containerCpuCap) .value() * node.getMinCpuCores(); @@ -496,7 +463,7 @@ public class NodeAgentImpl implements NodeAgent { updateNodeRepoWithCurrentAttributes(context); break; case provisioned: - nodeRepository.setNodeState(context.hostname().value(), Node.State.dirty); + nodeRepository.setNodeState(context.hostname().value(), NodeState.dirty); break; case dirty: removeContainerIfNeededUpdateContainerState(context, container); @@ -504,7 +471,7 @@ public class NodeAgentImpl implements NodeAgent { athenzCredentialsMaintainer.ifPresent(maintainer -> maintainer.clearCredentials(context)); storageMaintainer.archiveNodeStorage(context); updateNodeRepoWithCurrentAttributes(context); - nodeRepository.setNodeState(context.hostname().value(), Node.State.ready); + nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); break; default: throw new RuntimeException("UNKNOWN STATE " + node.getState().name()); @@ -534,13 +501,6 @@ public class NodeAgentImpl implements NodeAgent { } } - private void stopFilebeatSchedulerIfNeeded() { - if (currentFilebeatRestarter.isPresent()) { - currentFilebeatRestarter.get().cancel(true); - currentFilebeatRestarter = Optional.empty(); - } - } - @SuppressWarnings("unchecked") public void updateContainerNodeMetrics() { if (containerState != UNKNOWN) return; @@ -696,7 +656,7 @@ public class NodeAgentImpl implements NodeAgent { // to allow the node admin to make decisions that depend on the docker image. Or, each docker image // needs to contain routines for drain and suspend. For many images, these can just be dummy routines. private void orchestratorSuspendNode(NodeAgentContext context) { - if (context.node().getState() != Node.State.active) return; + if (context.node().getState() != NodeState.active) return; context.log(logger, "Ask Orchestrator for permission to suspend node"); orchestrator.suspend(context.hostname().value()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index c92027a659b..b5c6b6ab91d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -115,7 +115,7 @@ public class FileFinder { try { // Only need to traverse as deep as we want to match, unless we want to match everything in directories // already matched - Files.walkFileTree(basePath, Collections.emptySet(), maxDepth, new SimpleFileVisitor<Path>() { + Files.walkFileTree(basePath, Collections.emptySet(), maxDepth, new SimpleFileVisitor<>() { private final Stack<FileAttributes> matchingDirectoryStack = new Stack<>(); private int currentLevel = -1; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java index 33498575e8a..de80d4dca18 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java @@ -18,7 +18,7 @@ public enum IPVersion { IPv6(6, "ip6tables", "ip -6", "ipv6-icmp", "/128", "icmp6-port-unreachable", "ip6tables-restore"), IPv4(4, "iptables", "ip", "icmp", "/32", "icmp-port-unreachable", "iptables-restore"); - private static Pattern cidrNotationPattern = Pattern.compile("/\\d+$"); + private static final Pattern cidrNotationPattern = Pattern.compile("/\\d+$"); IPVersion(int version, String iptablesCmd, String ipCmd, String icmpProtocol, String singleHostCidr, String icmpPortUnreachable, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java new file mode 100644 index 00000000000..4a71d3bdbe8 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java @@ -0,0 +1,25 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Test; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class NodeStateTest { + + @Test + public void is_equal_to_node_repository_states() { + Set<String> nodeRepositoryStates = Stream.of(Node.State.values()).map(Enum::name).collect(Collectors.toSet()); + Set<String> nodeAdminStates = Stream.of(NodeState.values()).map(Enum::name).collect(Collectors.toSet()); + + assertEquals(nodeAdminStates, nodeRepositoryStates); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index ac13775d6fa..c2b4c1e0387 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; @@ -107,7 +106,7 @@ public class RealNodeRepositoryTest { final NodeSpec node = containersToRun.get(0); assertThat(node.getHostname(), is("host4.yahoo.com")); assertThat(node.getWantedDockerImage().get(), is(new DockerImage("docker-registry.domain.tld:8080/dist/vespa:6.42.0"))); - assertThat(node.getState(), is(Node.State.active)); + assertThat(node.getState(), is(NodeState.active)); assertThat(node.getWantedRestartGeneration().get(), is(0L)); assertThat(node.getCurrentRestartGeneration().get(), is(0L)); assertThat(node.getMinCpuCores(), is(0.2)); @@ -152,18 +151,18 @@ public class RealNodeRepositoryTest { @Test public void testMarkAsReady() { - nodeRepositoryApi.setNodeState("host5.yahoo.com", Node.State.dirty); - nodeRepositoryApi.setNodeState("host5.yahoo.com", Node.State.ready); + nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.dirty); + nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.ready); try { - nodeRepositoryApi.setNodeState("host4.yahoo.com", Node.State.ready); + nodeRepositoryApi.setNodeState("host4.yahoo.com", NodeState.ready); fail("Should not be allowed to be marked ready as it is not registered as provisioned, dirty, failed or parked"); } catch (RuntimeException ignored) { // expected } try { - nodeRepositoryApi.setNodeState("host101.yahoo.com", Node.State.ready); + nodeRepositoryApi.setNodeState("host101.yahoo.com", NodeState.ready); fail("Expected failure because host101 does not exist"); } catch (RuntimeException ignored) { // expected @@ -191,12 +190,4 @@ public class RealNodeRepositoryTest { assertTrue(nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").isPresent()); } - - @Test - public void testRebootScheduling() { - NodeSpec nodeSpec = nodeRepositoryApi.getNode("host5.yahoo.com"); - nodeRepositoryApi.scheduleReboot(nodeSpec.getHostname()); - NodeSpec newNodeSpec = nodeRepositoryApi.getNode(nodeSpec.getHostname()); - assertEquals(nodeSpec.getWantedRebootGeneration() + 1, newNodeSpec.getWantedRebootGeneration()); - } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java index e1e7ea240b0..bf86da612b1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java @@ -22,8 +22,8 @@ import static org.junit.Assert.assertEquals; */ public class NodeRepositoryNodeTest { private static final ObjectMapper mapper = new ObjectMapper(); - private NodeRepositoryNode node = new NodeRepositoryNode(); - private NodeAttributes attributes = new NodeAttributes(); + private final NodeRepositoryNode node = new NodeRepositoryNode(); + private final NodeAttributes attributes = new NodeAttributes(); /** diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index c7bc8f16477..ae22a54faa9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import org.junit.Test; import static org.mockito.ArgumentMatchers.any; @@ -28,7 +28,7 @@ public class DockerFailTest { .hostname(hostname) .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1L) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 6ae373e8b55..32e6aadaba3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; @@ -21,7 +22,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.test.file.TestFileSystem; import org.mockito.InOrder; import org.mockito.Mockito; @@ -81,7 +81,7 @@ public class DockerTester implements AutoCloseable { NodeSpec hostSpec = new NodeSpec.Builder() .hostname(HOST_HOSTNAME.value()) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.host) .flavor("default") .wantedRestartGeneration(1L) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index 16c9239dcd7..9cb89ae95ce 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import org.junit.Test; import static org.mockito.ArgumentMatchers.any; @@ -27,7 +27,7 @@ public class MultiDockerTest { tester.addChildNodeRepositoryNode( new NodeSpec.Builder(nodeSpec2) - .state(Node.State.dirty) + .state(NodeState.dirty) .minCpuCores(1) .minMainMemoryAvailableGb(1) .minDiskAvailableGb(1) @@ -36,7 +36,7 @@ public class MultiDockerTest { tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2"))); tester.inOrder(tester.storageMaintainer).archiveNodeStorage( argThat(context -> context.containerName().equals(new ContainerName("host2")))); - tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(Node.State.ready)); + tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(NodeState.ready)); addAndWaitForNode(tester, "host3.test.yahoo.com", new DockerImage("image1")); } @@ -46,7 +46,7 @@ public class MultiDockerTest { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname(hostName) .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1L) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index 5748d4034eb..ebf9d72ff1b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import java.util.Collections; import java.util.HashMap; @@ -59,7 +59,7 @@ public class NodeRepoMock implements NodeRepository { } @Override - public void setNodeState(String hostName, Node.State nodeState) { + public void setNodeState(String hostName, NodeState nodeState) { synchronized (monitor) { updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName)) .state(nodeState) @@ -67,11 +67,6 @@ public class NodeRepoMock implements NodeRepository { } } - @Override - public void scheduleReboot(String hostname) { - - } - void updateNodeRepositoryNode(NodeSpec nodeSpec) { synchronized (monitor) { nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index 295368e0426..a1610ae4da3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -3,11 +3,10 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import java.util.Arrays; @@ -51,7 +50,7 @@ public class RebootTest { return new NodeSpec.Builder() .hostname(hostname) .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("docker") .vespaVersion("6.50.0") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index f9f455cca79..ff2f15efd0b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import org.junit.Test; import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM; @@ -28,7 +28,7 @@ public class RestartTest { tester.addChildNodeRepositoryNode(new NodeSpec.Builder() .hostname(hostname) - .state(Node.State.active) + .state(NodeState.active) .wantedDockerImage(dockerImage) .nodeType(NodeType.tenant) .flavor("docker") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 4c753e5a95d..85562a92af9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -7,14 +7,16 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; import org.junit.Test; @@ -155,10 +157,10 @@ public class StorageMaintainerTest { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname("host123-5.test.domain.tld") .nodeType(nodeType) - .state(Node.State.active) + .state(NodeState.active) .parentHostname("host123.test.domain.tld") - .owner(new NodeSpec.Owner("tenant", "application", "instance")) - .membership(new NodeSpec.Membership("clusterType", "clusterId", null, 0, false)) + .owner(new NodeOwner("tenant", "application", "instance")) + .membership(new NodeMembership("clusterType", "clusterId", null, 0, false)) .vespaVersion("6.305.12") .flavor("d-2-8-50") .canonicalFlavor("d-2-8-50") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 2f835a535ff..3fcb08e6680 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -6,10 +6,10 @@ import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; @@ -19,6 +19,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -32,9 +34,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; - /** * @author bakksjo */ @@ -163,7 +162,7 @@ public class NodeAdminImplTest { private NodeAgentContext createNodeAgentContext(String hostname) { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname(hostname) - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("default") .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 2387c279672..b8894bbf814 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -5,11 +5,11 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import java.time.Duration; @@ -57,7 +57,7 @@ public class NodeAdminStateUpdaterTest { @Test public void state_convergence() { - mockNodeRepo(Node.State.active, 4); + mockNodeRepo(NodeState.active, 4); List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() .map(NodeSpec::getHostname) .collect(Collectors.toList()); @@ -115,7 +115,7 @@ public class NodeAdminStateUpdaterTest { @Test public void half_transition_revert() { final String exceptionMsg = "Cannot allow to suspend because some reason"; - mockNodeRepo(Node.State.active, 3); + mockNodeRepo(NodeState.active, 3); // Initially everything is frozen to force convergence when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); @@ -158,7 +158,7 @@ public class NodeAdminStateUpdaterTest { public void do_not_orchestrate_host_when_not_active() { when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); when(nodeAdmin.setFrozen(anyBoolean())).thenReturn(true); - mockNodeRepo(Node.State.ready, 3); + mockNodeRepo(NodeState.ready, 3); // Resume and suspend only require that node-agents are frozen and permission from // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. @@ -178,7 +178,7 @@ public class NodeAdminStateUpdaterTest { @Test public void uses_cached_acl() { - mockNodeRepo(Node.State.active, 1); + mockNodeRepo(NodeState.active, 1); mockAcl(Acl.EMPTY, 1); updater.adjustNodeAgentsToRunFromNodeRepository(); @@ -199,7 +199,7 @@ public class NodeAdminStateUpdaterTest { @Test public void node_spec_and_acl_aligned() { Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(Node.State.active, 3); + mockNodeRepo(NodeState.active, 3); mockAcl(acl, 1, 2, 3); updater.adjustNodeAgentsToRunFromNodeRepository(); @@ -216,11 +216,11 @@ public class NodeAdminStateUpdaterTest { @Test public void node_spec_and_acl_mismatch_missing_one_acl() { Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(Node.State.active, 3); + mockNodeRepo(NodeState.active, 3); mockAcl(acl, 1, 2); // Acl for 3 is missing updater.adjustNodeAgentsToRunFromNodeRepository(); - mockNodeRepo(Node.State.active, 2); // Next tick, the spec for 3 is no longer returned + mockNodeRepo(NodeState.active, 2); // Next tick, the spec for 3 is no longer returned updater.adjustNodeAgentsToRunFromNodeRepository(); updater.adjustNodeAgentsToRunFromNodeRepository(); @@ -234,7 +234,7 @@ public class NodeAdminStateUpdaterTest { @Test public void node_spec_and_acl_mismatch_additional_acl() { Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(Node.State.active, 2); + mockNodeRepo(NodeState.active, 2); mockAcl(acl, 1, 2, 3); // Acl for 3 is extra updater.adjustNodeAgentsToRunFromNodeRepository(); @@ -256,11 +256,11 @@ public class NodeAdminStateUpdaterTest { } } - private void mockNodeRepo(Node.State hostState, int numberOfNodes) { + private void mockNodeRepo(NodeState hostState, int numberOfNodes) { List<NodeSpec> containersToRun = IntStream.range(1, numberOfNodes + 1) .mapToObj(i -> new NodeSpec.Builder() .hostname("host" + i + ".yahoo.com") - .state(Node.State.active) + .state(NodeState.active) .nodeType(NodeType.tenant) .flavor("docker") .minCpuCores(1) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 35494b375f6..3130500c940 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -10,20 +10,22 @@ import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; -import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; @@ -90,7 +92,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -119,7 +121,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -140,7 +142,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -176,7 +178,7 @@ public class NodeAgentImplTest { final Optional<Long> restartGeneration = Optional.of(1L); final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration.get()) .currentRestartGeneration(restartGeneration.get()) @@ -213,7 +215,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(newDockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -240,7 +242,7 @@ public class NodeAgentImplTest { NodeSpec.Builder specBuilder = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion); @@ -287,7 +289,7 @@ public class NodeAgentImplTest { NodeSpec.Builder specBuilder = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion); @@ -324,7 +326,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(wantedRestartGeneration) @@ -353,7 +355,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRebootGeneration(wantedRebootGeneration) @@ -396,7 +398,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.failed) + .state(NodeState.failed) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -416,7 +418,7 @@ public class NodeAgentImplTest { @Test public void readyNodeLeadsToNoAction() { final NodeSpec node = nodeBuilder - .state(Node.State.ready) + .state(NodeState.ready) .build(); NodeAgentContext context = createContext(node); @@ -442,7 +444,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.inactive) + .state(NodeState.inactive) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .build(); @@ -465,7 +467,7 @@ public class NodeAgentImplTest { public void reservedNodeDoesNotUpdateNodeRepoWithVersion() { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) - .state(Node.State.reserved) + .state(NodeState.reserved) .wantedVespaVersion(vespaVersion) .build(); @@ -479,7 +481,7 @@ public class NodeAgentImplTest { verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); } - private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(Node.State nodeState, Optional<Long> wantedRestartGeneration) { + private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) { wantedRestartGeneration.ifPresent(restartGeneration -> nodeBuilder .wantedRestartGeneration(restartGeneration) .currentRestartGeneration(restartGeneration)); @@ -502,7 +504,7 @@ public class NodeAgentImplTest { inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any()); inOrder.verify(dockerOperations, times(1)).removeContainer(eq(context), any()); inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); - inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready)); + inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); verify(dockerOperations, never()).createContainer(eq(context), any()); verify(dockerOperations, never()).startContainer(eq(context)); @@ -517,19 +519,19 @@ public class NodeAgentImplTest { @Test public void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(Node.State.dirty, Optional.of(1L)); + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.of(1L)); } @Test public void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycledNoRestartGeneration() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(Node.State.dirty, Optional.empty()); + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.empty()); } @Test public void provisionedNodeIsMarkedAsDirty() { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) - .state(Node.State.provisioned) + .state(NodeState.provisioned) .build(); NodeAgentContext context = createContext(node); @@ -537,7 +539,7 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); nodeAgent.doConverge(context); - verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.dirty)); + verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.dirty)); } @Test @@ -545,7 +547,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .currentDockerImage(dockerImage) .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .vespaVersion(vespaVersion) .build(); @@ -567,7 +569,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .vespaVersion(vespaVersion) .build(); @@ -603,7 +605,7 @@ public class NodeAgentImplTest { public void start_container_subtask_failure_leads_to_container_restart() { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .build(); @@ -651,12 +653,12 @@ public class NodeAgentImplTest { ContainerStats stats1 = new ContainerStats(networks, precpu_stats, memory_stats, blkio_stats); ContainerStats stats2 = new ContainerStats(networks, cpu_stats, memory_stats, blkio_stats); - NodeSpec.Owner owner = new NodeSpec.Owner("tester", "testapp", "testinstance"); - NodeSpec.Membership membership = new NodeSpec.Membership("clustType", "clustId", "grp", 3, false); + NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance"); + NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false); final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .vespaVersion(vespaVersion) .owner(owner) .membership(membership) @@ -703,7 +705,7 @@ public class NodeAgentImplTest { @Test public void testGetRelevantMetricsForReadyNode() { final NodeSpec node = nodeBuilder - .state(Node.State.ready) + .state(NodeState.ready) .build(); NodeAgentContext context = createContext(node); @@ -722,7 +724,7 @@ public class NodeAgentImplTest { final NodeSpec node = nodeBuilder .nodeType(NodeType.config) .wantedDockerImage(dockerImage) - .state(Node.State.active) + .state(NodeState.active) .wantedVespaVersion(vespaVersion) .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java index 09f7d6117f9..29b012c1397 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java @@ -4,7 +4,6 @@ import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; -import java.io.IOException; import java.nio.file.FileSystem; import static org.junit.Assert.assertFalse; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java index 1531b06070e..7bdf9a44ec7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -2,16 +2,10 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.time.TestTimer; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Optional; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java index 88d26103777..333cb81f9d4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java @@ -1,14 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.process; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.time.TestTimer; import org.junit.Test; import org.mockito.ArgumentCaptor; -import java.io.IOException; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; diff --git a/parent/pom.xml b/parent/pom.xml index 1cececbd8e8..555da7c45af 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -722,7 +722,7 @@ <apache.httpclient.version>4.4.1</apache.httpclient.version> <apache.httpcore.version>4.4.1</apache.httpcore.version> <asm.version>7.0</asm.version> - <aws.sdk.version>1.11.357</aws.sdk.version> + <aws.sdk.version>1.11.289</aws.sdk.version> <jna.version>4.5.2</jna.version> <tensorflow.version>1.12.0</tensorflow.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> diff --git a/predicate-search/CMakeLists.txt b/predicate-search/CMakeLists.txt new file mode 100644 index 00000000000..19d37c829d5 --- /dev/null +++ b/predicate-search/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(predicate-search) diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index 24c38ad118b..e1441bbf8b4 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/config-indexschema.h> #include <vespa/config-attributes.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/log/log.h> @@ -18,6 +19,8 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::IndexschemaConfig; using vespa::config::search::IndexschemaConfigBuilder; +using vespa::config::search::SummaryConfig; +using vespa::config::search::SummaryConfigBuilder; using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; using search::attribute::Config; @@ -125,6 +128,24 @@ SummarymapConfig::Override make_attribute_combiner_override(const vespalib::stri return override; } +SummaryConfig::Classes::Fields make_summary_field(const vespalib::string &name, const vespalib::string &type) +{ + SummaryConfig::Classes::Fields field; + field.name = name; + field.type = type; + return field; +} + +SummaryConfig sCfg(std::vector<SummaryConfig::Classes::Fields> fields) +{ + SummaryConfigBuilder result; + result.classes.resize(1); + result.classes.back().id = 0; + result.classes.back().name = "default"; + result.classes.back().fields = std::move(fields); + return result; +} + SummarymapConfig smCfg(std::vector<SummarymapConfig::Override> overrides) { SummarymapConfigBuilder result; @@ -175,10 +196,10 @@ public: _oldIndexSchema.indexfield.emplace_back(field); } void setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, - const AttributesConfig &newAttributesConfig, const SummarymapConfig &newSummarymapConfig) { + const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig) { IndexschemaInspector indexschemaInspector(_oldIndexSchema); _delayer.setup(oldAttributesConfig, oldSummarymapConfig, - newAttributesConfig, newSummarymapConfig, + newAttributesConfig, newSummaryConfig, newSummarymapConfig, indexschemaInspector, _inspector); } void assertAttributeConfig(const std::vector<AttributesConfig::Attribute> &exp) @@ -195,14 +216,14 @@ public: TEST_F("require that empty config is OK", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({}), smCfg({}), attrCfg({}), sCfg({}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } TEST_F("require that simple attribute config is OK", Fixture) { - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -210,7 +231,7 @@ TEST_F("require that simple attribute config is OK", Fixture) TEST_F("require that adding attribute aspect is delayed if field type is unchanged", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -218,14 +239,14 @@ TEST_F("require that adding attribute aspect is delayed if field type is unchang TEST_F("require that adding attribute aspect is delayed if field type is unchanged, geopos override", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_geopos_override("a")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({make_geopos_override("a")})); } TEST_F("require that adding attribute is not delayed if field type changed", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -233,22 +254,30 @@ TEST_F("require that adding attribute is not delayed if field type changed", Fix TEST_F("require that removing attribute aspect is delayed if field type is unchanged", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } +TEST_F("require that summary map override is removed when summary aspect is removed, even if removing attribute aspect is delayed", Fixture) +{ + f.addFields({"a"}); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({}), smCfg({})); + TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); + TEST_DO(f.assertSummarymapConfig({})); +} + TEST_F("require that removing attribute aspect is delayed if field type is unchanged, gepos override", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")}), attrCfg({}), sCfg({}), smCfg({})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({})); } TEST_F("require that removing attribute aspect is not delayed if field type changed", Fixture) { - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -257,7 +286,7 @@ TEST_F("require that removing attribute aspect is not delayed if also indexed", { f.addFields({"a"}); f.addOldIndexField("a"); - f.setup(attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -265,7 +294,7 @@ TEST_F("require that removing attribute aspect is not delayed if also indexed", TEST_F("require that removing attribute aspect is not delayed for tensor", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "tensor")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -273,7 +302,7 @@ TEST_F("require that removing attribute aspect is not delayed for tensor", Fixtu TEST_F("require that removing attribute aspect is not delayed for predicate", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_predicate_cfg(4)}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_predicate_cfg(4)}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -281,7 +310,7 @@ TEST_F("require that removing attribute aspect is not delayed for predicate", Fi TEST_F("require that removing attribute aspect is not delayed for reference", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_reference_cfg()}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_reference_cfg()}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "longstring")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -289,7 +318,7 @@ TEST_F("require that removing attribute aspect is not delayed for reference", Fi TEST_F("require that fast access flag change is delayed, false->true edge", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_int32_sv_cfg())}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -297,7 +326,7 @@ TEST_F("require that fast access flag change is delayed, false->true edge", Fixt TEST_F("require that fast access flag change is delayed, true->false edge", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_fa(make_int32_sv_cfg())})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -305,7 +334,7 @@ TEST_F("require that fast access flag change is delayed, true->false edge", Fixt TEST_F("require that fast access flag change is delayed, false->true edge, tensor attr", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_tensor_cfg("tensor(x[10])")})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -313,7 +342,7 @@ TEST_F("require that fast access flag change is delayed, false->true edge, tenso TEST_F("require that fast access flag change is not delayed, true->false edge, tensor attr", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")}), attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")}), attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_tensor_cfg("tensor(x[10])")})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -322,14 +351,14 @@ TEST_F("require that fast access flag change is not delayed, true->false edge, s { f.addFields({"a"}); f.addOldIndexField("a"); - f.setup(attrCfg({make_fa(make_string_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_string_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_string_sv_cfg()}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_string_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } TEST_F("require that adding attribute aspect to struct field is not delayed if field type is changed", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({make_attribute_combiner_override("array")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg("array.a")})); TEST_DO(f.assertSummarymapConfig({make_attribute_combiner_override("array")})); } @@ -337,7 +366,7 @@ TEST_F("require that adding attribute aspect to struct field is not delayed if f TEST_F("require that adding attribute aspect to struct field is delayed if field type is unchanged", Fixture) { f.addFields({"array.a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({make_attribute_combiner_override("array")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -345,7 +374,7 @@ TEST_F("require that adding attribute aspect to struct field is delayed if field TEST_F("require that removing attribute aspect from struct field is not delayed", Fixture) { f.addFields({"array.a"}); - f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index df01ae0fc02..a2b824b88ba 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> @@ -86,6 +87,19 @@ public: _builder.attributes(make_shared<AttributesConfig>(builder)); return *this; } + MyConfigBuilder &addSummary(bool hasField) { + SummaryConfigBuilder builder; + builder.classes.resize(1); + builder.classes.back().id = 0; + builder.classes.back().name = "default"; + if (hasField) { + builder.classes.back().fields.resize(1); + builder.classes.back().fields.back().name = "my_attribute"; + builder.classes.back().fields.back().type = "integer"; + } + _builder.summary(make_shared<SummaryConfig>(builder)); + return *this; + } MyConfigBuilder &addSummarymap() { SummarymapConfigBuilder builder; builder.override.resize(1); @@ -114,11 +128,12 @@ struct Fixture { replayCfg(), nullCfg() { - basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().build(); + basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(true).build(); fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(true). addSummarymap(). build(); replayCfg = DocumentDBConfig::makeReplayConfig(fullCfg); @@ -152,11 +167,13 @@ struct DelayAttributeAspectFixture { addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(true). addSummarymap(). build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(hasDocField). build(); } diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 6bc7e1b5556..0f55a4c30de 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -189,7 +189,7 @@ defaultTransactionLogStats() flushengine::TlsStatsMap::Map result; result.insert(std::make_pair("handler1", flushengine::TlsStats(1000, 11, 110))); result.insert(std::make_pair("handler2", flushengine::TlsStats(2000, 11, 110))); - return std::move(result); + return result; } struct FlushStrategyFixture diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index cf803ec0368..4cf2df97fd0 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/common/config_hash.hpp> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/config-attributes.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> using search::attribute::ConfigConverter; @@ -14,6 +15,7 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; +using vespa::config::search::SummaryConfig; using search::attribute::BasicType; namespace proton { @@ -40,6 +42,30 @@ bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Confi return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name) && !isStructFieldAttribute(name); } +class KnownSummaryFields +{ + vespalib::hash_set<vespalib::string> _fields; + +public: + KnownSummaryFields(const SummaryConfig &summaryConfig); + ~KnownSummaryFields(); + + bool known(const vespalib::string &fieldName) const { + return _fields.find(fieldName) != _fields.end(); + } +}; + +KnownSummaryFields::KnownSummaryFields(const SummaryConfig &summaryConfig) + : _fields() +{ + for (const auto &summaryClass : summaryConfig.classes) { + for (const auto &summaryField : summaryClass.fields) { + _fields.insert(summaryField.name); + } + } +} + +KnownSummaryFields::~KnownSummaryFields() = default; } @@ -135,12 +161,14 @@ void handleOldAttributes(const AttributesConfig &oldAttributesConfig, const AttributesConfig &newAttributesConfig, const SummarymapConfig &oldSummarymapConfig, + const SummaryConfig &newSummaryConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector, AttributesConfigBuilder &attributesConfig, SummarymapConfigBuilder &summarymapConfig) { vespalib::hash_set<vespalib::string> delayed; + KnownSummaryFields knownSummaryFields(newSummaryConfig); AttributesConfigHash newAttrs(newAttributesConfig.attribute); for (const auto &oldAttr : oldAttributesConfig.attribute) { search::attribute::Config oldCfg = ConfigConverter::convert(oldAttr); @@ -159,7 +187,7 @@ handleOldAttributes(const AttributesConfig &oldAttributesConfig, for (const auto &override : oldSummarymapConfig.override) { if (override.command == "attribute") { auto itr = delayed.find(override.field); - if (itr != delayed.end()) { + if (itr != delayed.end() && knownSummaryFields.known(override.field)) { summarymapConfig.override.emplace_back(override); } } @@ -172,6 +200,7 @@ void AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, + const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector) @@ -181,7 +210,7 @@ AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, oldIndexschemaInspector, inspector, *_attributesConfig, *_summarymapConfig); handleOldAttributes(oldAttributesConfig, newAttributesConfig, - oldSummarymapConfig, + oldSummarymapConfig, newSummaryConfig, oldIndexschemaInspector, inspector, *_attributesConfig, *_summarymapConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h index 3a2d1b5e0b1..abd3829f5dd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h @@ -9,6 +9,7 @@ namespace vespa { namespace config { namespace search { namespace internal { class InternalAttributesType; class InternalIndexschemaType; class InternalSummarymapType; +class InternalSummaryType; } } } } namespace proton { @@ -28,6 +29,7 @@ class AttributeAspectDelayer using IndexschemaConfig = const vespa::config::search::internal::InternalIndexschemaType; using SummarymapConfigBuilder = vespa::config::search::internal::InternalSummarymapType; using SummarymapConfig = const vespa::config::search::internal::InternalSummarymapType; + using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; std::shared_ptr<AttributesConfigBuilder> _attributesConfig; std::shared_ptr<SummarymapConfigBuilder> _summarymapConfig; @@ -43,6 +45,7 @@ public: void setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, + const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index fd1f9f1155d..712bc553d08 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -265,7 +265,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum DocumentTypeInspector inspector(*oldCfg.getDocumentType(), *n.getDocumentType()); IndexschemaInspector oldIndexschemaInspector(oldCfg.getIndexschemaConfig()); attributeAspectDelayer.setup(oldCfg.getAttributesConfig(), oldCfg.getSummarymapConfig(), - n.getAttributesConfig(), n.getSummarymapConfig(), + n.getAttributesConfig(), n.getSummaryConfig(), n.getSummarymapConfig(), oldIndexschemaInspector, inspector); bool delayedAttributeAspects = (n.getAttributesConfig() != *attributeAspectDelayer.getAttributesConfig()) || (n.getSummarymapConfig() != *attributeAspectDelayer.getSummarymapConfig()); diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 2de71dc3e1b..4a515cf3b19 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -58,6 +58,10 @@ public: _importedFields = importedFields_in; return *this; } + DocumentDBConfigBuilder &summary(const DocumentDBConfig::SummaryConfigSP &summary_in) { + _summary = summary_in; + return *this; + } DocumentDBConfigBuilder &summarymap(const DocumentDBConfig::SummarymapConfigSP &summarymap_in) { _summarymap = summarymap_in; return *this; diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 9e75a85d289..84d925e0780 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -1040,7 +1040,7 @@ TEST("require that findIncompleteCompactedFiles does expected filtering") { EXPECT_TRUE(toRemove.find(FileChunk::NameId(201)) != toRemove.end()); EXPECT_TRUE(toRemove.find(FileChunk::NameId(205)) != toRemove.end()); - EXPECT_EXCEPTION(LogDataStore::findIncompleteCompactedFiles(create({1,3,100,200,201,202,204})).empty(), + EXPECT_EXCEPTION((void) LogDataStore::findIncompleteCompactedFiles(create({1,3,100,200,201,202,204})).empty(), vespalib::IllegalStateException, "3 consecutive files {200, 201, 202}. Impossible"); } diff --git a/searchlib/src/vespa/searchlib/fef/number_or_object.h b/searchlib/src/vespa/searchlib/fef/number_or_object.h index 259a1622516..f9fb3d061b4 100644 --- a/searchlib/src/vespa/searchlib/fef/number_or_object.h +++ b/searchlib/src/vespa/searchlib/fef/number_or_object.h @@ -16,7 +16,8 @@ namespace search::fef { union NumberOrObject { feature_t as_number; vespalib::eval::Value::CREF as_object; - NumberOrObject() { memset(this, 0, sizeof(NumberOrObject)); } + char as_bytes[std::max(sizeof(as_number), sizeof(as_object))]; + NumberOrObject() { memset(as_bytes, 0, sizeof(as_bytes)); } ~NumberOrObject() {} }; diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index c9c326df9ed..e74ad49b2f5 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -2,7 +2,7 @@ package com.yahoo.security.tls; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.tls.authz.PeerAuthorizerTrustManagersFactory; +import com.yahoo.security.tls.authz.PeerAuthorizerTrustManager; import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; @@ -12,6 +12,7 @@ import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -133,7 +134,9 @@ public class DefaultTlsContext implements TlsContext { builder.withTrustStore(caCertificates); } if (authorizedPeers != null) { - builder.withTrustManagerFactory(new PeerAuthorizerTrustManagersFactory(authorizedPeers, mode)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore)); + } else { + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, truststore)); } return builder.build(); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java index b57105f54f9..f1fc62de56a 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ReloadingTlsContext.java @@ -7,6 +7,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.authz.PeerAuthorizerTrustManager; +import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; @@ -20,6 +21,7 @@ import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.List; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -110,7 +112,7 @@ public class ReloadingTlsContext implements TlsContext { .withTrustManagerFactory( ignoredTruststore -> options.getAuthorizedPeers() .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) - .orElse(mutableTrustManager)) + .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) .build(); return new DefaultTlsContext(sslContext, options.getAcceptedCiphers()); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java index eee2e502183..3ddd0861f39 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java @@ -7,11 +7,14 @@ import com.yahoo.security.tls.TrustManagerUtils; import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; import javax.net.ssl.X509ExtendedTrustManager; import java.net.Socket; import java.security.KeyStore; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -55,24 +58,28 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { @Override public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { + overrideHostnameVerification(socket); defaultTrustManager.checkClientTrusted(chain, authType, socket); authorizePeer(chain[0], authType, true, null); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { + overrideHostnameVerification(socket); defaultTrustManager.checkServerTrusted(chain, authType, socket); authorizePeer(chain[0], authType, false, null); } @Override public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { + overrideHostnameVerification(sslEngine); defaultTrustManager.checkClientTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, true, sslEngine); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { + overrideHostnameVerification(sslEngine); defaultTrustManager.checkServerTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, false, sslEngine); } @@ -114,4 +121,31 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { certificate.getSubjectX500Principal(), X509CertificateUtils.getSubjectAlternativeNames(certificate), authType, isVerifyingClient); } + private static void overrideHostnameVerification(SSLEngine engine) { + SSLParameters params = engine.getSSLParameters(); + if (overrideHostnameVerification(params)) { + engine.setSSLParameters(params); + } + } + + private static void overrideHostnameVerification(Socket socket) { + if (socket instanceof SSLSocket) { + SSLSocket sslSocket = (SSLSocket) socket; + SSLParameters params = sslSocket.getSSLParameters(); + if (overrideHostnameVerification(params)) { + sslSocket.setSSLParameters(params); + } + } + } + + // Disable the default hostname verification that is performed by underlying trust manager when 'HTTPS' is used as endpoint identification algorithm. + // Some http clients, notably the new http client in Java 11, does not allow user configuration of the endpoint algorithm or custom HostnameVerifier. + private static boolean overrideHostnameVerification(SSLParameters params) { + if (Objects.equals("HTTPS", params.getEndpointIdentificationAlgorithm())) { + params.setEndpointIdentificationAlgorithm(""); + return true; + } + return false; + } + } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManagersFactory.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManagersFactory.java deleted file mode 100644 index 6ec8450c035..00000000000 --- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManagersFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.authz; - -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.tls.AuthorizationMode; -import com.yahoo.security.tls.policy.AuthorizedPeers; - -import java.security.KeyStore; - -/** - * @author bjorncs - */ -public class PeerAuthorizerTrustManagersFactory implements SslContextBuilder.TrustManagerFactory { - private final AuthorizedPeers authorizedPeers; - private AuthorizationMode mode; - - public PeerAuthorizerTrustManagersFactory(AuthorizedPeers authorizedPeers, AuthorizationMode mode) { - this.authorizedPeers = authorizedPeers; - this.mode = mode; - } - - @Override - public PeerAuthorizerTrustManager createTrustManager(KeyStore truststore) { - return new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore); - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index 88aefe42a14..ede11c48de4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -22,6 +22,7 @@ import static com.yahoo.yolean.Exceptions.uncheck; * @author hakonhall */ public class StateV1HealthClient implements AutoCloseable { + private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB private static final ObjectMapper MAPPER = new ObjectMapper(); private static final Logger logger = Logger.getLogger(StateV1HealthClient.class.getName()); @@ -71,4 +72,5 @@ public class StateV1HealthClient implements AutoCloseable { logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e); } } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java index 7a6494e0122..b28ceeddae0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java @@ -13,6 +13,7 @@ import java.time.Duration; * @author hakon */ class StateV1HealthMonitor implements HealthMonitor { + private final StateV1HealthUpdater updater; private final Cancellable periodicExecution; @@ -30,4 +31,5 @@ class StateV1HealthMonitor implements HealthMonitor { public void close() { periodicExecution.cancel(); } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java index 6a6768aa78b..985685ebb8d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author hakonhall */ class StateV1HealthUpdater implements HealthUpdater { + private final String endpoint; private final StateV1HealthClient healthClient; @@ -56,4 +57,5 @@ class StateV1HealthUpdater implements HealthUpdater { public void close() { healthClient.close(); } + } diff --git a/staging_vespalib/src/vespa/vespalib/gtest/gtest.h b/staging_vespalib/src/vespa/vespalib/gtest/gtest.h new file mode 100644 index 00000000000..ed4e65b71cb --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/gtest/gtest.h @@ -0,0 +1,14 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <gtest/gtest.h> + +/** + * Macro for creating a main function that runs all gtests. + */ +#define GTEST_MAIN_RUN_ALL_TESTS \ +int \ +main(int argc, char* argv[]) \ +{ \ + ::testing::InitGoogleTest(&argc, argv); \ + return RUN_ALL_TESTS(); \ +} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.h b/staging_vespalib/src/vespa/vespalib/metrics/handle.h index 3e32945ceed..8ba2197a55b 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/handle.h @@ -20,10 +20,13 @@ public: explicit Handle(size_t id) : _id(id) {} size_t id() const { return _id; } - static constexpr Handle empty_handle = Handle(); + static const Handle empty_handle; }; template <typename T> +const Handle<T> Handle<T>::empty_handle; + +template <typename T> bool operator< (const Handle<T> &a, const Handle<T> &b) noexcept { diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index 84bdc0a45d0..6cc2af1fc90 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -116,12 +116,13 @@ Identifiable::RuntimeClass::RuntimeClass(RuntimeInfo * info_) : { if (_rt->_factory) { Identifiable::UP tmp(create()); + Identifiable &tmpref = *tmp; assert(id() == tmp->getClass().id()); //printf("Class %s has typeinfo %s\n", name(), typeid(*tmp).name()); for (const RuntimeInfo * curr = _rt; curr && curr != curr->_base; curr = curr->_base) { //printf("\tinherits %s : typeinfo = %s\n", curr->_name, curr->_typeId().name()); if ( ! curr->_tryCast(tmp.get()) ) { - throw std::runtime_error(make_string("(%s, %s) is not a baseclass of (%s, %s)", curr->_name, curr->_typeId().name(), name(), typeid(*tmp).name())); + throw std::runtime_error(make_string("(%s, %s) is not a baseclass of (%s, %s)", curr->_name, curr->_typeId().name(), name(), typeid(tmpref).name())); } } } diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index beb7e93d343..d32afeeabee 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -88,7 +88,6 @@ private: Alloc _buffer; uint32_t _position; - double _growFactor; }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp index 6399faeefec..c4c18a762af 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp @@ -117,7 +117,7 @@ bool isLegalName(const std::string& name) { void convertToLegalName(std::string& name) { if (name.size() == 0) { - name == "__no_name__"; + name = "__no_name__"; } else { if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { name[0] = '_'; @@ -450,7 +450,6 @@ template XmlAttribute::XmlAttribute(const std::string &, std::string, unsigned i template XmlAttribute::XmlAttribute(const std::string &, vespalib::string, unsigned int); template XmlAttribute::XmlAttribute(const std::string &, vespalib::stringref, unsigned int); template XmlAttribute::XmlAttribute(const std::string &, CharP, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, ConstCharP, unsigned int); template XmlAttribute::XmlAttribute(const std::string &, bool, unsigned int); template XmlAttribute::XmlAttribute(const std::string &, int16_t, unsigned int); template XmlAttribute::XmlAttribute(const std::string &, int32_t, unsigned int); diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index 8fd80d47bb3..a720cd191e4 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -248,8 +248,7 @@ TestDistributorApp::configure(vespalib::stringref id) TestDistributorApp::TestDistributorApp(vespalib::stringref configId) : TestStorageApp( - StorageComponentRegisterImpl::UP( - new DistributorComponentRegisterImpl), + std::make_unique<DistributorComponentRegisterImpl>(), lib::NodeType::DISTRIBUTOR, getIndexFromConfig(configId), configId), _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( TestStorageApp::getComponentRegister())), @@ -263,7 +262,7 @@ TestDistributorApp::TestDistributorApp(vespalib::stringref configId) TestDistributorApp::TestDistributorApp(NodeIndex index, vespalib::stringref configId) : TestStorageApp( - StorageComponentRegisterImpl::UP(new StorageComponentRegisterImpl), + std::make_unique<DistributorComponentRegisterImpl>(), lib::NodeType::DISTRIBUTOR, index, configId), _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( TestStorageApp::getComponentRegister())), diff --git a/storage/src/tests/gtest_runner.cpp b/storage/src/tests/gtest_runner.cpp index f3400187ff4..762f6dba6db 100644 --- a/storage/src/tests/gtest_runner.cpp +++ b/storage/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storage_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index be926722497..27c13a3707e 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -19,7 +19,7 @@ using document::test::makeDocumentBucket; namespace storage { struct BouncerTest : public CppUnit::TestFixture { - std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<TestStorageApp> _node; std::unique_ptr<DummyStorageLink> _upper; Bouncer* _manager; DummyStorageLink* _lower; @@ -29,6 +29,8 @@ struct BouncerTest : public CppUnit::TestFixture { void setUp() override; void tearDown() override; + void setUpAsNode(const lib::NodeType& type); + void testFutureTimestamp(); void testAllowNotifyBucketChangeEvenWhenDistributorDown(); void rejectLowerPrioritizedFeedMessagesWhenConfigured(); @@ -40,6 +42,7 @@ struct BouncerTest : public CppUnit::TestFixture { void internalOperationsAreNotRejected(); void outOfBoundsConfigValuesThrowException(); void abort_request_when_derived_bucket_space_node_state_is_marked_down(); + void client_operations_are_allowed_through_on_cluster_state_down_distributor(); CPPUNIT_TEST_SUITE(BouncerTest); CPPUNIT_TEST(testFutureTimestamp); @@ -53,6 +56,7 @@ struct BouncerTest : public CppUnit::TestFixture { CPPUNIT_TEST(internalOperationsAreNotRejected); CPPUNIT_TEST(outOfBoundsConfigValuesThrowException); CPPUNIT_TEST(abort_request_when_derived_bucket_space_node_state_is_marked_down); + CPPUNIT_TEST(client_operations_are_allowed_through_on_cluster_state_down_distributor); CPPUNIT_TEST_SUITE_END(); using Priority = api::StorageMessage::Priority; @@ -81,39 +85,42 @@ CPPUNIT_TEST_SUITE_REGISTRATION(BouncerTest); BouncerTest::BouncerTest() : _node(), _upper(), - _manager(0), - _lower(0) + _manager(nullptr), + _lower(nullptr) { } -void -BouncerTest::setUp() { - try{ - vdstestlib::DirConfig config(getStandardConfig(true)); - _node.reset(new TestServiceLayerApp( - DiskCount(1), NodeIndex(2), config.getConfigId())); - _upper.reset(new DummyStorageLink()); - _manager = new Bouncer(_node->getComponentRegister(), - config.getConfigId()); - _lower = new DummyStorageLink(); - _upper->push_back(std::unique_ptr<StorageLink>(_manager)); - _upper->push_back(std::unique_ptr<StorageLink>(_lower)); - _upper->open(); - } catch (std::exception& e) { - std::cerr << "Failed to static initialize objects: " << e.what() - << "\n"; +void BouncerTest::setUpAsNode(const lib::NodeType& type) { + vdstestlib::DirConfig config(getStandardConfig(type == lib::NodeType::STORAGE)); + if (type == lib::NodeType::STORAGE) { + _node.reset(new TestServiceLayerApp(DiskCount(1), NodeIndex(2), config.getConfigId())); + } else { + _node.reset(new TestDistributorApp(NodeIndex(2), config.getConfigId())); } + _upper.reset(new DummyStorageLink()); + _manager = new Bouncer(_node->getComponentRegister(), config.getConfigId()); + _lower = new DummyStorageLink(); + _upper->push_back(std::unique_ptr<StorageLink>(_manager)); + _upper->push_back(std::unique_ptr<StorageLink>(_lower)); + _upper->open(); _node->getClock().setAbsoluteTimeInSeconds(10); } void +BouncerTest::setUp() { + setUpAsNode(lib::NodeType::STORAGE); +} + +void BouncerTest::tearDown() { - _manager = 0; - _lower = 0; - _upper->close(); - _upper->flush(); - _upper.reset(0); - _node.reset(0); + _manager = nullptr; + _lower = nullptr; + if (_upper) { + _upper->close(); + _upper->flush(); + _upper.reset(); + } + _node.reset(); } std::shared_ptr<api::StorageCommand> @@ -334,13 +341,31 @@ makeClusterStateBundle(const vespalib::string &baselineState, const std::map<doc void BouncerTest::abort_request_when_derived_bucket_space_node_state_is_marked_down() { + CPPUNIT_ASSERT_EQUAL(uint64_t(0), _manager->metrics().unavailable_node_aborts.getValue()); + auto state = makeClusterStateBundle("distributor:3 storage:3", {{ document::FixedBucketSpaces::default_space(), "distributor:3 storage:3 .2.s:d" }}); _node->getNodeStateUpdater().setClusterStateBundle(state); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); assertMessageBouncedWithAbort(); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().unavailable_node_aborts.getValue()); + _upper->reset(); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::global_space())); assertMessageNotBounced(); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().unavailable_node_aborts.getValue()); +} + +void BouncerTest::client_operations_are_allowed_through_on_cluster_state_down_distributor() { + tearDown(); + setUpAsNode(lib::NodeType::DISTRIBUTOR); + + // Distributor states never vary across bucket spaces, so not necessary to test with + // anything except baseline state here. + auto state = makeClusterStateBundle("distributor:3 .2.s:d storage:3", {}); + _node->getNodeStateUpdater().setClusterStateBundle(state); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); + assertMessageNotBounced(); + CPPUNIT_ASSERT_EQUAL(uint64_t(0), _manager->metrics().unavailable_node_aborts.getValue()); } } // storage diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 07b9e2a48a6..6e15ee03d12 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -30,6 +30,8 @@ public: Stats(const Stats &rhs); ~Stats() { } + Stats &operator=(const Stats &rhs) = default; + /** * For each node N, look at all the buckets that have or should have a * bucket copy on that node. For each of these buckets, there is a diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 82cf64423c6..0541c7322f1 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -118,6 +118,7 @@ Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, << " when node is in state " << state.toString(true); append_node_identity(ost); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, ost.str())); + _metrics->unavailable_node_aborts.inc(); sendUp(reply); } @@ -158,6 +159,10 @@ Bouncer::clusterIsUp() const return (*_clusterState == lib::State::UP); } +bool Bouncer::isDistributor() const { + return (_component.getNodeType() == lib::NodeType::DISTRIBUTOR); +} + uint64_t Bouncer::extractMutationTimestampIfAny(const api::StorageMessage& msg) { @@ -243,12 +248,11 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) int feedPriorityLowerBound; { vespalib::LockGuard lock(_lock); - state = &getDerivedNodeState(msg->getBucket().getBucketSpace()).getState(); - maxClockSkewInSeconds = _config->maxClockSkewSeconds; + state = &getDerivedNodeState(msg->getBucket().getBucketSpace()).getState(); + maxClockSkewInSeconds = _config->maxClockSkewSeconds; abortLoadWhenClusterDown = _config->stopExternalLoadWhenClusterDown; - isInAvailableState = state->oneOf( - _config->stopAllLoadWhenNodestateNotIn.c_str()); - feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; + isInAvailableState = state->oneOf(_config->stopAllLoadWhenNodestateNotIn.c_str()); + feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; } // Special case for messages storage nodes are expected to get during // initializing. Request bucket info will be queued so storage can @@ -258,13 +262,14 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) { return false; } - if (!isInAvailableState) { + const bool externalLoad = isExternalLoad(type); + if (!isInAvailableState && !(isDistributor() && externalLoad)) { abortCommandForUnavailableNode(*msg, *state); return true; } // Allow all internal load to go through at this point - if (!isExternalLoad(type)) { + if (!externalLoad) { return false; } if (priorityRejectionIsEnabled(feedPriorityLowerBound) diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index 258b0b18a32..19c8e084c7d 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -70,6 +70,8 @@ private: bool clusterIsUp() const; + bool isDistributor() const; + bool isExternalLoad(const api::MessageType&) const noexcept; bool isExternalWriteOperation(const api::MessageType&) const noexcept; diff --git a/storage/src/vespa/storage/storageserver/bouncer_metrics.cpp b/storage/src/vespa/storage/storageserver/bouncer_metrics.cpp index c0fac35263e..5f3ba5a236d 100644 --- a/storage/src/vespa/storage/storageserver/bouncer_metrics.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer_metrics.cpp @@ -7,7 +7,9 @@ namespace storage { BouncerMetrics::BouncerMetrics() : MetricSet("bouncer", {}, "Metrics for Bouncer component", nullptr), clock_skew_aborts("clock_skew_aborts", {}, "Number of client operations that were aborted due to " - "clock skew between sender and receiver exceeding acceptable range", this) + "clock skew between sender and receiver exceeding acceptable range", this), + unavailable_node_aborts("unavailable_node_aborts", {}, "Number of operations that were aborted due " + "to the node (or target bucket space) being unavailable", this) { } diff --git a/storage/src/vespa/storage/storageserver/bouncer_metrics.h b/storage/src/vespa/storage/storageserver/bouncer_metrics.h index 9beca6c73b7..9842bed1c6f 100644 --- a/storage/src/vespa/storage/storageserver/bouncer_metrics.h +++ b/storage/src/vespa/storage/storageserver/bouncer_metrics.h @@ -8,6 +8,7 @@ namespace storage { struct BouncerMetrics : metrics::MetricSet { metrics::LongCountMetric clock_skew_aborts; + metrics::LongCountMetric unavailable_node_aborts; BouncerMetrics(); ~BouncerMetrics() override; diff --git a/storageapi/src/tests/gtest_runner.cpp b/storageapi/src/tests/gtest_runner.cpp index c37be7231ac..d499a54af50 100644 --- a/storageapi/src/tests/gtest_runner.cpp +++ b/storageapi/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storageapi_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/storageframework/src/tests/gtest_runner.cpp b/storageframework/src/tests/gtest_runner.cpp index ce0310d7fa1..e9a1c9b1ed8 100644 --- a/storageframework/src/tests/gtest_runner.cpp +++ b/storageframework/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storageframework_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 15453078b35..0b4a83bf714 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -9,6 +9,7 @@ using vespalib::SlaveProc; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; +using vespalib::SimpleBuffer; auto null_crypto = std::make_shared<vespalib::NullCryptoEngine>(); diff --git a/vbench/src/tests/http_client/http_client_test.cpp b/vbench/src/tests/http_client/http_client_test.cpp index 36e04e012a0..1398b1ebc74 100644 --- a/vbench/src/tests/http_client/http_client_test.cpp +++ b/vbench/src/tests/http_client/http_client_test.cpp @@ -8,6 +8,8 @@ using namespace vbench; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; +using vespalib::SimpleBuffer; + auto null_crypto = std::make_shared<vespalib::NullCryptoEngine>(); void checkMemory(const string &expect, const Memory &mem) { diff --git a/vbench/src/tests/socket/socket_test.cpp b/vbench/src/tests/socket/socket_test.cpp index 8623c946c84..bcdaecdc7dc 100644 --- a/vbench/src/tests/socket/socket_test.cpp +++ b/vbench/src/tests/socket/socket_test.cpp @@ -11,6 +11,7 @@ auto null_crypto = std::make_shared<vespalib::NullCryptoEngine>(); auto tls_crypto = std::make_shared<vespalib::TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()); using OutputWriter = vespalib::OutputWriter; +using vespalib::CryptoEngine; const size_t numLines = 100; diff --git a/vbench/src/vbench/core/handler.cpp b/vbench/src/vbench/core/handler.cpp index 124bbf69841..5f6127f6a32 100644 --- a/vbench/src/vbench/core/handler.cpp +++ b/vbench/src/vbench/core/handler.cpp @@ -10,6 +10,6 @@ struct DummyItem {}; } // namespace vbench::<unnamed> -template class Handler<DummyItem>; +template struct Handler<DummyItem>; } // namespace vbench diff --git a/vbench/src/vbench/core/line_reader.h b/vbench/src/vbench/core/line_reader.h index 348971626e6..0f4b0d40ede 100644 --- a/vbench/src/vbench/core/line_reader.h +++ b/vbench/src/vbench/core/line_reader.h @@ -8,7 +8,6 @@ namespace vbench { using Input = vespalib::Input; -using InputReader = vespalib::InputReader; /** * Concrete utility class used to read individual lines of text from @@ -17,6 +16,8 @@ using InputReader = vespalib::InputReader; **/ class LineReader { +public: + using InputReader = vespalib::InputReader; private: InputReader _input; diff --git a/vbench/src/vbench/core/provider.cpp b/vbench/src/vbench/core/provider.cpp index f547263cbb7..b5ad275ecd7 100644 --- a/vbench/src/vbench/core/provider.cpp +++ b/vbench/src/vbench/core/provider.cpp @@ -10,6 +10,6 @@ struct DummyItem {}; } // namespace vbench::<unnamed> -template class Provider<DummyItem>; +template struct Provider<DummyItem>; } // namespace vbench diff --git a/vbench/src/vbench/core/socket.cpp b/vbench/src/vbench/core/socket.cpp index a11e03c5b22..822b96b2c07 100644 --- a/vbench/src/vbench/core/socket.cpp +++ b/vbench/src/vbench/core/socket.cpp @@ -48,7 +48,7 @@ Socket::~Socket() } } -Memory +Socket::Memory Socket::obtain() { if ((_input.get().size == 0) && !_eof && !_taint) { @@ -72,7 +72,7 @@ Socket::evict(size_t bytes) return *this; } -WritableMemory +Socket::WritableMemory Socket::reserve(size_t bytes) { return _output.reserve(bytes); diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h index 337384776c2..0e8848e8292 100644 --- a/vbench/src/vbench/core/socket.h +++ b/vbench/src/vbench/core/socket.h @@ -13,16 +13,17 @@ namespace vbench { -using Input = vespalib::Input; -using Memory = vespalib::Memory; -using Output = vespalib::Output; -using SimpleBuffer = vespalib::SimpleBuffer; -using WritableMemory = vespalib::WritableMemory; -using CryptoEngine = vespalib::CryptoEngine; -using SyncCryptoSocket = vespalib::SyncCryptoSocket; class Socket : public Stream { +public: + using Input = vespalib::Input; + using Memory = vespalib::Memory; + using Output = vespalib::Output; + using SimpleBuffer = vespalib::SimpleBuffer; + using WritableMemory = vespalib::WritableMemory; + using CryptoEngine = vespalib::CryptoEngine; + using SyncCryptoSocket = vespalib::SyncCryptoSocket; private: SyncCryptoSocket::UP _socket; SimpleBuffer _input; @@ -43,6 +44,8 @@ public: }; struct ServerSocket { + using CryptoEngine = vespalib::CryptoEngine; + using SyncCryptoSocket = vespalib::SyncCryptoSocket; vespalib::ServerSocket server_socket; ServerSocket() : server_socket(0) {} int port() const { return server_socket.address().port(); } diff --git a/vbench/src/vbench/http/http_client.h b/vbench/src/vbench/http/http_client.h index 77a53057740..9feb3167611 100644 --- a/vbench/src/vbench/http/http_client.h +++ b/vbench/src/vbench/http/http_client.h @@ -16,6 +16,8 @@ namespace vbench { **/ class HttpClient { +public: + using CryptoEngine = vespalib::CryptoEngine; private: static const size_t WRITE_SIZE = 2000; diff --git a/vbench/src/vbench/http/http_connection.h b/vbench/src/vbench/http/http_connection.h index b2742c3e0fd..fe768b3dbfe 100644 --- a/vbench/src/vbench/http/http_connection.h +++ b/vbench/src/vbench/http/http_connection.h @@ -21,6 +21,7 @@ private: double _lastUsed; public: + using CryptoEngine = vespalib::CryptoEngine; typedef std::unique_ptr<HttpConnection> UP; HttpConnection(CryptoEngine &crypto, const ServerSpec &server); diff --git a/vbench/src/vbench/http/http_connection_pool.h b/vbench/src/vbench/http/http_connection_pool.h index 919eceb1fef..5dd9eb6361c 100644 --- a/vbench/src/vbench/http/http_connection_pool.h +++ b/vbench/src/vbench/http/http_connection_pool.h @@ -20,6 +20,7 @@ class HttpConnectionPool private: typedef vespalib::ArrayQueue<HttpConnection::UP> Queue; typedef std::map<ServerSpec, size_t> Map; + using CryptoEngine = vespalib::CryptoEngine; vespalib::Lock _lock; Map _map; diff --git a/vbench/src/vbench/test/simple_http_result_handler.h b/vbench/src/vbench/test/simple_http_result_handler.h index a387ba29e95..d8185aae363 100644 --- a/vbench/src/vbench/test/simple_http_result_handler.h +++ b/vbench/src/vbench/test/simple_http_result_handler.h @@ -8,10 +8,10 @@ namespace vbench { -using SimpleBuffer = vespalib::SimpleBuffer; - class SimpleHttpResultHandler : public HttpResultHandler { +public: + using SimpleBuffer = vespalib::SimpleBuffer; private: std::vector<std::pair<string, string> > _headers; SimpleBuffer _content; diff --git a/vbench/src/vbench/vbench/request_scheduler.cpp b/vbench/src/vbench/vbench/request_scheduler.cpp index e12d89c1c04..9cd778789ec 100644 --- a/vbench/src/vbench/vbench/request_scheduler.cpp +++ b/vbench/src/vbench/vbench/request_scheduler.cpp @@ -49,7 +49,8 @@ RequestScheduler::abort() void RequestScheduler::handle(Request::UP request) { - _queue.insert(std::move(request), request->scheduledTime()); + double scheduledTime = request->scheduledTime(); + _queue.insert(std::move(request), scheduledTime); } void diff --git a/vbench/src/vbench/vbench/request_scheduler.h b/vbench/src/vbench/vbench/request_scheduler.h index f7f4a542d6f..2f9e9177c53 100644 --- a/vbench/src/vbench/vbench/request_scheduler.h +++ b/vbench/src/vbench/vbench/request_scheduler.h @@ -34,6 +34,7 @@ private: void run() override; public: typedef std::unique_ptr<RequestScheduler> UP; + using CryptoEngine = vespalib::CryptoEngine; RequestScheduler(CryptoEngine::SP crypto, Handler<Request> &next, size_t numWorkers); void abort(); void handle(Request::UP request) override; diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp index f04b3bcca7f..4f6efadfbdd 100644 --- a/vbench/src/vbench/vbench/vbench.cpp +++ b/vbench/src/vbench/vbench/vbench.cpp @@ -4,6 +4,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> +using vespalib::CryptoEngine; + namespace vbench { namespace { diff --git a/vdslib/src/tests/gtest_runner.cpp b/vdslib/src/tests/gtest_runner.cpp index d10f7182c07..e200813ef61 100644 --- a/vdslib/src/tests/gtest_runner.cpp +++ b/vdslib/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("vdslib_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java index 28f028832b4..bd2f76bac52 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java @@ -15,8 +15,7 @@ import javax.net.ssl.SSLContext; import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.util.Objects; -import java.util.logging.Logger; +import java.util.Optional; /** * Implementation of AWSCredentialsProvider using com.yahoo.vespa.athenz.client.zts.ZtsClient @@ -25,8 +24,6 @@ import java.util.logging.Logger; */ public class AwsCredentialsProvider implements AWSCredentialsProvider { - private static final Logger logger = Logger.getLogger(AwsCredentialsProvider.class.getName()); - private final static Duration MIN_EXPIRY = Duration.ofMinutes(5); private final AthenzDomain athenzDomain; private final AwsRole awsRole; @@ -72,8 +69,8 @@ public class AwsCredentialsProvider implements AWSCredentialsProvider { /* * Checks credential expiration, returns true if it will expipre in the next MIN_EXPIRY minutes */ - private static boolean shouldRefresh(AwsTemporaryCredentials credentials) { - Instant expiration = credentials.expiration(); - return Objects.isNull(expiration) || expiration.minus(MIN_EXPIRY).isAfter(Instant.now()); + static boolean shouldRefresh(AwsTemporaryCredentials credentials) { + Instant expiration = Optional.ofNullable(credentials).map(AwsTemporaryCredentials::expiration).orElse(Instant.EPOCH); + return Duration.between(Instant.now(), expiration).toMinutes() < MIN_EXPIRY.toMinutes(); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/package-info.java new file mode 100644 index 00000000000..74ef35a1e50 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.athenz.client.aws; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java new file mode 100644 index 00000000000..d637dcae14c --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java @@ -0,0 +1,35 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.aws; + +import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; +import org.junit.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class AwsCredentialProviderTest { + + @Test + public void refreshes_correctly() { + Clock clock = Clock.systemUTC(); + // Does not require refresh when expires in 10 minutes + assertFalse(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().plus(Duration.ofMinutes(10))))); + + // Requires refresh when expires in 3 minutes + assertTrue(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().plus(Duration.ofMinutes(3))))); + + // Requires refresh when expired + assertTrue(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().minus(Duration.ofMinutes(1))))); + + // Refreshes when no credentials provided + assertTrue(AwsCredentialsProvider.shouldRefresh(null)); + } + + private AwsTemporaryCredentials getCredentials(Instant expiration) { + return new AwsTemporaryCredentials("accesskey", "secretaccesskey", "sessionToken", expiration); + } +} diff --git a/vespalib/src/tests/arrayqueue/arrayqueue.cpp b/vespalib/src/tests/arrayqueue/arrayqueue.cpp index cb1857a0594..5178da30eca 100644 --- a/vespalib/src/tests/arrayqueue/arrayqueue.cpp +++ b/vespalib/src/tests/arrayqueue/arrayqueue.cpp @@ -25,6 +25,7 @@ struct Int ++ctorCnt; ++aliveCnt; } + Int &operator=(const Int &rhs) = default; operator int() const { return value; } ~Int() { ++dtorCnt; diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp index 67077f86f1e..e943ba68237 100644 --- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp +++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp @@ -191,6 +191,8 @@ void verify_handshake(CryptoSocket &socket) { case CryptoSocket::HandshakeResult::NEED_WRITE: ASSERT_TRUE(selector.wait_writable()); break; + case CryptoSocket::HandshakeResult::NEED_WORK: + socket.do_handshake_work(); } } } diff --git a/vespalib/src/vespa/vespalib/component/versionspecification.cpp b/vespalib/src/vespa/vespalib/component/versionspecification.cpp index 84cf446e8c5..6400097c9bf 100644 --- a/vespalib/src/vespa/vespalib/component/versionspecification.cpp +++ b/vespalib/src/vespa/vespalib/component/versionspecification.cpp @@ -24,6 +24,8 @@ VersionSpecification::VersionSpecification(const VersionSpecification &) = defau VersionSpecification::~VersionSpecification() = default; +VersionSpecification &VersionSpecification::operator=(const VersionSpecification &rhs) = default; + void VersionSpecification::initialize() { diff --git a/vespalib/src/vespa/vespalib/component/versionspecification.h b/vespalib/src/vespa/vespalib/component/versionspecification.h index 399db123a7a..88d825011f3 100644 --- a/vespalib/src/vespa/vespalib/component/versionspecification.h +++ b/vespalib/src/vespa/vespalib/component/versionspecification.h @@ -70,6 +70,7 @@ public: VersionSpecification(const VersionSpecification &); ~VersionSpecification(); + VersionSpecification &operator=(const VersionSpecification &rhs); /** * @brief Creates a version specification from the specified string. * diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index bf875a70985..a8808741806 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -44,6 +44,7 @@ public: NullCryptoSocket(SocketHandle socket) : _socket(std::move(socket)) {} int get_fd() const override { return _socket.get(); } HandshakeResult handshake() override { return HandshakeResult::DONE; } + void do_handshake_work() override {} size_t min_read_buffer_size() const override { return 1; } ssize_t read(char *buf, size_t len) override { return _socket.read(buf, len); } ssize_t drain(char *, size_t) override { return 0; } @@ -118,6 +119,7 @@ public: } return HandshakeResult::DONE; } + void do_handshake_work() override {} size_t min_read_buffer_size() const override { return 1; } ssize_t read(char *buf, size_t len) override { if (_input.obtain().size == 0) { diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h index 3a95cf33add..c409e5a0e83 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h @@ -26,7 +26,7 @@ struct CryptoSocket { **/ virtual int get_fd() const = 0; - enum class HandshakeResult { FAIL, DONE, NEED_READ, NEED_WRITE }; + enum class HandshakeResult { FAIL, DONE, NEED_READ, NEED_WRITE, NEED_WORK }; /** * Try to progress the initial connection handshake. Handshaking @@ -35,11 +35,21 @@ struct CryptoSocket { * permitted. This function will be called multiple times until * the status is either DONE or FAIL. When NEED_READ or NEED_WRITE * is returned, the handshake function will be called again when - * the appropriate io event has triggered. + * the appropriate io event has triggered. When NEED_WORK is + * returned, the 'do_handshake_work' function will be called + * exactly once before this function is called again. **/ virtual HandshakeResult handshake() = 0; /** + * This function is called to perform possibly expensive work + * needed by the 'handshake' function. The work is done by a + * separate function to enable performing it outside the critical + * path (transport thread). + **/ + virtual void do_handshake_work() = 0; + + /** * This function should be called after handshaking has completed * before calling the read function. It dictates the minimum size * of the application read buffer presented to the read diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp index e9b1579fbf6..29388035bda 100644 --- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp @@ -94,16 +94,19 @@ SyncCryptoSocket::create(CryptoEngine &engine, SocketHandle socket, bool is_serv { auto crypto_socket = engine.create_crypto_socket(std::move(socket), is_server); set_blocking(crypto_socket->get_fd()); - auto hs_res = crypto_socket->handshake(); - while ((hs_res == CryptoSocket::HandshakeResult::NEED_READ) || - (hs_res == CryptoSocket::HandshakeResult::NEED_WRITE)) - { - hs_res = crypto_socket->handshake(); - } - if (hs_res != CryptoSocket::HandshakeResult::DONE) { - return std::unique_ptr<SyncCryptoSocket>(nullptr); + for (;;) { + switch (crypto_socket->handshake()) { + case CryptoSocket::HandshakeResult::FAIL: + return std::unique_ptr<SyncCryptoSocket>(nullptr); + case CryptoSocket::HandshakeResult::DONE: + return UP(new SyncCryptoSocket(std::move(crypto_socket))); + case CryptoSocket::HandshakeResult::NEED_READ: + case CryptoSocket::HandshakeResult::NEED_WRITE: + break; + case CryptoSocket::HandshakeResult::NEED_WORK: + crypto_socket->do_handshake_work(); + } } - return UP(new SyncCryptoSocket(std::move(crypto_socket))); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp index 46c3e2d3195..660eee95132 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp @@ -72,16 +72,13 @@ CryptoCodecAdapter::handshake() for (;;) { auto in = _input.obtain(); auto out = _output.reserve(_codec->min_encode_buffer_size()); - ::vespalib::net::tls::HandshakeResult hs_res; - while ((hs_res = _codec->handshake(in.data, in.size, out.data, out.size)).needs_work()) { - _codec->do_handshake_work(); - } + auto hs_res = _codec->handshake(in.data, in.size, out.data, out.size); _input.evict(hs_res.bytes_consumed); _output.commit(hs_res.bytes_produced); switch (hs_res.state) { case ::vespalib::net::tls::HandshakeResult::State::Failed: return HandshakeResult::FAIL; case ::vespalib::net::tls::HandshakeResult::State::Done: return hs_try_flush(); - case ::vespalib::net::tls::HandshakeResult::State::NeedsWork: abort(); // Impossible + case ::vespalib::net::tls::HandshakeResult::State::NeedsWork: return HandshakeResult::NEED_WORK; case ::vespalib::net::tls::HandshakeResult::State::NeedsMorePeerData: auto flush_res = hs_try_flush(); if (flush_res != HandshakeResult::DONE) { @@ -96,6 +93,12 @@ CryptoCodecAdapter::handshake() return HandshakeResult::DONE; } +void +CryptoCodecAdapter::do_handshake_work() +{ + _codec->do_handshake_work(); +} + ssize_t CryptoCodecAdapter::read(char *buf, size_t len) { diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h index 6af04d67d41..4926c757bfb 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h @@ -37,6 +37,7 @@ public: void inject_read_data(const char *buf, size_t len) override; int get_fd() const override { return _socket.get(); } HandshakeResult handshake() override; + void do_handshake_work() override; size_t min_read_buffer_size() const override { return _codec->min_decode_buffer_size(); } ssize_t read(char *buf, size_t len) override; ssize_t drain(char *, size_t) override; diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index b6969250ae5..9af6703acbc 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -61,6 +61,7 @@ public: } return HandshakeResult::DONE; } + void do_handshake_work() override {} size_t min_read_buffer_size() const override { return 1; } ssize_t read(char *buf, size_t len) override { int drain_result = drain(buf, len); diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h index bd4ea9d7d50..cd2d84bcc08 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.h @@ -25,6 +25,7 @@ public: MaybeTlsCryptoSocket(SocketHandle socket, std::shared_ptr<AbstractTlsCryptoEngine> tls_engine); int get_fd() const override { return _socket->get_fd(); } HandshakeResult handshake() override { return _socket->handshake(); } + void do_handshake_work() override { _socket->do_handshake_work(); } size_t min_read_buffer_size() const override { return _socket->min_read_buffer_size(); } ssize_t read(char *buf, size_t len) override { return _socket->read(buf, len); } ssize_t drain(char *buf, size_t len) override { return _socket->drain(buf, len); } diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index 2bed10bfd2d..cfee71e4a7d 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -102,11 +102,14 @@ HttpConnection::set_state(State state, bool read, bool write) void HttpConnection::do_handshake() { - switch (_socket->handshake()) { - case vespalib::CryptoSocket::HandshakeResult::FAIL: return set_state(State::NOTIFY, false, false); - case vespalib::CryptoSocket::HandshakeResult::DONE: return set_state(State::READ_REQUEST, true, false); - case vespalib::CryptoSocket::HandshakeResult::NEED_READ: return set_state(State::HANDSHAKE, true, false); - case vespalib::CryptoSocket::HandshakeResult::NEED_WRITE: return set_state(State::HANDSHAKE, false, true); + for (;;) { + switch (_socket->handshake()) { + case vespalib::CryptoSocket::HandshakeResult::FAIL: return set_state(State::NOTIFY, false, false); + case vespalib::CryptoSocket::HandshakeResult::DONE: return set_state(State::READ_REQUEST, true, false); + case vespalib::CryptoSocket::HandshakeResult::NEED_READ: return set_state(State::HANDSHAKE, true, false); + case vespalib::CryptoSocket::HandshakeResult::NEED_WRITE: return set_state(State::HANDSHAKE, false, true); + case vespalib::CryptoSocket::HandshakeResult::NEED_WORK: _socket->do_handshake_work(); + } } } diff --git a/vespalib/src/vespa/vespalib/test/time_tracer.h b/vespalib/src/vespa/vespalib/test/time_tracer.h index c872ed84bbc..2e3a2d9e351 100644 --- a/vespalib/src/vespa/vespalib/test/time_tracer.h +++ b/vespalib/src/vespa/vespalib/test/time_tracer.h @@ -128,13 +128,13 @@ private: class ThreadState { private: uint32_t _thread_id; - mutable std::atomic_flag _lock; + mutable std::atomic_flag _lock = ATOMIC_FLAG_INIT; vespalib::Stash _stash; const LogEntry * _list; public: using UP = std::unique_ptr<ThreadState>; ThreadState(uint32_t thread_id) - : _thread_id(thread_id), _lock{ATOMIC_FLAG_INIT}, _stash(64 * 1024), _list(nullptr) {} + : _thread_id(thread_id), _stash(64 * 1024), _list(nullptr) {} uint32_t thread_id() const { return _thread_id; } const LogEntry *get_log_entries() const { Guard guard(_lock); diff --git a/vespalib/src/vespa/vespalib/util/gencnt.h b/vespalib/src/vespa/vespalib/util/gencnt.h index 43e127c242e..7bfc5a7e49b 100644 --- a/vespalib/src/vespa/vespalib/util/gencnt.h +++ b/vespalib/src/vespa/vespalib/util/gencnt.h @@ -31,6 +31,8 @@ public: **/ GenCnt(uint32_t val) : _val(val) {} + GenCnt(const GenCnt &rhs) = default; + /** * @brief empty destructor **/ diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 20a4b425d72..4bb13f7d791 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -25,7 +25,7 @@ static ns_log::Logger logger(__VA_ARGS__) static ns_log::Logger *logger=NULL; \ static bool logInitialised = false; \ static const char *logName = x; \ -static const char *rcsId = id +static const char *indirectRcsId = id #define LOG_RCSID(x) \ @@ -55,7 +55,7 @@ do { \ if (!logInitialised) { \ logInitialised = true; \ logger = static_cast<Logger *>(malloc(sizeof *logger)); \ - new (logger) Logger(logName, rcsId); \ + new (logger) Logger(logName, indirectRcsId); \ } #define LOG_INDIRECT(level, ...) \ do { \ |