diff options
231 files changed, 4084 insertions, 4223 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index 23389de3fad..d2fe304a72c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -39,6 +39,8 @@ public class MasterElectionTest extends FleetControllerTest { @Rule public TestRule cleanupZookeeperLogsOnSuccess = new CleanupZookeeperLogsOnSuccess(); + private static int defaultZkSessionTimeoutInMillis() { return 10_000; } + protected void setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions options) throws Exception { if (zooKeeperServer == null) { zooKeeperServer = new ZooKeeperTestServer(); @@ -46,7 +48,7 @@ public class MasterElectionTest extends FleetControllerTest { slobrok = new Slobrok(); usingFakeTimer = useFakeTimer; this.options = options; - this.options.zooKeeperSessionTimeout = 10 * timeoutMS; + this.options.zooKeeperSessionTimeout = defaultZkSessionTimeoutInMillis(); this.options.zooKeeperServerAddress = zooKeeperServer.getAddress(); this.options.slobrokConnectionSpecs = new String[1]; this.options.slobrokConnectionSpecs[0] = "tcp/localhost:" + slobrok.port(); @@ -62,7 +64,7 @@ public class MasterElectionTest extends FleetControllerTest { int fleetControllerIndex, int fleetControllerCount) throws Exception { FleetControllerOptions options = o.clone(); - options.zooKeeperSessionTimeout = 10 * timeoutMS; + options.zooKeeperSessionTimeout = defaultZkSessionTimeoutInMillis(); options.zooKeeperServerAddress = zooKeeperServer.getAddress(); options.slobrokConnectionSpecs = new String[1]; options.slobrokConnectionSpecs[0] = "tcp/localhost:" + slobrok.port(); // Spec.fromLocalHostName(slobrok.port()).toString(); @@ -251,7 +253,6 @@ public class MasterElectionTest extends FleetControllerTest { FleetControllerOptions options = defaultOptions("mycluster"); // "Magic" port value is in range allocated to module for testing. zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342); - options.zooKeeperSessionTimeout = 100; options.masterZooKeeperCooldownPeriod = 100; setUpFleetController(2, true, options); waitForMaster(0); @@ -273,7 +274,6 @@ public class MasterElectionTest extends FleetControllerTest { public void testZooKeeperUnavailable() throws Exception { startingTest("MasterElectionTest::testZooKeeperUnavailable"); FleetControllerOptions options = defaultOptions("mycluster"); - options.zooKeeperSessionTimeout = 100; options.masterZooKeeperCooldownPeriod = 100; options.zooKeeperServerAddress = "localhost"; setUpFleetController(5, true, options); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 55373f425e0..514ca2a00f5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -56,7 +56,8 @@ public interface ModelContext { boolean useFdispatchByDefault(); boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); - boolean enableMetricsProxyContainer(); + // TODO: Remove when 7.61 is the oldest model in use + default boolean enableMetricsProxyContainer() { return false; } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 4b35af53154..0a54dd6790d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -37,7 +37,6 @@ public class TestProperties implements ModelContext.Properties { private boolean useFdispatchByDefault = true; private boolean dispatchWithProtobuf = true; private boolean useAdaptiveDispatch = false; - private boolean enableMetricsProxyContainer = false; @Override public boolean multitenant() { return multitenant; } @@ -55,7 +54,6 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } @Override public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } - @Override public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; @@ -87,10 +85,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setEnableMetricsProxyContainer(boolean enableMetricsProxyContainer) { - this.enableMetricsProxyContainer = enableMetricsProxyContainer; - return this; - } public static class Spec implements ConfigServerSpec { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 3bc38cad1d1..2a698233713 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -19,10 +19,8 @@ import java.util.LinkedHashMap; import java.util.Map; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CANONICAL_FLAVOR; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_TYPE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.FLAVOR; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; /** @@ -37,8 +35,6 @@ public class MetricsProxyContainer extends Container implements { static final class NodeDimensionNames { - static final String FLAVOR = "flavor"; - static final String CANONICAL_FLAVOR = "canonicalFlavor"; static final String CLUSTER_TYPE = "clustertype"; static final String CLUSTER_ID = "clusterid"; } @@ -118,11 +114,6 @@ public class MetricsProxyContainer extends Container implements public void getConfig(NodeDimensionsConfig.Builder builder) { Map<String, String> dimensions = new LinkedHashMap<>(); if (isHostedVespa) { - getHostResource().getFlavor().ifPresent(flavor -> { - dimensions.put(FLAVOR, flavor.name()); - dimensions.put(CANONICAL_FLAVOR, flavor.canonicalName()); - }); - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { dimensions.put(CLUSTER_TYPE, cluster.type().name()); dimensions.put(CLUSTER_ID, cluster.id().value()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index a8f0f5941b0..7a6b1064b24 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -209,7 +209,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("vespa-testrunner-components-jar-with-dependencies.jar"))); + addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); } public final void addDefaultHandlersExceptStatus() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index d2bf4b601a6..f755871ac4b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -11,7 +11,6 @@ import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_FLAVOR; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; @@ -33,7 +32,6 @@ public class MetricsProxyContainerTest { public void one_metrics_proxy_container_is_added_to_every_node() { var numberOfHosts = 4; var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(servicesWithManyNodes(), true); @@ -108,8 +106,6 @@ public class MetricsProxyContainerTest { assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.FLAVOR)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.CANONICAL_FLAVOR)); } @@ -162,7 +158,7 @@ public class MetricsProxyContainerTest { " </admin>", " <content version='1.0' id='my-content'>", " <documents />", - " <nodes count='1' flavor='" + MY_FLAVOR + "' />", + " <nodes count='1' />", " </content>", "</services>" ); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 13589c763e2..bac9d674460 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -27,7 +27,6 @@ class MetricsProxyModelTester { static final String MY_TENANT = "mytenant"; static final String MY_APPLICATION = "myapp"; static final String MY_INSTANCE = "myinstance"; - static final String MY_FLAVOR = "myflavor"; static final String CLUSTER_CONFIG_ID = "admin/metrics"; @@ -37,7 +36,6 @@ class MetricsProxyModelTester { static VespaModel getModel(String servicesXml) { var numberOfHosts = 1; var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); tester.addHosts(numberOfHosts); tester.setHosted(false); return tester.createModel(servicesXml, true); @@ -46,8 +44,7 @@ class MetricsProxyModelTester { static VespaModel getHostedModel(String servicesXml) { var numberOfHosts = 2; var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(flavorFromString(MY_FLAVOR), numberOfHosts); + tester.addHosts(numberOfHosts); tester.setHosted(true); tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); return tester.createModel(servicesXml, true); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 866c4027711..2e5acb9025d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -48,7 +48,6 @@ public class VespaModelTester { private Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private boolean enableMetricsProxyContainer = false; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -105,10 +104,6 @@ public class VespaModelTester { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; } - public void enableMetricsProxyContainer(boolean enableMetricsProxyContainer) { - this.enableMetricsProxyContainer = enableMetricsProxyContainer; - } - /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { return createModel(Zone.defaultZone(), services, true, retiredHostNames); @@ -149,8 +144,7 @@ public class VespaModelTester { .setMultitenant(true) .setHostedVespa(hosted) .setApplicationId(applicationId) - .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) - .setEnableMetricsProxyContainer(enableMetricsProxyContainer); + .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver); DeployState deployState = new DeployState.Builder() .applicationPackage(appPkg) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index de4f3a555bd..96942c53a12 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -26,6 +26,13 @@ import java.util.Set; */ public class AllocatedHosts { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String mappingKey = "mapping"; private static final String hostSpecKey = "hostSpec"; private static final String hostSpecHostNameKey = "hostName"; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 0f6a87020da..29e48ae0316 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -72,6 +72,7 @@ public enum SystemName { public static Set<SystemName> all() { return EnumSet.allOf(SystemName.class); } public static Set<SystemName> allOf(Predicate<SystemName> predicate) { - return Stream.of(values()).filter(predicate::test).collect(Collectors.toSet()); + return Stream.of(values()).filter(predicate).collect(Collectors.toUnmodifiableSet()); } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index 9f6ba29d8de..fd76dc10bdb 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -17,8 +17,4 @@ public interface ZoneApi { default RegionName getRegionName() { return getId().region(); } CloudName getCloudName(); - - default ZoneId toDeprecatedId() { - return ZoneId.from(getEnvironment(), getRegionName(), getCloudName(), getSystemName()); - } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index 5e664e00b4c..b0ac59718aa 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -20,30 +20,16 @@ public class ZoneId { private final Environment environment; private final RegionName region; - private final SystemName system; - private ZoneId(Environment environment, RegionName region, CloudName cloud, SystemName system) { + private ZoneId(Environment environment, RegionName region) { this.environment = Objects.requireNonNull(environment, "environment must be non-null"); this.region = Objects.requireNonNull(region, "region must be non-null"); - this.system = Objects.requireNonNull(system, "system must be non-null"); - } - - private ZoneId(Environment environment, RegionName region) { - this(environment, region, CloudName.defaultName(), SystemName.defaultSystem()); } public static ZoneId from(Environment environment, RegionName region) { return new ZoneId(environment, region); } - public static ZoneId from(SystemName system, Environment environment, RegionName region) { - return new ZoneId(environment, region, CloudName.defaultName(), system); - } - - public static ZoneId from(Environment environment, RegionName region, CloudName cloud, SystemName system) { - return new ZoneId(environment, region, cloud, system); - } - public static ZoneId from(String environment, String region) { return from(Environment.from(environment), RegionName.from(region)); } @@ -55,20 +41,14 @@ public class ZoneId { case 2: return from(parts[0], parts[1]); case 4: - return from(parts[2], parts[3], parts[0], parts[1]); + // Deprecated: parts[0] == cloud, parts[1] == system + // TODO: Figure out whether this can be removed + return from(parts[2], parts[3]); default: throw new IllegalArgumentException("Cannot deserialize zone id '" + value + "'"); } } - public static ZoneId from(Environment environment, RegionName region, CloudName cloud) { - return new ZoneId(environment, region, cloud, SystemName.defaultSystem()); - } - - public static ZoneId from(String environment, String region, String cloud, String system) { - return new ZoneId(Environment.from(environment), RegionName.from(region), CloudName.from(cloud), SystemName.from(system)); - } - public static ZoneId defaultId() { return new ZoneId(Environment.defaultEnvironment(), RegionName.defaultName()); } @@ -81,10 +61,6 @@ public class ZoneId { return region; } - public SystemName system() { - return system; - } - /** Returns the serialised value of this. Inverse of {@code ZoneId.from(String value)}. */ public String value() { return environment + "." + region; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java index 27d45ba7d7d..434badbe9bf 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java @@ -26,17 +26,6 @@ public class ZoneIdTest { ZoneId zoneId = ZoneId.from(environment, region); assertEquals(region, zoneId.region()); assertEquals(environment, zoneId.environment()); - assertEquals(SystemName.defaultSystem(), zoneId.system()); - - ZoneId zoneIdWithSystem = ZoneId.from(system, environment, region); - assertEquals(region, zoneIdWithSystem.region()); - assertEquals(environment, zoneIdWithSystem.environment()); - assertEquals(system, zoneIdWithSystem.system()); - - ZoneId zoneIdWithCloudAndSystem = ZoneId.from(environment, region, cloud, system); - assertEquals(region, zoneIdWithCloudAndSystem.region()); - assertEquals(environment, zoneIdWithCloudAndSystem.environment()); - assertEquals(system, zoneIdWithCloudAndSystem.system()); } @Test @@ -45,12 +34,6 @@ public class ZoneIdTest { assertEquals(environment.value() + "." + region.value(), zoneId.value()); assertEquals(ZoneId.from(zoneId.value()), zoneId); - ZoneId zoneIdWithCloudAndSystem = ZoneId.from(environment, region, cloud, system); - assertEquals(environment.value() + "." + region.value(), zoneIdWithCloudAndSystem.value()); - assertEquals(ZoneId.from(zoneIdWithCloudAndSystem.value()), zoneIdWithCloudAndSystem); - // TODO: Expect cloud and system to be part of deserialized value when the new format is supported everywhere - //assertEquals(cloud.value() + "." + system.name() + "." + environment.value() + "." + region.value() , zoneId.value()); - String serializedZoneId = "some.illegal.value"; expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Cannot deserialize zone id '" + serializedZoneId + "'"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 0279d175488..fc6667087c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -132,7 +132,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useFdispatchByDefault; private final boolean useAdaptiveDispatch; private final boolean dispatchWithProtobuf; - private final boolean enableMetricsProxyContainer; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -165,8 +164,6 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.enableMetricsProxyContainer = Flags.ENABLE_METRICS_PROXY_CONTAINER.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -218,8 +215,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } - @Override - public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index 379af7f71ea..91f9e3c8eed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -18,6 +18,13 @@ import java.util.List; */ public class ContainerEndpointSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String clusterIdField = "clusterId"; private static final String namesField = "names"; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 38ce2aa2cf2..06713d14d88 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6141,6 +6141,7 @@ "public void <init>(java.lang.String)", "public void <init>(com.yahoo.component.ComponentId)", "public com.yahoo.search.query.profile.types.QueryProfileType unfrozen()", + "public com.yahoo.processing.request.CompoundName getComponentIdAsCompoundName()", "public void setBuiltin(boolean)", "public boolean isBuiltin()", "public java.util.List inherited()", @@ -7147,7 +7148,10 @@ "public com.yahoo.data.access.Inspector inspect()", "public java.lang.String toString()", "public java.lang.String toJson()", - "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)" + "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", + "public java.lang.Double getDouble(java.lang.String)", + "public com.yahoo.tensor.Tensor getTensor(java.lang.String)", + "public java.util.Set featureNames()" ], "fields": [] }, diff --git a/container-search/src/main/java/com/yahoo/data/JsonProducer.java b/container-search/src/main/java/com/yahoo/data/JsonProducer.java index 6d925b41379..c9dc0946a3e 100644 --- a/container-search/src/main/java/com/yahoo/data/JsonProducer.java +++ b/container-search/src/main/java/com/yahoo/data/JsonProducer.java @@ -12,6 +12,7 @@ public interface JsonProducer { * be human-readable and containing embedded newlines; also the * exact indentation etc may change, so use compact=true for a * canonical format. + * * @param target the StringBuilder to append to. * @return the target passed in is also returned (to allow chaining). */ @@ -20,7 +21,8 @@ public interface JsonProducer { /** * Convenience method equivalent to: * writeJson(new StringBuilder()).toString() - * @return String containing JSON representation of this object's data. + * + * @return a String containing JSON representation of this object's data. */ default String toJson() { return writeJson(new StringBuilder()).toString(); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java index 1f60dd3d1cf..b0003f4321e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java @@ -6,9 +6,8 @@ import com.yahoo.data.access.Type; import com.yahoo.search.result.FeatureData; /** - * Class representing a "feature data" field. This was historically - * just a string containing JSON; now it's a structure of - * data (that will be rendered as JSON by default). + * Class representing a "feature data" field: A map of values which are + * either floats or tensors. */ public class FeatureDataField extends LongstringField { @@ -23,12 +22,8 @@ public class FeatureDataField extends LongstringField { @Override public Object convert(Inspector value) { - if (! value.valid()) { - return null; - } - if (value.type() == Type.STRING) { - return value.asString(); - } + if ( ! value.valid()) return null; + if (value.type() == Type.STRING) return value.asString(); return new FeatureData(value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java index 2f9c6d5b325..5de38e43c96 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java @@ -5,10 +5,6 @@ */ package com.yahoo.prelude.fastsearch; -import java.nio.ByteBuffer; - -import com.yahoo.io.SlowInflate; -import com.yahoo.text.Utf8; import com.yahoo.data.access.Inspector; /** diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java index 2330ca2382a..5f921b67702 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java @@ -3,6 +3,7 @@ package com.yahoo.prelude.searcher; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.processing.request.CompoundName; @@ -27,7 +28,7 @@ public class JSONDebugSearcher extends Searcher { private static CompoundName PROPERTYNAME = new CompoundName("dumpjson"); @Override - public Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { Result r = execution.search(query); String propertyName = query.properties().getString(PROPERTYNAME); if (propertyName != null) { diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 766af8f05fd..7105ba8c7ad 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -236,15 +236,16 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) { - String prefix = getPrefix(arguments); + CompoundName prefix = getPrefix(arguments); for (FieldDescription field : arguments.fields().values()) { for (String alias : field.getAliases()) - aliases.put(alias, new CompoundName(prefix+field.getName())); + aliases.put(alias, prefix.append(field.getName())); } } - private static String getPrefix(QueryProfileType type) { - if (type.getId().getName().equals("native")) return ""; // The arguments of this directly - return type.getId().getName() + "."; + + private static CompoundName getPrefix(QueryProfileType type) { + if (type.getId().getName().equals("native")) return CompoundName.empty; // The arguments of this directly + return type.getComponentIdAsCompoundName(); } public static void addNativeQueryProfileTypesTo(QueryProfileTypeRegistry registry) { @@ -391,22 +392,22 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * dependent objects for the appropriate subset of the given property values */ private void setFieldsFrom(Properties properties, Map<String, String> context) { - setFrom("", properties, Query.getArgumentType(), context); + setFrom(CompoundName.empty, properties, Query.getArgumentType(), context); } /** * For each field in the given query profile type, take the corresponding value from originalProperties * (if any) set it to properties(), recursively. */ - private void setFrom(String prefix, Properties originalProperties, QueryProfileType arguments, Map<String, String> context) { - prefix = prefix + getPrefix(arguments); + private void setFrom(CompoundName prefix, Properties originalProperties, QueryProfileType arguments, Map<String, String> context) { + prefix = prefix.append(getPrefix(arguments)); for (FieldDescription field : arguments.fields().values()) { if (field.getType() == FieldType.genericQueryProfileType) { // Generic map - CompoundName fullName = new CompoundName(prefix + field.getName()); + CompoundName fullName = prefix.append(field.getName()); for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName, context).entrySet()) { try { - properties().set(fullName + "." + entry.getKey(), entry.getValue(), context); + properties().set(fullName.append(entry.getKey()), entry.getValue(), context); } catch (IllegalArgumentException e) { throw new QueryException("Invalid request parameter", e); } @@ -416,7 +417,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setFrom(prefix, originalProperties, ((QueryProfileFieldType)field.getType()).getQueryProfileType(), context); } else { - CompoundName fullName = new CompoundName(prefix + field.getName()); + CompoundName fullName = prefix.append(field.getName()); Object value = originalProperties.get(fullName, context); if (value != null) { try { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 2ec3df4a976..07c9e4475ec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -8,7 +8,11 @@ import com.yahoo.component.provider.FreezableSimpleComponent; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.QueryProfile; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static com.yahoo.text.Lowercase.toLowerCase; @@ -19,11 +23,12 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class QueryProfileType extends FreezableSimpleComponent { + private final CompoundName componentIdAsCompoundName; /** The fields of this query profile type */ - private Map<String, FieldDescription> fields = new HashMap<>(); + private Map<String, FieldDescription> fields; /** The query profile types this inherits */ - private List<QueryProfileType> inherited = new ArrayList<>(); + private List<QueryProfileType> inherited; /** If this is true, keys which are not declared in this type cannot be set in instances */ private boolean strict = false; @@ -41,15 +46,20 @@ public class QueryProfileType extends FreezableSimpleComponent { } public QueryProfileType(ComponentId id) { + this(id, new HashMap<>(), new ArrayList<>()); + } + + private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited) { super(id); QueryProfile.validateName(id.getName()); + componentIdAsCompoundName = new CompoundName(getId().getName()); + this.fields = fields; + this.inherited = inherited; } private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited, boolean strict, boolean matchAsPath, boolean builtin, Map<String,String> aliases) { - super(id); - this.fields = new HashMap<>(fields); - this.inherited = new ArrayList<>(inherited); + this(id, new HashMap<>(fields), new ArrayList<>(inherited)); this.strict = strict; this.matchAsPath = matchAsPath; this.builtin = builtin; @@ -84,6 +94,8 @@ public class QueryProfileType extends FreezableSimpleComponent { return new QueryProfileType(getId(), unfrozenFields, unfrozenInherited, strict, matchAsPath, builtin, aliases); } + public CompoundName getComponentIdAsCompoundName() { return componentIdAsCompoundName; } + /** Mark this type as built into the system. Do not use */ public void setBuiltin(boolean builtin) { this.builtin=builtin; } diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 53e77631ff9..7e5d6b12f30 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -6,29 +6,42 @@ import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Type; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.simple.JsonRender; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.JsonFormat; +import com.yahoo.tensor.serialization.TypedBinaryFormat; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; /** - * A wrapper for structured data representing feature values. + * A wrapper for structured data representing feature values: A map of floats and tensors. + * This class is not thread safe even when it is only consumed. */ public class FeatureData implements Inspectable, JsonProducer { private final Inspector value; + private Set<String> featureNames = null; + public FeatureData(Inspector value) { this.value = value; } + /** + * Returns the fields of this as an inspector, where tensors are represented as binary data + * which can be decoded using + * <code>com.yahoo.tensor.serialization.TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(featureValue.asData()))</code> + */ @Override - public Inspector inspect() { - return value; - } + public Inspector inspect() { return value; } + @Override public String toString() { - if (value.type() == Type.EMPTY) { - return ""; - } else { - return toJson(); - } + if (value.type() == Type.EMPTY) return ""; + return toJson(); } @Override @@ -38,7 +51,64 @@ public class FeatureData implements Inspectable, JsonProducer { @Override public StringBuilder writeJson(StringBuilder target) { - return JsonRender.render(value, target, true); + return JsonRender.render(value, new Encoder(target, true)); + } + + /** + * Returns the value of a scalar feature, or null if it is not present. + * + * @throws IllegalArgumentException if the value exists but isn't a scalar + * (that is, if it is a tensor with nonzero rank) + */ + public Double getDouble(String featureName) { + Inspector featureValue = value.field(featureName); + if ( ! featureValue.valid()) return null; + + switch (featureValue.type()) { + case DOUBLE: return featureValue.asDouble(); + case DATA: throw new IllegalArgumentException("Feature '" + featureName + "' is a tensor, not a double"); + default: throw new IllegalStateException("Unexpected feature value type " + featureValue.type()); + } + } + + /** + * Returns the value of a tensor feature, or null if it is not present. + * This will return any feature value: Scalars are returned as a rank 0 tensor. + */ + public Tensor getTensor(String featureName) { + Inspector featureValue = value.field(featureName); + if ( ! featureValue.valid()) return null; + + switch (featureValue.type()) { + case DOUBLE: return Tensor.from(featureValue.asDouble()); + case DATA: return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(featureValue.asData())); + default: throw new IllegalStateException("Unexpected feature value type " + featureValue.type()); + } + } + + /** Returns the names of the features available in this */ + public Set<String> featureNames() { + if (featureNames != null) return featureNames; + + featureNames = new HashSet<>(); + value.fields().forEach(field -> featureNames.add(field.getKey())); + return featureNames; + } + + /** A JSON encoder which encodes DATA as a tensor */ + private static class Encoder extends JsonRender.StringEncoder { + + Encoder(StringBuilder out, boolean compact) { + super(out, compact); + } + + @Override + public void encodeDATA(byte[] value) { + // This could be done more efficiently ... + target().append(new String(JsonFormat.encodeWithType(TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value))), + StandardCharsets.UTF_8)); + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java index 483849a5435..203e0206f1e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java +++ b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java @@ -10,7 +10,7 @@ import com.yahoo.data.access.simple.JsonRender; /** * A wrapper for structured data representing an array of position values. - **/ + */ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { private final Inspector value; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java index da457dba3b0..2a62cfec507 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types.test; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.yolean.Exceptions; import com.yahoo.search.query.profile.QueryProfile; @@ -52,6 +53,12 @@ public class NameTestCase { "Illegal name '-some-other'"); } + @Test + public void testComponentIdAsCompoundName() { + String name = "a/b"; + assertEquals(new CompoundName(name), new QueryProfileType(name).getComponentIdAsCompoundName()); + } + private void assertLegalName(String name) { new QueryProfile(name); new QueryProfileType(name); diff --git a/container-search/src/test/java/com/yahoo/search/result/FeatureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/result/FeatureDataTestCase.java new file mode 100644 index 00000000000..9cc7cc743fc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/FeatureDataTestCase.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import org.junit.Test; + +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class FeatureDataTestCase { + + private static final double delta = 0.00000001; + + @Test + public void testFeatureData() { + Cursor features = new Slime().setObject(); + features.setDouble("scalar1", 1.5); + features.setDouble("scalar2", 2.5); + Tensor tensor1 = Tensor.from("tensor(x[3]):[1.5, 2, 2.5]"); + features.setData("tensor1", TypedBinaryFormat.encode(tensor1)); + Tensor tensor2 = Tensor.from(0.5); + features.setData("tensor2", TypedBinaryFormat.encode(tensor2)); + + FeatureData featureData = new FeatureData(new SlimeAdapter(features)); + assertEquals("scalar1,scalar2,tensor1,tensor2", + featureData.featureNames().stream().sorted().collect(Collectors.joining(","))); + assertEquals(1.5, featureData.getDouble("scalar1"), delta); + assertEquals(2.5, featureData.getDouble("scalar2"), delta); + assertEquals(Tensor.from(1.5), featureData.getTensor("scalar1")); + assertEquals(Tensor.from(2.5), featureData.getTensor("scalar2")); + assertEquals(tensor1, featureData.getTensor("tensor1")); + assertEquals(tensor2, featureData.getTensor("tensor2")); + + String expectedJson = + "{" + + "\"scalar1\":1.5," + + "\"scalar2\":2.5," + + "\"tensor1\":{\"type\":\"tensor(x[3])\",\"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.5},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":2.5}]}," + + "\"tensor2\":{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":0.5}]}" + + "}"; + assertEquals(expectedJson, featureData.toJson()); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 94e111455ac..a1beef23dbb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -154,7 +154,7 @@ public enum JobType { case test: return Optional.of(systemTest); case staging: return Optional.of(stagingTest); } - return from(system, ZoneId.from(system, environment, region)); + return from(system, ZoneId.from(environment, region)); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index ec44b5605b6..6745b75a9ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -50,7 +50,9 @@ enum PathGroup { application(Matcher.tenant, Matcher.application, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/{application}"), + "/application/v4/tenant/{tenant}/application/{application}", + "/application/v4/tenant/{tenant}/application/{application}/instance/", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}"), /** Paths used for user management on the application level. */ applicationUsers(Matcher.tenant, @@ -63,7 +65,13 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended", @@ -74,6 +82,8 @@ enum PathGroup { developmentRestart(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/dev/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/perf/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{ignored}/restart"), @@ -81,6 +91,9 @@ enum PathGroup { productionRestart(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/staging/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/staging/region/{region}/instance/{ignored}/restart"), @@ -90,6 +103,10 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{job}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{instance}", @@ -99,6 +116,12 @@ enum PathGroup { productionDeployment(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/prod/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/prod/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/test/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/test/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/staging/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/staging/region/{region}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{instance}", @@ -110,13 +133,15 @@ enum PathGroup { submission(Matcher.tenant, Matcher.application, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/{application}/submit"), + "/application/v4/tenant/{tenant}/application/{application}/submit", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit"), /** Paths used for other tasks by build services. */ // TODO: This will vanish. buildService(Matcher.tenant, Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport", "/application/v4/tenant/{tenant}/application/{application}/promote", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 9edff876fb4..7b03664e0ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -231,8 +231,6 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(ApplicationId id, Optional<Credentials> credentials) { - if ( ! (id.instance().isDefault())) // TODO: Support instances properly - throw new IllegalArgumentException("Only the instance name 'default' is supported at the moment"); if (id.instance().isTester()) throw new IllegalArgumentException("'" + id + "' is a tester application!"); try (Lock lock = lock(id)) { @@ -251,7 +249,7 @@ public class ApplicationController { if (credentials.isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - if (id.instance().isDefault()) // Only store the application permits for non-user applications. + if ( ! id.instance().isTester()) // Only store the application permits for non-user applications. accessControl.createApplication(id, credentials.get()); } LockedApplication application = new LockedApplication(new Application(id, clock.instant()), lock); @@ -548,19 +546,19 @@ public class ApplicationController { .orElse(id.applicationId().instance().isTester())) throw new NotExistsException("Deployment", id.toString()); + // TODO jvenstad: Swap to use routingPolicies first, when this is ready. try { - return Optional.of(routingGenerator.clusterEndpoints(id)) - .filter(endpoints -> ! endpoints.isEmpty()) - .orElseGet(() -> routingPolicies.get(id).stream() - .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), - policy -> policy.endpointIn(controller.system()).url()))); + var endpoints = routingGenerator.clusterEndpoints(id); + if ( ! endpoints.isEmpty()) + return endpoints; } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " - + Exceptions.toMessageString(e)); - return Collections.emptyMap(); + log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " + Exceptions.toMessageString(e)); } + return routingPolicies.get(id).stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + policy -> policy.endpointIn(controller.system()).url())); } /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 7955505a2b0..7cf710623ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; /** * Fetch utilization metrics and update applications with this data. @@ -24,7 +26,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { private final Controller controller; public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); + super(controller, duration, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.controller = controller; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 7dfbc135df9..0080d7c23c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -11,9 +11,9 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.EnumSet; import java.util.Objects; import java.util.Optional; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,7 +28,7 @@ public class ContactInformationMaintainer extends Maintainer { private final ContactRetriever contactRetriever; public ContactInformationMaintainer(Controller controller, Duration interval, JobControl jobControl, ContactRetriever contactRetriever) { - super(controller, interval, jobControl, null, EnumSet.of(SystemName.cd, SystemName.main)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.contactRetriever = Objects.requireNonNull(contactRetriever, "organization must be non-null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 159eb234aa7..b8bb9a7ef79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -3,11 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.yolean.Exceptions; @@ -68,7 +67,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { boolean currentAppConverged = convergedOn(target, application, zone); - if (!currentAppConverged) { + // In dynamically provisioned zones there may be no tenant hosts at the time of upgrade, so we + // should always set the target version. + if (application == SystemApplication.tenantHost || !currentAppConverged) { upgrade(target, application, zone); } converged &= currentAppConverged; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index a0520f324be..9302ecbe738 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -18,6 +18,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.function.Predicate; import java.util.stream.Collectors; import static com.yahoo.yolean.Exceptions.uncheck; @@ -46,7 +47,7 @@ public class ResourceMeterMaintainer extends Maintainer { Clock clock, Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { - super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd, SystemName.main)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.clock = clock; this.nodeRepository = nodeRepository; this.metric = metric; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 5a40dd591fd..156e8d0d242 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -34,7 +34,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); - controller().applications().deploy(application, zone.toDeprecatedId(), target); + controller().applications().deploy(application, zone.getId(), target); } } @@ -45,7 +45,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { if (minVersion.isEmpty()) return true; return minVersion.get().equals(target) - && application.configConvergedIn(zone.toDeprecatedId(), controller(), Optional.of(target)); + && application.configConvergedIn(zone.getId(), controller(), Optional.of(target)); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 47c7b54264e..1f20bdf5533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -54,6 +54,13 @@ import java.util.TreeMap; */ public class ApplicationSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // Application fields private final String idField = "id"; private final String createdAtField = "createdAt"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 7fee9a7f9b4..d18e561ce5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -19,6 +19,13 @@ import java.util.function.Function; */ public class AuditLogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String entriesField = "entries"; private static final String atField = "at"; private static final String principalField = "principal"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java index a87875da104..2cb981aac03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java @@ -18,6 +18,13 @@ import java.util.Map; */ public class ConfidenceOverrideSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private final static String overridesField = "overrides"; public Slime toSlime(Map<Version, Confidence> overrides) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 40781ac6e92..418038d4f1e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -27,6 +27,13 @@ import java.util.stream.Collectors; */ class LogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String idField = "id"; private static final String typeField = "type"; private static final String timestampField = "at"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index 3dfb5ffe5f8..e3dedd65e68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -24,6 +24,13 @@ import java.util.ArrayList; */ public class NameServiceQueueSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String requestsField = "requests"; private static final String requestType = "requestType"; private static final String recordsField = "records"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index 21f8b1bcb80..d68e24a27ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -20,6 +20,13 @@ import java.util.TreeSet; */ public class OsVersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String cloudField = "cloud"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index 3e3c0df1673..88805f54d65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -26,6 +26,13 @@ import java.util.TreeMap; */ public class OsVersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String nodesField = "nodes"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index a9c6c54a44a..9cfce8dc16a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -24,6 +24,13 @@ import java.util.function.Function; */ public class RoutingPolicySerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String routingPoliciesField = "routingPolicies"; private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index f29af1055d0..1c95c9766f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -56,6 +56,13 @@ import static java.util.Comparator.comparing; */ class RunSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String stepsField = "steps"; private static final String applicationField = "id"; private static final String jobTypeField = "type"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 56e80068908..3a4e6c3954c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -29,6 +29,13 @@ import java.util.Optional; */ public class TenantSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String nameField = "name"; private static final String typeField = "type"; private static final String athenzDomainField = "athenzDomain"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java index 5edae803fdb..e5897963254 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java @@ -13,6 +13,13 @@ import com.yahoo.slime.Slime; */ public class VersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionField = "version"; public Slime toSlime(Version version) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 72d38bbee5f..405a2e452d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -27,6 +27,13 @@ import java.util.Set; */ public class VersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // VersionStatus fields private static final String versionsField = "versions"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 4402a6267fe..3db8c447572 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -176,16 +176,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return application(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -198,23 +210,33 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -223,21 +245,26 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePATCH(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) - return patchApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return patchApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return patchApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), "default"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), path.get("instance")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -302,25 +329,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse applications(String tenantName, HttpRequest request) { + private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); Slime slime = new Slime(); Cursor array = slime.setArray(); - for (Application application : controller.applications().asList(tenant)) + for (Application application : controller.applications().asList(tenant)) { + if (applicationName.isPresent() && ! application.id().application().toString().equals(applicationName.get())) + continue; toSlime(application, array.addObject(), request); + } return new SlimeJsonResponse(slime); } - private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse application(String tenantName, String applicationName, String instanceName, HttpRequest request) { Slime slime = new Slime(); - toSlime(slime.setObject(), getApplication(tenantName, applicationName), request); + toSlime(slime.setObject(), getApplication(tenantName, applicationName, instanceName), request); return new SlimeJsonResponse(slime); } - private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse patchApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes."); - controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, "default"), application -> { + controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, instanceName), application -> { Inspector majorVersionField = requestObject.field("majorVersion"); if (majorVersionField.valid()) { Integer majorVersion = majorVersionField.asLong() == 0 ? null : (int) majorVersionField.asLong(); @@ -340,8 +370,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse(messageBuilder.toString()); } - private Application getApplication(String tenantName, String applicationName) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); + private Application getApplication(String tenantName, String applicationName, String instanceName) { + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); return controller.applications().get(applicationId) .orElseThrow(() -> new NotExistsException(applicationId + " not found")); } @@ -532,7 +562,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deploymentObject.setString("url", withPath(request.getUri().getPath() + "/environment/" + deployment.zone().environment().value() + "/region/" + deployment.zone().region().value() + - "/instance/" + application.id().instance().value(), + ( request.getUri().getPath().contains("/instance/") ? "" : "/instance/" + application.id().instance().value()), request.getUri()).toString()); } } @@ -725,11 +755,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse deploying(String tenant, String application, HttpRequest request) { - Application app = controller.applications().require(ApplicationId.from(tenant, application, "default")); + private HttpResponse deploying(String tenant, String application, String instance, HttpRequest request) { + Application app = controller.applications().require(ApplicationId.from(tenant, application, instance)); Slime slime = new Slime(); Cursor root = slime.setObject(); - if (!app.change().isEmpty()) { + if ( ! app.change().isEmpty()) { app.change().platform().ifPresent(version -> root.setString("platform", version.toString())); app.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); root.setBool("pinned", app.change().isPinned()); @@ -804,9 +834,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } - private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse createApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); try { Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() @@ -826,10 +856,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ - private HttpResponse deployPlatform(String tenantName, String applicationName, boolean pin, HttpRequest request) { + private HttpResponse deployPlatform(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { request = controller.auditLogger().log(request); String versionString = readToString(request.getData()); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Version version = Version.fromString(versionString); @@ -837,12 +867,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { version = controller.systemVersion(); if ( ! systemHasVersion(version)) throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + - "Version is not active in this system. " + - "Active versions: " + controller.versionStatus().versions() - .stream() - .map(VespaVersion::versionNumber) - .map(Version::toString) - .collect(joining(", "))); + "Version is not active in this system. " + + "Active versions: " + controller.versionStatus().versions() + .stream() + .map(VespaVersion::versionNumber) + .map(Version::toString) + .collect(joining(", "))); Change change = Change.of(version); if (pin) change = change.withPin(); @@ -854,9 +884,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { controller.auditLogger().log(request); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = Change.of(application.get().deploymentJobs().statusOf(JobType.component).get().lastSuccess().get().application()); @@ -867,8 +897,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ - private HttpResponse cancelDeploy(String tenantName, String applicationName, String choice) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = application.get().change(); @@ -1041,8 +1071,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(tenant.get(), request); } - private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse deleteApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); @@ -1059,9 +1089,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // TODO: Change to return JSON return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, ApplicationResource.API_PATH, applicationName, + "instance", instanceName, EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + "region", region)); } /** @@ -1180,7 +1210,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } Cursor applicationArray = object.setArray("applications"); for (Application application : controller.applications().asList(tenant.name())) { - if (application.id().instance().isDefault()) {// TODO: Skip non-default applications until supported properly + if ( ! application.id().instance().isTester()) { if (recurseOverApplications(request)) toSlime(applicationArray.addObject(), application, request); else @@ -1269,8 +1299,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); - object.setString("url", withPath("/application/v4/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), request.getUri()).toString()); + object.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + application.id().instance().value(), + request.getUri()).toString()); } private Slime toSlime(ActivateResult result) { @@ -1423,7 +1456,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } - private HttpResponse submit(String tenant, String application, HttpRequest request) { + private HttpResponse submit(String tenant, String application, String instance, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions); @@ -1438,6 +1471,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, + instance, sourceRevision, authorEmail, projectId, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index ef7510ad18c..cd7c0d6236d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -416,10 +416,10 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, + static HttpResponse submitResponse(JobController jobController, String tenant, String application, String instance, SourceRevision sourceRevision, String authorEmail, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), + ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, instance), sourceRevision, authorEmail, projectId, @@ -444,8 +444,8 @@ class JobControllerApiHandlerHelper { } /** Unregisters the application from the internal deployment pipeline. */ - static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName, String instanceName) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); jobs.unregister(id); Slime slime = new Slime(); slime.setObject().setString("message", "Unregistered '" + id + "' from internal deployment pipeline."); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index a093aac430b..3ce32347e35 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -156,8 +156,12 @@ public class DeploymentTester { } public Application createApplication(String applicationName, String tenantName, long projectId, long propertyId) { + return createApplication("default", applicationName, tenantName, projectId, propertyId); + } + + public Application createApplication(String instanceName, String applicationName, String tenantName, long projectId, long propertyId) { TenantName tenant = tester.createTenant(tenantName, UUID.randomUUID().toString(), propertyId); - return tester.createApplication(tenant, applicationName, "default", projectId); + return tester.createApplication(tenant, applicationName, instanceName, projectId); } public void restartController() { tester.createNewController(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index bf7c8cadd3c..0d8d32299ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; @@ -109,6 +110,46 @@ public class DeploymentTriggerTest { } @Test + public void testIndependentInstances() { + Application instance1 = tester.createApplication("instance1", "app", "tenant", 1, 1L); + Application instance2 = tester.createApplication("instance2", "app", "tenant", 2, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + Version version = Version.fromString("6.2"); + tester.upgradeSystem(version); + + // Deploy completely once + tester.jobCompletion(component).application(instance1).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.productionUsWest1); + + tester.jobCompletion(component).application(instance2).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.productionUsWest1); + + // New version is released + Version newVersion = Version.fromString("6.3"); + tester.upgradeSystem(newVersion); + + // instance1 upgrades, but not instance 2 + tester.deployAndNotify(instance1, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.productionUsWest1); + + Version instance1Version = tester.application(instance1.id()).deployments().get(JobType.productionUsWest1.zone(main)).version(); + Version instance2Version = tester.application(instance2.id()).deployments().get(JobType.productionUsWest1.zone(main)).version(); + + assertEquals(newVersion, instance1Version); + assertEquals(version, instance2Version); + } + + @Test public void abortsInternalJobsOnNewApplicationChange() { InternalDeploymentTester iTester = new InternalDeploymentTester(); DeploymentTester tester = iTester.tester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index bc411d4377d..d4550ecc338 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -9,6 +9,8 @@ import org.junit.Test; import java.io.IOException; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -28,7 +30,7 @@ public class TestConfigSerializerTest { Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), URI.create("https://server/"))), Map.of(zone, List.of("facts"))); - byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes(); + byte[] expected = Files.readAllBytes(Paths.get("src/test/resources/testConfig.json")); assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), new String(json)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 700e6e9cb42..57f29fb72af 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -83,7 +83,7 @@ public class ZoneFilterMock implements ZoneList { @Override public List<ZoneId> ids() { - return List.copyOf(zones.stream().map(ZoneApi::toDeprecatedId).collect(Collectors.toList())); + return List.copyOf(zones.stream().map(ZoneApi::getId).collect(Collectors.toList())); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 39ff29f4ae0..38aa4af4756 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -57,16 +57,16 @@ public class OsUpgraderTest { ); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.tenantHost)); // Add system applications that exist in a real system, but are currently not upgraded - tester.configServer().addNodes(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), + tester.configServer().addNodes(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.configServer)); // Fail a few nodes. Failed nodes should not affect versions - failNodeIn(zone1.toDeprecatedId(), SystemApplication.tenantHost); - failNodeIn(zone3.toDeprecatedId(), SystemApplication.tenantHost); + failNodeIn(zone1.getId(), SystemApplication.tenantHost); + failNodeIn(zone3.getId(), SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -78,37 +78,37 @@ public class OsUpgraderTest { // zone 1: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone1.getId()); // Other zones remain on previous version (none) - assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); + assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2.getId(), zone3.getId(), zone4.getId()); // zone 1: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone1.getId()); statusUpdater.maintain(); assertEquals(2, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId()); + assertWanted(version1, SystemApplication.proxy, zone2.getId(), zone3.getId()); // zone 4: still on previous version - assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4.toDeprecatedId()); + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4.getId()); // zone 2 and 3: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone2.toDeprecatedId(), zone3.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone2.getId(), zone3.getId()); // zone 4: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone4.getId()); // zone 4: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone4.getId()); // Next run does nothing as all zones are upgraded osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index cb5e7cc90a1..7b817c175b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -50,7 +50,7 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.configServer, SystemApplication.proxy); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); @@ -144,7 +144,7 @@ public class SystemUpgraderTest { SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer, SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -184,7 +184,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), SystemApplication.all()); + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -282,7 +282,7 @@ public class SystemUpgraderTest { public void does_not_deploy_proxy_app_in_zones_without_proxy() { List<SystemApplication> applications = List.of( SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), applications); + tester.configServer().bootstrap(List.of(zone1.getId()), applications); tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); @@ -309,7 +309,7 @@ public class SystemUpgraderTest { private void convergeServices(SystemApplication application, ZoneApi... zones) { for (ZoneApi zone : zones) { - tester.controllerTester().configServer().convergeServices(application.id(), zone.toDeprecatedId()); + tester.controllerTester().configServer().convergeServices(application.id(), zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index c851cb18e8c..76c505ff8f8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -72,11 +72,10 @@ public class ContainerControllerTester { public ContainerTester containerTester() { return containerTester; } public Application createApplication() { - return createApplication("domain1","tenant1", - "application1"); + return createApplication("domain1","tenant1", "application1", "default"); } - public Application createApplication(String athensDomain, String tenant, String application) { + public Application createApplication(String athensDomain, String tenant, String application, String instance) { AthenzDomain domain1 = addTenantAthenzDomain(athensDomain, "user"); AthenzPrincipal user = new AthenzPrincipal(new AthenzUser("user")); AthenzCredentials credentials = new AthenzCredentials(user, domain1, new OktaAccessToken("okta-token")); @@ -86,7 +85,7 @@ public class ContainerControllerTester { Optional.of(new PropertyId("1234"))); controller().tenants().create(tenantSpec, credentials); - ApplicationId app = ApplicationId.from(tenant, application, "default"); + ApplicationId app = ApplicationId.from(tenant, application, instance); return controller().applications().createApplication(app, Optional.of(credentials)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 99ea9cc02d0..71c4b41a276 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -193,7 +193,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("tenant-with-contact-info.json")); // POST (create) an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); @@ -204,12 +204,15 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), new File("application-list.json")); + // GET tenant applications (instances of "application1" only) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), + new File("application-list.json")); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) .userIdentity(USER_ID), @@ -217,7 +220,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. - ApplicationId id = ApplicationId.from("tenant1", "application1", "default"); + ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1"); long screwdriverProjectId = 123; addScrewdriverUserToDeployRole(SCREWDRIVER_ID, @@ -232,13 +235,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // ... systemtest - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/test/region/us-east-1"); controllerTester.jobCompletion(JobType.systemTest) .application(id) @@ -246,20 +249,20 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // ... staging - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/staging/region/us-east-3"); controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) .submit(); // ... prod zone - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -274,10 +277,10 @@ public class ApplicationApiTest extends ControllerContainerTest { Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber - 1)), true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application package found for tenant1.application1 with version 1.0.41-commit1\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application package found for tenant1.application1.instance1 with version 1.0.41-commit1\"}", 400); // POST an application deployment to a production zone - operator emergency deployment - works with known package @@ -285,7 +288,7 @@ public class ApplicationApiTest extends ControllerContainerTest { Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber)), true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), new File("deploy-result.json")); @@ -368,15 +371,15 @@ public class ApplicationApiTest extends ControllerContainerTest { setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID), new File("application.json")); // GET an application deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", GET) .userIdentity(USER_ID), new File("deployment.json")); - addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "default")); + addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "instance1")); // GET at root, with "&recursive=deployment", returns info about all tenants, their applications and their deployments tester.assertResponse(request("/application/v4/", GET) .userIdentity(USER_ID) @@ -393,13 +396,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .recursive("true"), new File("tenant1-recursive.json")); // GET at an application, with "&recursive=true", returns full info about its deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID) .recursive("true"), new File("application1-recursive.json")); // GET nodes - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/nodes", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/nodes", GET) .userIdentity(USER_ID), new File("application-nodes.json")); @@ -409,148 +412,143 @@ public class ApplicationApiTest extends ControllerContainerTest { "INFO - All good"); // DELETE (cancel) ongoing change - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'\"}"); + "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1.instance1'\"}"); // DELETE (cancel) again is a no-op - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) .userIdentity(USER_ID) .data("{\"cancel\":\"all\"}"), - "{\"message\":\"No deployment in progress for application 'tenant1.application1' at this time\"}"); + "{\"message\":\"No deployment in progress for application 'tenant1.application1.instance1' at this time\"}"); // POST pinning to a given version to an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1.0"), - "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue("Action is logged to audit log", tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/deploying/pin"))); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin"))); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false}"); // POST pinning again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1"), - "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the version, but leave the pin - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/platform", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"pinned\":true}"); // DELETE also the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{}"); // POST a pause to a production job - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", POST) .userIdentity(USER_ID), - "{\"message\":\"production-us-west-1 for tenant1.application1 paused for " + DeploymentTrigger.maxPause + "\"}"); + "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 paused for " + DeploymentTrigger.maxPause + "\"}"); // POST a triggering to the same production job - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", POST) .userIdentity(USER_ID), - "{\"message\":\"Triggered production-us-west-1 for tenant1.application1\"}"); + "{\"message\":\"Triggered production-us-west-1 for tenant1.application1.instance1\"}"); // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging environment command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging test command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging dev command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart?hostname=host1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=host1", POST) .screwdriverIdentity(SCREWDRIVER_ID), "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"No node with the hostname host1 is known.\"}", 500); // GET suspended - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/suspended", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/suspended", GET) .userIdentity(USER_ID), new File("suspended.json")); // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service", GET) .userIdentity(USER_ID), new File("services.json")); // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) .userIdentity(USER_ID), new File("service.json")); // DELETE application with active deployments fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("delete-with-active-deployments.json"), 400); - // GET config for running a test against a deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/dev-us-east-1/test-config", GET) - .userIdentity(USER_ID), - new File("test-config.json")); - // DELETE (deactivate) a deployment - dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1"); // DELETE (deactivate) a deployment - prod - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); // DELETE (deactivate) a deployment is idempotent - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); // POST an application package to start a deployment to dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/deploy/dev-us-east-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1", POST) .userIdentity(USER_ID) .data(createApplicationDeployData(applicationPackage, false)), new File("deployment-job-accepted.json")); // POST an application package and a test jar, submitting a new application for internal pipeline deployment. // First attempt does not have an Athenz service definition in deployment spec, and is accepted. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackage)), "{\"message\":\"Application package version: 1.0.43-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); @@ -561,7 +559,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithServiceForWrongDomain)), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz domain in deployment.xml: [domain2] must match tenant domain: [domain1]\"}", 400); @@ -572,13 +570,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN.getName()), AthenzService.from("service")) .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithService)), "{\"message\":\"Application package version: 1.0.44-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); // Fourth attempt has a wrong content hash in a header, and fails. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .header("X-Content-Hash", "not/the/right/hash") .data(createApplicationSubmissionData(packageWithService)), @@ -586,14 +584,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // Fifth attempt has the right content hash in a header, and succeeds. MultiPartStreamer streamer = createApplicationSubmissionData(packageWithService); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) .data(streamer), "{\"message\":\"Application package version: 1.0.45-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) + ApplicationId app1 = ApplicationId.from("tenant1", "application1", "instance1"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/jobreport", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(asJson(DeploymentJobs.JobReport.ofComponent(app1, 1234, @@ -607,31 +605,31 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET deployment job overview, after triggering system and staging test jobs. assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job", GET) .userIdentity(USER_ID), new File("jobs.json")); // GET system test job overview. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET) .userIdentity(USER_ID), new File("system-test-job.json")); // GET system test run 1 details. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", GET) .userIdentity(USER_ID), new File("system-test-details.json")); // DELETE a running job to have it aborted. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Aborting run 1 of staging-test for tenant1.application1\"}"); + "{\"message\":\"Aborting run 1 of staging-test for tenant1.application1.instance1\"}"); // DELETE submission to unsubscribe from continuous deployment. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Unregistered 'tenant1.application1' from internal deployment pipeline.\"}"); + "{\"message\":\"Unregistered 'tenant1.application1.instance1' from internal deployment pipeline.\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/jobreport", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(asJson(DeploymentJobs.JobReport.ofComponent(app1, 1234, @@ -657,16 +655,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), ""); - // Promote from pipeline - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); - // DELETE an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE).userIdentity(USER_ID) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), ""); // DELETE a tenant @@ -700,7 +690,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -717,37 +707,37 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // Invalid deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", 404); // Change status of non-existing deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", 404); // GET global rotation status setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), new File("global-rotation.json")); // GET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", GET) .userIdentity(USER_ID), new File("global-rotation-get.json")); // SET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-put.json")); // DELETE global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-delete.json")); @@ -767,7 +757,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); @@ -779,7 +769,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a prod zone - allowed when project ID is not specified MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(entity) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -798,7 +788,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); // POST (deploy) a system application without an application package - tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), new File("deploy-no-deployment.json"), 400); @@ -818,7 +808,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -837,7 +827,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 101); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -849,7 +839,7 @@ public class ApplicationApiTest extends ControllerContainerTest { setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -859,7 +849,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); setDeploymentMaintainedInfo(controllerTester); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID), new File("application-without-change-multiple-deployments.json")); } @@ -943,16 +933,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // POST (create) an (empty) application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); // Create the same application again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1': Application already exists\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Application already exists\"}", 400); ConfigServerMock configServer = (ConfigServerMock) container.components().getComponent(ConfigServerMock.class.getName()); @@ -960,28 +950,28 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application with an invalid application package MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-failure.json"), 400); // POST (deploy) an application without available capacity configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-out-of-capacity.json"), 400); // POST (deploy) an application where activation fails configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-activation-conflict.json"), 409); // POST (deploy) an application where we get an internal server error configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-internal-server-error.json"), 500); @@ -994,15 +984,15 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // DELETE application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), ""); // DELETE application again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1.instance1': Application not found\"}", 404); // DELETE tenant @@ -1016,12 +1006,6 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); - // Promote application chef env for nonexistent tenant/application - tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", - 403); - // Create legancy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, new Property("property1"), Optional.empty(), Optional.empty())); @@ -1071,14 +1055,14 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Creating an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(unauthorizedUser) .oktaAccessToken(OKTA_AT), "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT), new File("application-reference.json"), @@ -1139,7 +1123,7 @@ public class ApplicationApiTest extends ControllerContainerTest { long screwdriverProjectId = 123; createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); ScrewdriverId screwdriverId = new ScrewdriverId(Long.toString(screwdriverProjectId)); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); @@ -1170,7 +1154,7 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); // Allow systemtest to succeed by notifying completion of system test @@ -1268,7 +1252,7 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); // Allow systemtest to succeed by notifying completion of system test @@ -1508,14 +1492,14 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); addScrewdriverUserToDeployRole(SCREWDRIVER_ID, ATHENZ_TENANT_DOMAIN, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); - return ApplicationId.from("tenant1", "application1", "default"); + return ApplicationId.from("tenant1", "application1", "instance1"); } private void startAndTestChange(ContainerControllerTester controllerTester, ApplicationId application, @@ -1533,8 +1517,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // system-test - String testPath = String.format("/application/v4/tenant/%s/application/%s/environment/test/region/us-east-1/instance/default", - application.tenant().value(), application.application().value()); + String testPath = String.format("/application/v4/tenant/%s/application/%s/instance/%s/environment/test/region/us-east-1", + application.tenant().value(), application.application().value(), application.instance().value()); tester.assertResponse(request(testPath, POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), @@ -1548,8 +1532,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // staging - String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default", - application.tenant().value(), application.application().value()); + String stagingPath = String.format("/application/v4/tenant/%s/application/%s/instance/%s/environment/staging/region/us-east-3", + application.tenant().value(), application.application().value(), application.instance().value()); tester.assertResponse(request(stagingPath, POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), @@ -1578,7 +1562,9 @@ public class ApplicationApiTest extends ControllerContainerTest { List<String> hostnames = new ArrayList<>(); hostnames.add("host1"); hostnames.add("host2"); - clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); + clusterInfo.put(ClusterSpec.Id.from("cluster1"), + new ClusterInfo("flavor1", 37, 2, 4, 50, + ClusterSpec.Type.content, hostnames)); Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5, @@ -1626,7 +1612,10 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void updateContactInformation() { - Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), URI.create("www.issues.tld/1234"), List.of(List.of("alice"), List.of("bob")), "queue", Optional.empty()); + Contact contact = new Contact(URI.create("www.contacts.tld/1234"), + URI.create("www.properties.tld/1234"), + URI.create("www.issues.tld/1234"), + List.of(List.of("alice"), List.of("bob")), "queue", Optional.empty()); tester.controller().tenants().lockIfPresent(TenantName.from("tenant2"), LockedTenant.Athenz.class, lockedTenant -> tester.controller().tenants().store(lockedTenant.with(contact))); @@ -1634,7 +1623,10 @@ public class ApplicationApiTest extends ControllerContainerTest { private void registerContact(long propertyId) { PropertyId p = new PropertyId(String.valueOf(propertyId)); - contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), URI.create("www.contacts.tld/" + p.id()), URI.create("www.properties.tld/" + p.id()), List.of(Collections.singletonList("alice"), + contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), + URI.create("www.contacts.tld/" + p.id()), + URI.create("www.properties.tld/" + p.id()), + List.of(Collections.singletonList("alice"), Collections.singletonList("bob")), "queue", Optional.empty())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index eb53ff7161e..e673a610f87 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -1,7 +1,7 @@ { "nodes": [ { - "hostname": "host-tenant1:application1:default-prod.us-central-1", + "hostname": "host-tenant1:application1:instance1-prod.us-central-1", "state": "active", "orchestration": "unorchestrated", "version": "6.1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json index ff22b95739d..0c9dc2ca1e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json @@ -2,5 +2,5 @@ "tenant": "tenant2", "application": "application2", "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2" + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2/instance/default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json index 1d56944f6bc..60243633614 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json @@ -1,6 +1,6 @@ { "tenant": "tenant1", "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index b52fec761d8..383a1b667f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -241,8 +241,8 @@ }, "environment": "prod", "region": "us-west-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" }, { "bcpStatus": { @@ -250,8 +250,8 @@ }, "environment": "prod", "region": "us-east-3", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" } ], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index ce29165d497..5d1819bf0f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -225,9 +225,9 @@ "instances": [ { "environment": "dev", - "region": "us-east-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default" + "region": "us-west-1", + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1" }, { "bcpStatus": { @@ -235,8 +235,8 @@ }, "environment": "prod", "region": "us-central-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1" } ], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index de99019447a..c0e9d10a40c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json index 25fed881dec..ad7e4f00027 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: dev.us-east-1, prod.us-central-1" + "message": "Could not delete 'application 'tenant1.application1.instance1'': It has active deployments in: dev.us-west-1, prod.us-central-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json index ef0112f5ff8..65ecb3c6979 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json @@ -1,4 +1,4 @@ { - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1", + "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1", "run": 1 }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 6caac3bd532..25948e998f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -1,7 +1,7 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "prod", "region": "us-central-1", "endpoints": [], @@ -12,8 +12,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 7cbbe4be43b..1a2025e4de2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -1,9 +1,9 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "dev", - "region": "us-east-1", + "region": "us-west-1", "endpoints": [], "serviceUrls": [ "http://old-endpoint.vespa.yahooapis.com:4080", @@ -12,8 +12,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json index 048ddcbc5c5..2df97a6c765 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json @@ -1 +1 @@ -{"message":"Successfully set tenant1.application1 in prod.us-west-1 in service"} +{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 in service"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json index f67b8dd56d9..6a41b0000e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json @@ -1 +1 @@ -{"message":"Successfully set tenant1.application1 in prod.us-west-1 out of service"} +{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 out of service"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index a5e4dd23079..c5cfd5981c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -65,10 +65,10 @@ "tasks": { "deploy": "running" }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test" }, "staging-test": { "runs": [ @@ -101,10 +101,10 @@ "report": "unfinished" }, "tasks": {}, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1" } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test" }, "us-west-1": { "runs": [ @@ -126,7 +126,7 @@ } } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1" } }, "devJobs": {} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 9b08fccf883..4810c8f92b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -4,7 +4,7 @@ }, "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "prod", "region": "us-central-1", "endpoints": [], @@ -15,8 +15,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json index 782d8e42aa3..1a434afafbb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json @@ -3,10 +3,10 @@ { "name": "cluster1", "type": "content", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", "services": [ { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "serviceType": "storagenode", "serviceName": "storagenode", "configId": "cluster1/storage/0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 107d969e8ad..800c3976188 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -28,6 +28,6 @@ "tasks": { "deploy": "running" }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index b222c33291c..edd3d7cc34f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -7,8 +7,8 @@ { "tenant": "tenant1", "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 6eb7663e4ab..35ec5b0e37e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -20,7 +20,7 @@ public class BadgeApiTest extends ControllerContainerTest { @Test public void testBadgeApi() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); - Application application = tester.createApplication("domain", "tenant", "application"); + Application application = tester.createApplication("domain", "tenant", "application", "default"); ApplicationPackage packageWithService = new ApplicationPackageBuilder() .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 73977d7c2fa..fa3848a6ba5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -39,12 +39,9 @@ public class DeploymentApiTest extends ControllerContainerTest { .build(); // 3 applications deploy on current system version - Application failingApplication = tester.createApplication("domain1", "tenant1", - "application1"); - Application productionApplication = tester.createApplication("domain2", "tenant2", - "application2"); - Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", - "application3"); + Application failingApplication = tester.createApplication("domain1", "tenant1", "application1", "default"); + Application productionApplication = tester.createApplication("domain2", "tenant2", "application2", "default"); + Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", "application3", "default"); tester.deployCompletely(failingApplication, applicationPackage, 1L, false); tester.deployCompletely(productionApplication, applicationPackage, 2L, false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index b2dfd7b4cb6..745a7af203b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -81,12 +81,12 @@ public class OsApiTest extends ControllerContainerTest { // Status is updated after some zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone1.toDeprecatedId()); + completeUpgrade(zone1.getId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-partially-upgraded.json"); // All zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone2.toDeprecatedId(), zone3.toDeprecatedId()); + completeUpgrade(zone2.getId(), zone3.getId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json"); // Downgrade with force is permitted diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json index 2a779f0ee55..6cf4dc76173 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json @@ -2,5 +2,5 @@ "tenant": "my-tenant", "application": "my-app", "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" + "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" } diff --git a/dist/vespa.spec b/dist/vespa.spec index f7360419ce6..99b24c2ff8e 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -42,17 +42,6 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: cmake >= 3.9.1 BuildRequires: maven BuildRequires: vespa-protobuf-devel >= 3.7.0-4 -%if 0%{?fc27} -BuildRequires: llvm-devel >= 5.0.2 -BuildRequires: boost-devel >= 1.64 -BuildRequires: vespa-gtest >= 1.8.1-1 -%endif -%if 0%{?fc28} -BuildRequires: llvm-devel >= 6.0.1 -BuildRequires: boost-devel >= 1.66 -BuildRequires: gtest-devel -BuildRequires: gmock-devel -%endif %if 0%{?fc29} BuildRequires: llvm-devel >= 7.0.0 BuildRequires: boost-devel >= 1.66 @@ -125,14 +114,6 @@ Requires: vespa-protobuf >= 3.7.0-4 %endif %if 0%{?fedora} Requires: vespa-protobuf >= 3.7.0-4 -%if 0%{?fc27} -Requires: llvm-libs >= 5.0.2 -%define _vespa_llvm_version 5.0 -%endif -%if 0%{?fc28} -Requires: llvm-libs >= 6.0.1 -%define _vespa_llvm_version 6.0 -%endif %if 0%{?fc29} Requires: llvm-libs >= 7.0.0 %define _vespa_llvm_version 7 diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 8cdb0bee7c2..43313392cdb 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -26,9 +26,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; -import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Counter; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import java.io.ByteArrayOutputStream; import java.time.Duration; @@ -56,19 +55,18 @@ public class DockerImpl implements Docker { private final DockerClient dockerClient; private final DockerImageGarbageCollector dockerImageGC; - private final CounterWrapper numberOfDockerDaemonFails; + private final Counter numberOfDockerApiFails; @Inject - public DockerImpl(MetricReceiverWrapper metricReceiverWrapper) { - this(createDockerClient(), metricReceiverWrapper); + public DockerImpl(Metrics metrics) { + this(createDockerClient(), metrics); } - DockerImpl(DockerClient dockerClient, MetricReceiverWrapper metricReceiver) { + DockerImpl(DockerClient dockerClient, Metrics metrics) { this.dockerClient = dockerClient; this.dockerImageGC = new DockerImageGarbageCollector(this); - Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); - numberOfDockerDaemonFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "daemon.api_fails"); + numberOfDockerApiFails = metrics.declareCounter("docker.api_fails"); } @Override @@ -86,7 +84,7 @@ public class DockerImpl implements Docker { return true; } } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to pull image '" + image.asString() + "'", e); } } @@ -110,7 +108,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { return Optional.empty(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to inspect image '" + dockerImage.asString() + "'", e); } } @@ -146,7 +144,7 @@ public class DockerImpl implements Docker { return new ProcessResult(state.getExitCode(), new String(output.toByteArray()), new String(errors.toByteArray())); } catch (RuntimeException | InterruptedException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Container '" + containerName.asString() + "' failed to execute " + Arrays.toString(command), e); } @@ -171,7 +169,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to get info for container '" + container + "'", e); } } @@ -186,7 +184,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException | InterruptedException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to get stats for container '" + containerName.asString() + "'", e); } } @@ -200,7 +198,7 @@ public class DockerImpl implements Docker { } catch (NotModifiedException ignored) { // If is already started, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to start container '" + containerName.asString() + "'", e); } } @@ -214,7 +212,7 @@ public class DockerImpl implements Docker { } catch (NotModifiedException ignored) { // If is already stopped, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to stop container '" + containerName.asString() + "'", e); } } @@ -226,7 +224,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { throw new ContainerNotFoundException(containerName); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to delete container '" + containerName.asString() + "'", e); } } @@ -253,7 +251,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { throw new ContainerNotFoundException(containerName); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to update container '" + containerName.asString() + "' to " + resources, e); } } @@ -307,7 +305,7 @@ public class DockerImpl implements Docker { try { return dockerClient.listContainersCmd().withShowAll(true).exec(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to list all containers", e); } } @@ -316,7 +314,7 @@ public class DockerImpl implements Docker { try { return dockerClient.listImagesCmd().withShowAll(true).exec(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to list all images", e); } } @@ -327,7 +325,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { // Image was already deleted, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to delete docker image " + dockerImage.asString(), e); } } @@ -357,7 +355,7 @@ public class DockerImpl implements Docker { logger.log(LogLevel.INFO, "Download completed: " + dockerImage.asString()); removeScheduledPoll(dockerImage); } else { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerClientException("Could not download image: " + dockerImage); } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java new file mode 100644 index 00000000000..3a0b820c846 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +/** + * @author freva + */ +public class Counter implements MetricValue { + private final Object lock = new Object(); + + private long value = 0; + + public void increment() { + add(1L); + } + + public void add(long n) { + synchronized (lock) { + value += n; + } + } + + @Override + public Number getValue() { + synchronized (lock) { + return value; + } + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java deleted file mode 100644 index 55c42271674..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.Counter; - -/** - * Forwards sample to {@link com.yahoo.metrics.simple.Counter} to be displayed in /state/v1/metrics, - * while also saving the value so it can be accessed programatically later. - * - * @author valerijf - */ -public class CounterWrapper implements MetricValue { - private final Object lock = new Object(); - - private final Counter counter; - private long value = 0; - - CounterWrapper(Counter counter) { - this.counter = counter; - } - - public void add() { - add(1L); - } - - public void add(long n) { - synchronized (lock) { - counter.add(n); - value += n; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java index 770ff5e2216..ef59c4b17d6 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java @@ -4,47 +4,67 @@ package com.yahoo.vespa.hosted.dockerapi.metrics; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; /** * @author freva */ public class DimensionMetrics { - private final static ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Map<String, Object> routing = Map.of("yamas", Map.of("namespaces", List.of("Vespa"))); private final String application; private final Dimensions dimensions; private final Map<String, Number> metrics; DimensionMetrics(String application, Dimensions dimensions, Map<String, Number> metrics) { - this.application = application; - this.dimensions = dimensions; - this.metrics = metrics; + this.application = Objects.requireNonNull(application); + this.dimensions = Objects.requireNonNull(dimensions); + this.metrics = Objects.requireNonNull(metrics); } - Map<String, Object> getMetrics() { - final Map<String, Object> routing = new HashMap<>(); - final Map<String, Object> routingMonitoring = new HashMap<>(); - routing.put("yamas", routingMonitoring); - routingMonitoring.put("namespaces", Collections.singletonList("Vespa")); - - Map<String, Object> report = new HashMap<>(); + public String toSecretAgentReport() throws JsonProcessingException { + Map<String, Object> report = new TreeMap<>(); report.put("application", application); - report.put("dimensions", dimensions.dimensionsMap); - report.put("metrics", metrics); + report.put("dimensions", new TreeMap<>(dimensions.asMap())); + report.put("metrics", new TreeMap<>(metrics)); report.put("routing", routing); - return report; - } - - public String toSecretAgentReport() throws JsonProcessingException { - Map<String, Object> report = getMetrics(); report.put("timestamp", System.currentTimeMillis() / 1000); return objectMapper.writeValueAsString(report); } + public String getApplication() { + return application; + } + + public Dimensions getDimensions() { + return dimensions; + } + + public Map<String, Number> getMetrics() { + return metrics; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DimensionMetrics that = (DimensionMetrics) o; + return application.equals(that.application) && + dimensions.equals(that.dimensions) && + metrics.equals(that.metrics); + } + + @Override + public int hashCode() { + return Objects.hash(application, dimensions, metrics); + } + public static class Builder { private final String application; private final Dimensions dimensions; diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java index 586622100fb..63b92e06505 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java @@ -1,20 +1,24 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi.metrics; -import java.util.Collections; import java.util.HashMap; import java.util.Map; /** - * Each metric reported to secret agent has dimensions. - * - * @author valerijf + * @author freva */ public class Dimensions { - final Map<String, Object> dimensionsMap; - private Dimensions(Map<String, Object> dimensionsMap) { - this.dimensionsMap = dimensionsMap; + public static final Dimensions NONE = new Dimensions(Map.of()); + + private final Map<String, String> dimensionsMap; + + public Dimensions(Map<String, String> dimensionsMap) { + this.dimensionsMap = Map.copyOf(dimensionsMap); + } + + public Map<String, String> asMap() { + return dimensionsMap; } @Override @@ -45,7 +49,7 @@ public class Dimensions { } public Dimensions build() { - return new Dimensions(Collections.unmodifiableMap(new HashMap<>(dimensionsMap))); + return new Dimensions(dimensionsMap); } } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java new file mode 100644 index 00000000000..b413475fc2b --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java @@ -0,0 +1,24 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +/** + * @author freva + */ +public class Gauge implements MetricValue { + private final Object lock = new Object(); + + private double value; + + public void sample(double x) { + synchronized (lock) { + this.value = x; + } + } + + @Override + public Number getValue() { + synchronized (lock) { + return value; + } + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java deleted file mode 100644 index 02e1f15a94f..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.Gauge; - -/** - * Forwards sample to {@link com.yahoo.metrics.simple.Gauge} to be displayed in /state/v1/metrics, - * while also saving the value so it can be accessed programatically later. - * - * @author valerijf - */ -public class GaugeWrapper implements MetricValue { - private final Object lock = new Object(); - - private final Gauge gauge; - private double value; - - GaugeWrapper(Gauge gauge) { - this.gauge = gauge; - } - - public void sample(double x) { - synchronized (lock) { - gauge.sample(x); - this.value = x; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java deleted file mode 100644 index 58126a59cbb..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.google.inject.Inject; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.metrics.simple.Point; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Export metrics to both /state/v1/metrics and makes them available programmatically. - * Each metric belongs to a monitoring application - * - * @author freva - */ -public class MetricReceiverWrapper { - // Application names used - public static final String APPLICATION_DOCKER = "docker"; - public static final String APPLICATION_HOST = "vespa.host"; - public static final String APPLICATION_NODE = "vespa.node"; - - private final Object monitor = new Object(); - private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>(); - private final MetricReceiver metricReceiver; - - @Inject - public MetricReceiverWrapper(MetricReceiver metricReceiver) { - this.metricReceiver = metricReceiver; - } - - /** - * Declaring the same dimensions and name results in the same CounterWrapper instance (idempotent). - */ - public CounterWrapper declareCounter(String application, Dimensions dimensions, String name) { - return declareCounter(application, dimensions, name, DimensionType.DEFAULT); - } - - public CounterWrapper declareCounter(String application, Dimensions dimensions, String name, DimensionType type) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, type); - if (!metricsByDimensions.containsKey(dimensions)) metricsByDimensions.put(dimensions, new HashMap<>()); - if (!metricsByDimensions.get(dimensions).containsKey(name)) { - CounterWrapper counter = new CounterWrapper(metricReceiver.declareCounter(name, new Point(dimensions.dimensionsMap))); - metricsByDimensions.get(dimensions).put(name, counter); - } - - return (CounterWrapper) metricsByDimensions.get(dimensions).get(name); - } - } - - /** - * Declaring the same dimensions and name results in the same GaugeWrapper instance (idempotent). - */ - public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name) { - return declareGauge(application, dimensions, name, DimensionType.DEFAULT); - } - - public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name, DimensionType type) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, type); - if (!metricsByDimensions.containsKey(dimensions)) - metricsByDimensions.put(dimensions, new HashMap<>()); - if (!metricsByDimensions.get(dimensions).containsKey(name)) { - GaugeWrapper gauge = new GaugeWrapper(metricReceiver.declareGauge(name, new Point(dimensions.dimensionsMap))); - metricsByDimensions.get(dimensions).put(name, gauge); - } - - return (GaugeWrapper) metricsByDimensions.get(dimensions).get(name); - } - } - - public List<DimensionMetrics> getDefaultMetrics() { - return getMetricsByType(DimensionType.DEFAULT); - } - - // For testing, returns same as getDefaultMetrics(), but without "timestamp" - public Set<Map<String, Object>> getDefaultMetricsRaw() { - synchronized (monitor) { - Set<Map<String, Object>> dimensionMetrics = new HashSet<>(); - metrics.getOrDefault(DimensionType.DEFAULT, new HashMap<>()) - .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() - .map(entry -> new DimensionMetrics(application, entry.getKey(), - entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) - .map(DimensionMetrics::getMetrics) - .forEach(dimensionMetrics::add)); - return dimensionMetrics; - } - } - - public List<DimensionMetrics> getMetricsByType(DimensionType type) { - synchronized (monitor) { - List<DimensionMetrics> dimensionMetrics = new ArrayList<>(); - metrics.getOrDefault(type, new HashMap<>()) - .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() - .map(entry -> new DimensionMetrics(application, entry.getKey(), - entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) - .forEach(dimensionMetrics::add)); - return dimensionMetrics; - } - } - - public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) { - synchronized (monitor) { - Optional.ofNullable(metrics.get(type)) - .map(m -> m.get(name)) - .map(ApplicationMetrics::metricsByDimensions) - .ifPresent(m -> m.remove(dimensionsToRemove)); - } - } - - // For testing - Map<String, Number> getMetricsForDimension(String application, Dimensions dimensions) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, DimensionType.DEFAULT); - return metricsByDimensions.getOrDefault(dimensions, Collections.emptyMap()) - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); - } - } - - private Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { - Map<String, ApplicationMetrics> applicationMetrics = metrics.computeIfAbsent(type, m -> new HashMap<>()); - if (! applicationMetrics.containsKey(application)) { - ApplicationMetrics metrics = new ApplicationMetrics(); - applicationMetrics.put(application, metrics); - } - return applicationMetrics.get(application).metricsByDimensions(); - } - - // "Application" is the monitoring application, not Vespa application - private static class ApplicationMetrics { - private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>(); - - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() { - return metricsByDimensions; - } - } - - // Used to distinguish whether metrics have been populated with all tag vaules - public enum DimensionType {DEFAULT, PRETAGGED} -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java index 7bd4968747f..b20aa1b11ff 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java @@ -1,8 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi.metrics; /** - * @author valerijf + * @author freva */ public interface MetricValue { Number getValue(); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java new file mode 100644 index 00000000000..f9b169f0a93 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java @@ -0,0 +1,128 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +import com.google.inject.Inject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Stores the latest metric for the given application, name, dimension triplet in memory + * + * @author freva + */ +public class Metrics { + // Application names used + public static final String APPLICATION_HOST = "vespa.host"; + public static final String APPLICATION_NODE = "vespa.node"; + + private final Object monitor = new Object(); + private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>(); + + @Inject + public Metrics() { } + + /** + * Creates a counter metric under vespa.host application, with no dimensions and default dimension type + * See {@link #declareCounter(String, String, Dimensions, DimensionType)} + */ + public Counter declareCounter(String name) { + return declareCounter(name, Dimensions.NONE); + } + + /** + * Creates a counter metric under vespa.host application, with the given dimensions and default dimension type + * See {@link #declareCounter(String, String, Dimensions, DimensionType)} + */ + public Counter declareCounter(String name, Dimensions dimensions) { + return declareCounter(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); + } + + /** Creates a counter metric. This method is idempotent. */ + public Counter declareCounter(String application, String name, Dimensions dimensions, DimensionType type) { + synchronized (monitor) { + return (Counter) getOrCreateApplicationMetrics(application, type) + .computeIfAbsent(dimensions, d -> new HashMap<>()) + .computeIfAbsent(name, n -> new Counter()); + } + } + + /** + * Creates a gauge metric under vespa.host application, with no dimensions and default dimension type + * See {@link #declareGauge(String, String, Dimensions, DimensionType)} + */ + public Gauge declareGauge(String name) { + return declareGauge(name, Dimensions.NONE); + } + + /** + * Creates a gauge metric under vespa.host application, with the given dimensions and default dimension type + * See {@link #declareGauge(String, String, Dimensions, DimensionType)} + */ + public Gauge declareGauge(String name, Dimensions dimensions) { + return declareGauge(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); + } + + /** Creates a gauge metric. This method is idempotent */ + public Gauge declareGauge(String application, String name, Dimensions dimensions, DimensionType type) { + synchronized (monitor) { + return (Gauge) getOrCreateApplicationMetrics(application, type) + .computeIfAbsent(dimensions, d -> new HashMap<>()) + .computeIfAbsent(name, n -> new Gauge()); + } + } + + public List<DimensionMetrics> getDefaultMetrics() { + return getMetricsByType(DimensionType.DEFAULT); + } + + public List<DimensionMetrics> getMetricsByType(DimensionType type) { + synchronized (monitor) { + List<DimensionMetrics> dimensionMetrics = new ArrayList<>(); + metrics.getOrDefault(type, Map.of()) + .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() + .map(entry -> new DimensionMetrics(application, entry.getKey(), + entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) + .forEach(dimensionMetrics::add)); + return dimensionMetrics; + } + } + + public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) { + synchronized (monitor) { + Optional.ofNullable(metrics.get(type)) + .map(m -> m.get(name)) + .map(ApplicationMetrics::metricsByDimensions) + .ifPresent(m -> m.remove(dimensionsToRemove)); + } + } + + Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { + return metrics.computeIfAbsent(type, m -> new HashMap<>()) + .computeIfAbsent(application, app -> new ApplicationMetrics()) + .metricsByDimensions(); + } + + // "Application" is the monitoring application, not Vespa application + private static class ApplicationMetrics { + private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>(); + + Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() { + return metricsByDimensions; + } + } + + // Used to distinguish whether metrics have been populated with all tag vaules + public enum DimensionType { + /** Default metrics get added default dimensions set in check config */ + DEFAULT, + + /** Pretagged metrics will only get the dimensions explicitly set when creating the counter/gauge */ + PRETAGGED + } +} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java index df221302575..4843d8f9685 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java @@ -14,8 +14,7 @@ import com.github.dockerjava.api.command.PullImageCmd; import com.github.dockerjava.api.exception.NotFoundException; import com.github.dockerjava.core.command.ExecStartResultCallback; import com.yahoo.config.provision.DockerImage; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Matchers; @@ -37,8 +36,8 @@ import static org.mockito.Mockito.when; public class DockerImplTest { private final DockerClient dockerClient = mock(DockerClient.class); - private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - private final DockerImpl docker = new DockerImpl(dockerClient, metricReceiver); + private final Metrics metrics = new Metrics(); + private final DockerImpl docker = new DockerImpl(dockerClient, metrics); @Test public void testExecuteCompletes() { diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java deleted file mode 100644 index c20e64d906e..00000000000 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.MetricReceiver; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class MetricReceiverWrapperTest { - private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - private static final String applicationDocker = MetricReceiverWrapper.APPLICATION_DOCKER; - - @Test - public void testDefaultValue() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - - metricReceiver.declareCounter(applicationDocker, hostDimension, "some.name"); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("some.name"), 0L); - } - - @Test - public void testSimpleIncrementMetric() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - CounterWrapper counter = metricReceiver.declareCounter(applicationDocker, hostDimension, "a_counter.value"); - - counter.add(5); - counter.add(8); - - Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension); - assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); - assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 - } - - @Test - public void testSimpleGauge() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - - gauge.sample(42); - gauge.sample(-342.23); - - Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension); - assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); - assertEquals(latestMetrics.get("test.gauge"), -342.23); - } - - @Test - public void testRedeclaringSameGauge() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - gauge.sample(42); - - // Same as hostDimension, but new instance. - Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge"); - newGauge.sample(56); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 56.); - } - - @Test - public void testSameMetricNameButDifferentDimensions() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - gauge.sample(42); - - // Not the same as hostDimension. - Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge"); - newGauge.sample(56); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 42.); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, newDimension).get("test.gauge"), 56.); - } - - @Test - public void testDeletingMetric() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - - Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - metricReceiver.declareGauge(applicationDocker, differentDimension, "test.gauge"); - - assertEquals(2, metricReceiver.getDefaultMetricsRaw().size()); - metricReceiver.deleteMetricByDimension(applicationDocker, differentDimension, MetricReceiverWrapper.DimensionType.DEFAULT); - assertEquals(1, metricReceiver.getDefaultMetricsRaw().size()); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).size(), 1); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, differentDimension).size(), 0); - } -} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java new file mode 100644 index 00000000000..fc153ee0562 --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java @@ -0,0 +1,99 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +import org.junit.Test; + +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.dockerapi.metrics.Metrics.APPLICATION_HOST; +import static com.yahoo.vespa.hosted.dockerapi.metrics.Metrics.DimensionType.DEFAULT; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class MetricsTest { + private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + private final Metrics metrics = new Metrics(); + + @Test + public void testDefaultValue() { + metrics.declareCounter("some.name", hostDimension); + + assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L); + } + + @Test + public void testSimpleIncrementMetric() { + Counter counter = metrics.declareCounter("a_counter.value", hostDimension); + + counter.add(5); + counter.add(8); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); + assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 + } + + @Test + public void testSimpleGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + + gauge.sample(42); + gauge.sample(-342.23); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); + assertEquals(latestMetrics.get("test.gauge"), -342.23); + } + + @Test + public void testRedeclaringSameGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Same as hostDimension, but new instance. + Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.); + } + + @Test + public void testSameMetricNameButDifferentDimensions() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Not the same as hostDimension. + Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.); + assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.); + } + + @Test + public void testDeletingMetric() { + metrics.declareGauge("test.gauge", hostDimension); + + Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + metrics.declareGauge("test.gauge", differentDimension); + + assertEquals(2, metrics.getMetricsByType(DEFAULT).size()); + metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT); + assertEquals(1, metrics.getMetricsByType(DEFAULT).size()); + assertEquals(getMetricsForDimension(hostDimension).size(), 1); + assertEquals(getMetricsForDimension(differentDimension).size(), 0); + } + + private Map<String, Number> getMetricsForDimension(Dimensions dimensions) { + return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT) + .getOrDefault(dimensions, Map.of()) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); + } +} diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index b6a0f165ca6..e772a3138da 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -85,6 +85,7 @@ import static org.junit.Assert.assertThat; /** * Testcases for vespa-documentgen-plugin + * * @author vegardh */ @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 906a56d3f34..66c8da86403 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -105,9 +105,9 @@ public class Flags { "Takes effect on next deployment", APPLICATION_ID); - public static final UnboundListFlag<String> DYNAMIC_PROVISIONING_FLAVORS = defineListFlag( - "dynamic-provisioning-flavors", List.of(), - "List of additional Vespa flavor names that can be used for dynamic provisioning", + public static final UnboundListFlag<String> DISABLED_DYNAMIC_PROVISIONING_FLAVORS = defineListFlag( + "disabled-dynamic-provisioning-flavors", List.of(), + "List of disabled Vespa flavor names that cannot be used for dynamic provisioning", "Takes effect on next provisioning"); public static final UnboundBooleanFlag ENABLE_DISK_WRITE_TEST = defineFeatureFlag( diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java index 897b5d3236d..015d5702c59 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java @@ -44,7 +44,7 @@ public class TestConfig { ZoneId zone = ZoneId.from(config.field("zone").asString()); SystemName system = SystemName.from(config.field("system").asString()); Map<ZoneId, Map<String, URI>> deployments = new HashMap<>(); - config.field("clusterEndpoints").traverse((ObjectTraverser) (zoneId, endpointsObject) -> { + config.field("zoneEndpoints").traverse((ObjectTraverser) (zoneId, endpointsObject) -> { Map<String, URI> endpoints = new HashMap<>(); endpointsObject.traverse((ObjectTraverser) (cluster, uri) -> endpoints.put(cluster, URI.create(uri.asString()))); deployments.put(ZoneId.from(zoneId), endpoints); diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java new file mode 100644 index 00000000000..51fb7a8cf4a --- /dev/null +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java @@ -0,0 +1,37 @@ +package ai.vespa.hosted.api; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class TestConfigTest { + + @Test + public void testDeserialization() throws IOException { + TestConfig config = TestConfig.fromJson(Files.readAllBytes(Paths.get("src/test/resources/test-config.json"))); + assertEquals(ApplicationId.from("t", "a", "i"), + config.application()); + assertEquals(ZoneId.from("dev", "aws-us-east-1c"), + config.zone()); + assertEquals(SystemName.PublicCd, + config.system()); + assertEquals(Map.of(ZoneId.from("dev", "aws-us-east-1c"), + Map.of("default", URI.create("https://dev.endpoint:443/")), + ZoneId.from("prod", "aws-us-east-1a"), + Map.of("default", URI.create("https://prod.endpoint:443/"))), + config.deployments()); + } + +} diff --git a/hosted-api/src/test/resources/test-config.json b/hosted-api/src/test/resources/test-config.json new file mode 100644 index 00000000000..9d36f9496a0 --- /dev/null +++ b/hosted-api/src/test/resources/test-config.json @@ -0,0 +1,13 @@ +{ + "application": "t:a:i", + "zone": "dev.aws-us-east-1c", + "system": "publiccd", + "zoneEndpoints": { + "dev.aws-us-east-1c": { + "default": "https://dev.endpoint:443/" + }, + "prod.aws-us-east-1a": { + "default": "https://prod.endpoint:443/" + } + } +} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 47a9b04291d..83d6a4a0def 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -46,11 +46,7 @@ public class LogWriter { * </UL> */ private Writer nextWriter() throws IOException { - - if (writer != null) { - writer.close(); - } - + close(); int maxAttempts = 1000; while (maxAttempts-- > 0) { String name = prefix + "-" + generation++; @@ -119,15 +115,15 @@ public class LogWriter { } - public void flush() throws IOException { + public synchronized void flush() throws IOException { if (writer != null) { writer.flush(); } } - public void close() throws IOException { - flush(); + public synchronized void close() throws IOException { if (writer != null) { + writer.flush(); writer.close(); writer = null; } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java index 3d692297f1c..5c1da722f57 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java @@ -14,7 +14,7 @@ import java.util.Map; public class LogWriterLRUCache extends LinkedHashMap<Integer, LogWriter> { private static final Logger log = Logger.getLogger(LogWriterLRUCache.class.getName()); - final int maxEntries = 100; + final int maxEntries = 5; public LogWriterLRUCache(int initialCapacity, float loadFactor) { super(initialCapacity, loadFactor, true); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index fe823c72127..14d1203824b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -81,6 +81,8 @@ public class MetricsManager { */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { if (services.isEmpty()) return Collections.emptyList(); + + log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index 054fa704ecb..2ca24dad1e2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -14,7 +14,6 @@ import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; import java.util.ArrayList; import java.util.Collections; @@ -32,7 +31,6 @@ import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static com.google.common.base.Strings.isNullOrEmpty; -import static com.yahoo.log.LogLevel.DEBUG; /** * @author Unknown @@ -77,8 +75,6 @@ public class VespaMetrics { public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) { List<MetricsPacket.Builder> metricsPackets = new ArrayList<>(); - log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); - Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(); for (VespaService service : services) { @@ -86,42 +82,58 @@ public class VespaMetrics { Optional<MetricsPacket.Builder> systemCheck = getSystemMetrics(service); systemCheck.ifPresent(metricsPackets::add); - // One metrics packet per set of metrics that share the same dimensions+consumers - // TODO: Move aggregation into MetricsPacket itself? - Metrics serviceMetrics = getServiceMetrics(service, consumersByMetric); - Map<AggregationKey, List<Metric>> aggregatedMetrics = - aggregateMetrics(service.getDimensions(), serviceMetrics); - - aggregatedMetrics.forEach((aggregationKey, metrics) -> { - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())) - .putMetrics(metrics) - .putDimension(METRIC_TYPE_DIMENSION_ID, "standard") - .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) - .putDimensions(aggregationKey.getDimensions()); - setMetaInfo(builder, serviceMetrics.getTimeStamp()); - builder.addConsumers(aggregationKey.getConsumers()); - metricsPackets.add(builder); - }); + Metrics allServiceMetrics = service.getMetrics(); + + if (! allServiceMetrics.getMetrics().isEmpty()) { + Metrics serviceMetrics = getServiceMetrics(allServiceMetrics, consumersByMetric); + + // One metrics packet per set of metrics that share the same dimensions+consumers + // TODO: Move aggregation into MetricsPacket itself? + Map<AggregationKey, List<Metric>> aggregatedMetrics = aggregateMetrics(service.getDimensions(), serviceMetrics); + + aggregatedMetrics.forEach((aggregationKey, metrics) -> { + MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())) + .putMetrics(metrics) + .putDimension(METRIC_TYPE_DIMENSION_ID, "standard") + .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) + .putDimensions(aggregationKey.getDimensions()); + setMetaInfo(builder, serviceMetrics.getTimeStamp()); + builder.addConsumers(aggregationKey.getConsumers()); + metricsPackets.add(builder); + }); + } else { + // Service did not return any metrics, so add metrics packet based on service health. + // TODO: Make VespaService.getMetrics return MetricsPacket and handle health on its own. + metricsPackets.add(getHealth(service)); + } } - return metricsPackets; } + private MetricsPacket.Builder getHealth(VespaService service) { + HealthMetric health = service.getHealth(); + return new MetricsPacket.Builder(toServiceId(service.getMonitoringName())) + .timestamp(System.currentTimeMillis() / 1000) + .statusCode(health.getStatus().ordinal()) // TODO: MetricsPacket should use StatusCode instead of int + .statusMessage(health.getMessage()) + .putDimensions(service.getDimensions()) + .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()); + } + /** * Returns the metrics to output for the given service, with updated timestamp * In order to include a metric, it must exist in the given map of metric to consumers. * Each returned metric will contain a collection of consumers that it should be routed to. */ - private Metrics getServiceMetrics(VespaService service, Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) { - Metrics serviceMetrics = new Metrics(); - Metrics allServiceMetrics = service.getMetrics(); - serviceMetrics.setTimeStamp(getMostRecentTimestamp(allServiceMetrics)); + private Metrics getServiceMetrics(Metrics allServiceMetrics, Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) { + Metrics configuredServiceMetrics = new Metrics(); + configuredServiceMetrics.setTimeStamp(getMostRecentTimestamp(allServiceMetrics)); for (Metric candidate : allServiceMetrics.getMetrics()) { getConfiguredMetrics(candidate.getName(), consumersByMetric.keySet()).forEach( - configuredMetric -> serviceMetrics.add( + configuredMetric -> configuredServiceMetrics.add( metricWithConfigProperties(candidate, configuredMetric, consumersByMetric))); } - return serviceMetrics; + return configuredServiceMetrics; } private Map<DimensionId, String> extractDimensions(Map<DimensionId, String> dimensions, List<ConsumersConfig.Consumer.Metric.Dimension> configuredDimensions) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java index 41a8c3d414e..4961cc8b2a6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java @@ -4,49 +4,51 @@ package ai.vespa.metricsproxy.metric; +import ai.vespa.metricsproxy.metric.model.StatusCode; + +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; +import static ai.vespa.metricsproxy.metric.model.StatusCode.UNKNOWN; +import static ai.vespa.metricsproxy.metric.model.StatusCode.UP; + /** + * TODO: Use MetricsPacket instead of this class. + * * @author Jo Kristian Bergum */ public class HealthMetric { private final String message; - private final String status; + private final StatusCode status; private final boolean isAlive; - private HealthMetric(String status, String message, boolean isAlive) { + private HealthMetric(StatusCode status, String message, boolean isAlive) { this.message = message; this.status = status; this.isAlive = isAlive; } public static HealthMetric get(String status, String message) { - if (status == null) { - status = ""; - } - if (message == null) { - message = ""; - } - status = status.toLowerCase(); + if (message == null) message = ""; + var statusCode = StatusCode.fromString(status); + return new HealthMetric(statusCode, message, statusCode == UP); + } - if (status.equals("up") || status.equals("ok")) { - return new HealthMetric(status, message, true); - } else { - return new HealthMetric(status, message, false); - } + public static HealthMetric getDown(String message) { + return new HealthMetric(DOWN, message, false); } - public static HealthMetric getFailed(String message) { - return new HealthMetric("down", message, false); + public static HealthMetric getUnknown(String message) { + return new HealthMetric(UNKNOWN, message, false); } public static HealthMetric getOk(String message) { - return new HealthMetric("up", message, true); + return new HealthMetric(UP, message, true); } public String getMessage() { return this.message; } - public String getStatus() { + public StatusCode getStatus() { return this.status; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java index ca611368730..f1d029d8746 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java @@ -30,7 +30,6 @@ public class Metrics { private void ensureNotFrozen() { if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!"); - } public long getTimeStamp() { @@ -84,22 +83,6 @@ public class Metrics { } - /** - * Get a single metric based on the metric name - * TODO: Remove, might be multiple metrics with same name, but different - * - * @param key metric name - * @return The value or null if metric was not found or expired - */ - public Number get(String key) { - isFrozen = true; - Metric m = getMetric(key); - if (m != null) { - return m.getValue(); - } - return null; - } - public String toString() { StringBuilder sb = new StringBuilder(); for (Metric m : metrics) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index fa45c6251f6..098fd48c8b3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -14,7 +14,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -88,7 +87,7 @@ public class MetricsPacket { // Except for 'service' for which we require an explicit non-null value. private ServiceId service; private int statusCode = 0; - private String statusMessage = "<null>"; + private String statusMessage = ""; private long timestamp = 0L; private Map<MetricId, Number> metrics = new LinkedHashMap<>(); private final Map<DimensionId, String> dimensions = new LinkedHashMap<>(); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java new file mode 100644 index 00000000000..7f5a7d0e64b --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/StatusCode.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + */ + +package ai.vespa.metricsproxy.metric.model; + +/** + * Status code for a Vespa service. + * + * @author gjoranv + */ +public enum StatusCode { + + UP(0, "up"), + DOWN(1, "down"), + UNKNOWN(2, "unknown"); + + public final int code; + public final String status; + + StatusCode(int code, String status) { + this.code = code; + this.status = status; + } + + public static StatusCode fromString(String statusString) { + if ("ok".equalsIgnoreCase(statusString)) return UP; + try { + return valueOf(statusString.trim().toUpperCase()); + } catch (IllegalArgumentException | NullPointerException e) { + return UNKNOWN; + } + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java index 495e3ec1f7d..aadcc1418af 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java @@ -6,6 +6,7 @@ package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.ServiceId; +import ai.vespa.metricsproxy.metric.model.StatusCode; import java.util.ArrayList; import java.util.List; @@ -34,9 +35,13 @@ public class GenericJsonUtil { var genericMetricsList = packets.stream() .map(packet -> new GenericMetrics(packet.metrics(), packet.dimensions())) .collect(toList()); - var genericService = new GenericService(serviceId.id, - packets.get(0).timestamp, - genericMetricsList); + var genericService = packets.stream().findFirst() + .map(firstPacket -> new GenericService(serviceId.id, + firstPacket.timestamp, + StatusCode.values()[firstPacket.statusCode], + firstPacket.statusMessage, + genericMetricsList)) + .get(); if (VESPA_NODE_SERVICE_ID.equals(serviceId)) { jsonModel.node = new GenericNode(genericService.timestamp, genericService.metrics); } else { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java index bd3dbf935ed..f348bd4beca 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java @@ -4,12 +4,12 @@ package ai.vespa.metricsproxy.metric.model.json; +import ai.vespa.metricsproxy.metric.model.StatusCode; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.ArrayList; import java.util.List; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; @@ -37,10 +37,11 @@ public class GenericService { public GenericService() { } - GenericService(String name, Long timestamp, List<GenericMetrics> metrics) { + // TODO: take StatusCode instead of int + GenericService(String name, Long timestamp, StatusCode statusCode, String message, List<GenericMetrics> metrics) { this.name = name; this.timestamp = timestamp; - status = new Status("up"); + status = new Status(statusCode, message); this.metrics = metrics; } @@ -50,8 +51,9 @@ public class GenericService { public static class Status { public Status() { } - Status(String code) { - this.code = code; + Status(StatusCode statusCode, String description) { + code = statusCode.status; + this.description = description; } @JsonProperty("code") diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java index f87171a42dc..c9bfc8b365c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java @@ -28,7 +28,7 @@ public class DummyHealthMetricFetcher extends RemoteHealthMetricFetcher { if (service.isAlive()) { return HealthMetric.getOk("Service is running - pid check only"); } else { - return HealthMetric.getFailed("Service is not running - pid check only"); + return HealthMetric.getDown("Service is not running - pid check only"); } } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 9094ef22c20..81358041502 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -25,8 +25,6 @@ import java.util.logging.Logger; public abstract class HttpMetricFetcher { private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName()); public final static String STATE_PATH = "/state/v1/"; - final static String METRICS_PATH = STATE_PATH + "metrics"; - final static String HEALTH_PATH = STATE_PATH + "health"; // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. public static int CONNECTION_TIMEOUT = 5000; private final static int SOCKET_TIMEOUT = 60000; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 503f582a827..068a8faade8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -18,13 +18,10 @@ import java.util.logging.Logger; * @author Jo Kristian Bergum */ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { - private final static Logger log = Logger.getLogger(RemoteHealthMetricFetcher.class.getPackage().getName()); - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ + private final static String HEALTH_PATH = STATE_PATH + "health"; + public RemoteHealthMetricFetcher(VespaService service, int port) { super(service, port, HEALTH_PATH); } @@ -45,8 +42,8 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { /** * Connect to remote service over http and fetch metrics */ - HealthMetric createHealthMetrics(String data, int fetchCount) { - HealthMetric healthMetric = HealthMetric.getFailed("Failed fetching status page for service"); + private HealthMetric createHealthMetrics(String data, int fetchCount) { + HealthMetric healthMetric = HealthMetric.getDown("Failed fetching status page for service"); try { healthMetric = parse(data); } catch (Exception e) { @@ -57,7 +54,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { private HealthMetric parse(String data) { if (data == null || data.isEmpty()) { - return HealthMetric.getFailed("Empty response from status page"); + return HealthMetric.getUnknown("Empty response from status page"); } try { JSONObject o = new JSONObject(data); @@ -71,7 +68,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { } catch (JSONException e) { log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data); - return HealthMetric.getFailed("Not able to parse json from status page"); + return HealthMetric.getUnknown("Not able to parse json from status page"); } } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index a606ec7d8cd..552b4dc4010 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -25,10 +25,9 @@ import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; * @author Jo Kristian Bergum */ public class RemoteMetricsFetcher extends HttpMetricFetcher { - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ + + final static String METRICS_PATH = STATE_PATH + "metrics"; + RemoteMetricsFetcher(VespaService service, int port) { super(service, port, METRICS_PATH); } @@ -50,7 +49,7 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { /** * Connect to remote service over http and fetch metrics */ - public Metrics createMetrics(String data, int fetchCount) { + Metrics createMetrics(String data, int fetchCount) { Metrics remoteMetrics = new Metrics(); try { remoteMetrics = parse(data); @@ -61,7 +60,7 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { return remoteMetrics; } - Metrics parse(String data) throws JSONException { + private Metrics parse(String data) throws JSONException { JSONObject o = new JSONObject(data); if (!(o.has("metrics"))) { return new Metrics(); //empty diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index 1635ccab197..eb620fd37be 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -6,7 +6,7 @@ package ai.vespa.metricsproxy.core; import ai.vespa.metricsproxy.TestUtil; import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.ExternalMetrics; +import ai.vespa.metricsproxy.metric.HealthMetric; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; @@ -15,6 +15,7 @@ import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.service.DownService; import ai.vespa.metricsproxy.service.DummyService; import ai.vespa.metricsproxy.service.VespaService; import ai.vespa.metricsproxy.service.VespaServices; @@ -23,6 +24,7 @@ import org.junit.Before; import org.junit.Test; import java.time.Instant; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -62,6 +64,21 @@ public class MetricsManagerTest { } @Test + public void service_that_is_down_has_a_separate_metrics_packet() { + // Reset to use only the service that is down + var downService = new DownService(HealthMetric.getDown("No response")); + List<VespaService> testServices = Collections.singletonList(downService); + MetricsManager metricsManager = TestUtil.createMetricsManager(new VespaServices(testServices), + getMetricsConsumers(),getApplicationDimensions(), getNodeDimensions()); + + List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); + assertThat(packets.size(), is(1)); + assertTrue(packets.get(0).metrics().isEmpty()); + assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is(DownService.NAME)); + assertThat(packets.get(0).dimensions().get(toDimensionId("global")), is("value")); + } + + @Test public void each_service_gets_separate_metrics_packets() { List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); assertThat(packets.size(), is(2)); @@ -77,11 +94,11 @@ public class MetricsManagerTest { @Test public void verify_expected_output_from_getMetricsById() { - String dummy0Metrics = metricsManager.getMetricsByConfigId("dummy/id/0"); + String dummy0Metrics = metricsManager.getMetricsByConfigId(SERVICE_0_ID); assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050")); assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1")); - String dummy1Metrics = metricsManager.getMetricsByConfigId("dummy/id/1"); + String dummy1Metrics = metricsManager.getMetricsByConfigId(SERVICE_1_ID); assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350")); assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6")); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java index 744c96d7744..301dbf56c3f 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java @@ -8,6 +8,7 @@ import ai.vespa.metricsproxy.TestUtil; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.HealthMetric; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -17,6 +18,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; import ai.vespa.metricsproxy.metric.model.json.GenericService; +import ai.vespa.metricsproxy.service.DownService; import ai.vespa.metricsproxy.service.DummyService; import ai.vespa.metricsproxy.service.VespaService; import ai.vespa.metricsproxy.service.VespaServices; @@ -34,9 +36,11 @@ import java.util.concurrent.Executors; import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID; import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; @@ -47,8 +51,9 @@ import static org.junit.Assert.assertNotNull; public class GenericMetricsHandlerTest { private static final List<VespaService> testServices = ImmutableList.of( - new DummyService(0, "dummy/id/0"), - new DummyService(1, "dummy/id/0")); + new DummyService(0, ""), + new DummyService(1, ""), + new DownService(HealthMetric.getDown("No response"))); private static final String CPU_METRIC = "cpu"; @@ -93,7 +98,7 @@ public class GenericMetricsHandlerTest { String response = testDriver.sendRequest(URI).readAll(); var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class); - assertEquals(1, jsonModel.services.size()); + assertEquals(2, jsonModel.services.size()); GenericService dummyService = jsonModel.services.get(0); assertEquals(2, dummyService.metrics.size()); @@ -107,6 +112,22 @@ public class GenericMetricsHandlerTest { } @Test + public void response_contains_health_from_service_that_is_down() throws Exception { + String response = testDriver.sendRequest(URI).readAll(); + var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class); + + GenericService downService = jsonModel.services.get(1); + assertEquals(DOWN.status, downService.status.code); + assertEquals("No response", downService.status.description); + + // Service should output metric dimensions, even without metrics, because they contain important info about the service. + assertEquals(1, downService.metrics.size()); + assertEquals(0, downService.metrics.get(0).values.size()); + assertFalse(downService.metrics.get(0).dimensions.isEmpty()); + assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id)); + } + + @Test public void all_timestamps_are_equal_and_non_zero() throws Exception { String response = testDriver.sendRequest(URI).readAll(); var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java index b9e6377c27b..91eda8f744c 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java @@ -4,6 +4,7 @@ package ai.vespa.metricsproxy.metric; +import ai.vespa.metricsproxy.metric.model.StatusCode; import ai.vespa.metricsproxy.service.DummyService; import ai.vespa.metricsproxy.service.VespaService; import org.junit.Test; @@ -46,7 +47,8 @@ public class MetricsTest { public void testBasicMetric() { Metrics m = new Metrics(); m.add(new Metric("count", 1, System.currentTimeMillis() / 1000)); - assertThat(m.get("count").intValue(), is(1)); + assertThat(m.getMetrics().size(), is(1)); + assertThat(m.getMetrics().get(0).getName(), is("count")); } @Test @@ -62,7 +64,7 @@ public class MetricsTest { m = HealthMetric.get("bad", "test message"); assertThat(m.isOk(), is(false)); - assertThat(m.getStatus(), is("bad")); + assertThat(m.getStatus(), is(StatusCode.UNKNOWN)); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java new file mode 100644 index 00000000000..8a66f98f42a --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/StatusCodeTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + */ + +package ai.vespa.metricsproxy.metric.model; + +import org.junit.Test; + +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; +import static ai.vespa.metricsproxy.metric.model.StatusCode.UNKNOWN; +import static ai.vespa.metricsproxy.metric.model.StatusCode.UP; +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class StatusCodeTest { + + @Test + public void strings_are_converted_to_correct_status_code() { + assertEquals(UP, StatusCode.fromString("up")); + assertEquals(UP, StatusCode.fromString("UP")); + assertEquals(UP, StatusCode.fromString("ok")); + assertEquals(UP, StatusCode.fromString("OK")); + + assertEquals(DOWN, StatusCode.fromString("down")); + assertEquals(DOWN, StatusCode.fromString("DOWN")); + + assertEquals(UNKNOWN, StatusCode.fromString("unknown")); + assertEquals(UNKNOWN, StatusCode.fromString("UNKNOWN")); + assertEquals(UNKNOWN, StatusCode.fromString("anything else is unknown")); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java index dc3eb12ff2c..5d248db8b18 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java @@ -5,6 +5,7 @@ package ai.vespa.metricsproxy.metric.model.json; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; @@ -19,6 +20,7 @@ import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; /** * @author gjoranv @@ -53,6 +55,7 @@ public class GenericJsonModelTest { var servicePacket = new MetricsPacket.Builder(toServiceId("my-service")) .timestamp(123456L) + .statusCode(0) .putMetric(toMetricId("service-metric"), 1234) .putDimension(toDimensionId("service-dim"), "service-dim-value") .build(); @@ -69,6 +72,9 @@ public class GenericJsonModelTest { assertEquals(1, jsonModel.services.size()); GenericService service = jsonModel.services.get(0); + assertEquals(StatusCode.UP.status, service.status.code); + assertEquals("", service.status.description); + assertEquals(1, service.metrics.size()); GenericMetrics serviceMetrics = service.metrics.get(0); assertEquals(1234L, serviceMetrics.values.get("service-metric").longValue()); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 0a84c03acec..725501aacaa 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; +import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -34,7 +35,7 @@ public class ContainerServiceTest { csPort = 18637; // see factory/doc/port-ranges.txt try { String response = getFileContents("metrics-container-state-multi-chain.json"); - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); + service = new MockHttpServer(csPort, response, METRICS_PATH); } catch (Exception e) { e.printStackTrace(); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java new file mode 100644 index 00000000000..f78c496fcd1 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DownService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + */ + +package ai.vespa.metricsproxy.service; + +import ai.vespa.metricsproxy.metric.HealthMetric; +import ai.vespa.metricsproxy.metric.Metrics; + +/** + * @author gjoranv + */ +public class DownService extends VespaService { + public static final String NAME = "down-service"; + + private final HealthMetric healthMetric; + + public DownService(HealthMetric healthMetric) { + super(NAME, ""); + this.healthMetric = healthMetric; + } + + @Override + public Metrics getMetrics() { + return new Metrics(); + } + + @Override + public HealthMetric getHealth() { + return healthMetric; + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java index 278b2a3143a..ce33ce9f7bf 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java @@ -22,10 +22,10 @@ public class MetricsFetcherTest { String jsonData = TestUtil.getFileContents("metrics-state.json"); RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); Metrics metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(10)); - assertThat("Wrong value for metric", metrics.get("query_hits.count").intValue(), is(28)); - assertThat("Wrong value for metric ", metrics.get("queries.rate").doubleValue(), is(0.4667)); - assertThat("Wrong timestamp", metrics.getTimeStamp(), is(1334134700L)); + assertThat(metrics.size(), is(10)); + assertThat(metrics.getMetric("query_hits.count").getValue().intValue(), is(28)); + assertThat(metrics.getMetric("queries.rate").getValue().doubleValue(), is(0.4667)); + assertThat(metrics.getTimeStamp(), is(1334134700L)); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java index 702f741c41f..a0c6b5333cc 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java @@ -10,6 +10,7 @@ import org.junit.Before; import org.junit.Test; import static ai.vespa.metricsproxy.TestUtil.getFileContents; +import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -30,7 +31,7 @@ public class VespaServiceTest { public void setupHTTPServer() { csPort = 18632; // see factory/doc/port-ranges.txt try { - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); + service = new MockHttpServer(csPort, response, METRICS_PATH); } catch (Exception e) { e.printStackTrace(); } diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index db00c686c99..d5a4dce7c5a 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -5,7 +5,7 @@ <!-- Please update container test when changing this file --> <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" symlinkName="access.log" /> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> - <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> + <component id="metrics" class="com.yahoo.vespa.hosted.dockerapi.metrics.Metrics" bundle="docker-api"/> <preprocess:include file="variant.xml" required="false"/> </container> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 0e6000c651b..73c86fc8de1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import org.apache.http.HttpHeaders; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; @@ -38,6 +37,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; /** * Retries request on config server a few times before giving up. Assumes that all requests should be sent with @@ -47,7 +47,7 @@ import java.util.Optional; * @author bjorncs */ public class ConfigServerApiImpl implements ConfigServerApi { - private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerApiImpl.class); + private static final Logger logger = Logger.getLogger(ConfigServerApiImpl.class.getName()); private final ObjectMapper mapper = new ObjectMapper(); @@ -106,7 +106,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { try { return mapper.readValue(response.getEntity().getContent(), wantedReturnType); } catch (IOException e) { - throw new RuntimeException("Failed parse response from config server", e); + throw new UncheckedIOException("Failed parse response from config server", e); } } catch (HttpException e) { if (!e.isRetryable()) throw e; @@ -117,15 +117,15 @@ public class ConfigServerApiImpl implements ConfigServerApi { // Failure to communicate with a config server is not abnormal during upgrades if (e.getMessage().contains("(Connection refused)")) { - NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); + logger.info("Connection refused to " + configServer + " (upgrading?), will try next"); } else { - NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); + logger.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); } } } - throw new RuntimeException("All requests against the config servers (" - + configServers + ") failed, last as follows:", lastException); + throw HttpException.handleException( + "All requests against the config servers (" + configServers + ") failed, last as follows:", lastException); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java index 256fe38ec68..3825107bfa6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java @@ -1,13 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import org.apache.http.NoHttpResponseException; + import javax.ws.rs.core.Response; +import java.io.EOFException; +import java.net.SocketException; +import java.net.SocketTimeoutException; /** * @author hakonhall */ @SuppressWarnings("serial") -public class HttpException extends RuntimeException { +public class HttpException extends ConvergenceException { private final boolean isRetryable; @@ -21,7 +27,12 @@ public class HttpException extends RuntimeException { this.isRetryable = isRetryable; } - public boolean isRetryable() { + private HttpException(String message) { + super(message); + this.isRetryable = false; + } + + boolean isRetryable() { return isRetryable; } @@ -55,6 +66,22 @@ public class HttpException extends RuntimeException { throw new HttpException(status, message, true); } + /** + * Returns {@link HttpException} if the given Throwable is of a known and well understood error or + * a RuntimeException with the given exception as cause otherwise. + */ + public static RuntimeException handleException(String prefix, Throwable t) { + for (; t != null; t = t.getCause()) { + if (t instanceof SocketException || + t instanceof SocketTimeoutException || + t instanceof NoHttpResponseException || + t instanceof EOFException) + return new HttpException(prefix + t.getMessage()); + } + + return new RuntimeException(prefix, t); + } + public static class NotFoundException extends HttpException { public NotFoundException(String message) { super(Response.Status.NOT_FOUND, message, false); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java index 6094518c3fc..10e61d373d2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java @@ -1,7 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; -public class NodeRepositoryException extends RuntimeException { +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; + +public class NodeRepositoryException extends ConvergenceException { public NodeRepositoryException(String message) { super(message); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index d402e75ff7b..52d6f16dd78 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -91,6 +91,13 @@ public class NodeSpec { Set<String> additionalIpAddresses, NodeReports reports, Optional<String> parentHostname) { + if (state == NodeState.active) { + Objects.requireNonNull(wantedVespaVersion, "Unknown vespa version for active node"); + Objects.requireNonNull(wantedDockerImage, "Unknown docker image for active node"); + Objects.requireNonNull(wantedRestartGeneration, "Unknown restartGeneration for active node"); + Objects.requireNonNull(currentRestartGeneration, "Unknown currentRestartGeneration for active node"); + } + this.hostname = Objects.requireNonNull(hostname); this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage); this.currentDockerImage = Objects.requireNonNull(currentDockerImage); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index ca52eca13d2..fe19b81614d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -12,10 +12,8 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Ge import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetNodesResponse; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeMessageResponse; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.time.Instant; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -23,6 +21,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,7 +29,7 @@ import java.util.stream.Stream; * @author stiankri, dybis */ public class RealNodeRepository implements NodeRepository { - private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(RealNodeRepository.class); + private static final Logger logger = Logger.getLogger(RealNodeRepository.class.getName()); private final ConfigServerApi configServerApi; @@ -46,7 +45,7 @@ public class RealNodeRepository implements NodeRepository { NodeMessageResponse response = configServerApi.post("/nodes/v2/node", nodesToPost, NodeMessageResponse.class); if (Strings.isNullOrEmpty(response.errorCode)) return; - throw new NodeRepositoryException("Failed to add nodes to node-repo: " + response.message + " " + response.errorCode); + throw new NodeRepositoryException("Failed to add nodes: " + response.message + " " + response.errorCode); } @Override @@ -80,43 +79,37 @@ public class RealNodeRepository implements NodeRepository { */ @Override public Map<String, Acl> getAcls(String hostName) { - try { - String path = String.format("/nodes/v2/acl/%s?children=true", hostName); - GetAclResponse response = configServerApi.get(path, GetAclResponse.class); - - // Group ports by container hostname that trusts them - Map<String, Set<Integer>> trustedPorts = response.trustedPorts.stream() - .collect(Collectors.groupingBy( - GetAclResponse.Port::getTrustedBy, - Collectors.mapping(port -> port.port, Collectors.toSet()))); - - // Group node ip-addresses by container hostname that trusts them - Map<String, Set<Acl.Node>> trustedNodes = response.trustedNodes.stream() - .collect(Collectors.groupingBy( - GetAclResponse.Node::getTrustedBy, - Collectors.mapping( - node -> new Acl.Node(node.hostname, node.ipAddress), - Collectors.toSet()))); - - // Group trusted networks by container hostname that trusts them - Map<String, Set<String>> trustedNetworks = response.trustedNetworks.stream() - .collect(Collectors.groupingBy(GetAclResponse.Network::getTrustedBy, - Collectors.mapping(node -> node.network, Collectors.toSet()))); - - - // For each hostname create an ACL - return Stream.of(trustedNodes.keySet(), trustedPorts.keySet(), trustedNetworks.keySet()) - .flatMap(Set::stream) - .distinct() - .collect(Collectors.toMap( - Function.identity(), - hostname -> new Acl(trustedPorts.get(hostname), trustedNodes.get(hostname), - trustedNetworks.get(hostname)))); - } catch (HttpException.NotFoundException e) { - NODE_ADMIN_LOGGER.warning("Failed to fetch ACLs for " + hostName + " No ACL will be applied"); - } - - return Collections.emptyMap(); + String path = String.format("/nodes/v2/acl/%s?children=true", hostName); + GetAclResponse response = configServerApi.get(path, GetAclResponse.class); + + // Group ports by container hostname that trusts them + Map<String, Set<Integer>> trustedPorts = response.trustedPorts.stream() + .collect(Collectors.groupingBy( + GetAclResponse.Port::getTrustedBy, + Collectors.mapping(port -> port.port, Collectors.toSet()))); + + // Group node ip-addresses by container hostname that trusts them + Map<String, Set<Acl.Node>> trustedNodes = response.trustedNodes.stream() + .collect(Collectors.groupingBy( + GetAclResponse.Node::getTrustedBy, + Collectors.mapping( + node -> new Acl.Node(node.hostname, node.ipAddress), + Collectors.toSet()))); + + // Group trusted networks by container hostname that trusts them + Map<String, Set<String>> trustedNetworks = response.trustedNetworks.stream() + .collect(Collectors.groupingBy(GetAclResponse.Network::getTrustedBy, + Collectors.mapping(node -> node.network, Collectors.toSet()))); + + + // For each hostname create an ACL + return Stream.of(trustedNodes.keySet(), trustedPorts.keySet(), trustedNetworks.keySet()) + .flatMap(Set::stream) + .distinct() + .collect(Collectors.toMap( + Function.identity(), + hostname -> new Acl(trustedPorts.get(hostname), trustedNodes.get(hostname), + trustedNetworks.get(hostname)))); } @Override @@ -127,7 +120,7 @@ public class RealNodeRepository implements NodeRepository { NodeMessageResponse.class); if (Strings.isNullOrEmpty(response.errorCode)) return; - throw new NodeRepositoryException("Unexpected message " + response.message + " " + response.errorCode); + throw new NodeRepositoryException("Failed to update node attributes: " + response.message + " " + response.errorCode); } @Override @@ -137,10 +130,10 @@ public class RealNodeRepository implements NodeRepository { "/nodes/v2/state/" + state + "/" + hostName, Optional.empty(), /* body */ NodeMessageResponse.class); - NODE_ADMIN_LOGGER.info(response.message); + logger.info(response.message); if (Strings.isNullOrEmpty(response.errorCode)) return; - throw new NodeRepositoryException("Unexpected message " + response.message + " " + response.errorCode); + throw new NodeRepositoryException("Failed to set node state: " + response.message + " " + response.errorCode); } private static NodeSpec createNodeSpec(NodeRepositoryNode node) { @@ -149,30 +142,13 @@ public class RealNodeRepository implements NodeRepository { Objects.requireNonNull(node.state, "Unknown node state"); NodeState nodeState = NodeState.valueOf(node.state); - if (nodeState == NodeState.active) { - Objects.requireNonNull(node.wantedVespaVersion, "Unknown vespa version for active node"); - Objects.requireNonNull(node.wantedDockerImage, "Unknown docker image for active node"); - Objects.requireNonNull(node.restartGeneration, "Unknown restartGeneration for active node"); - Objects.requireNonNull(node.currentRestartGeneration, "Unknown currentRestartGeneration for active node"); - } - - String hostName = Objects.requireNonNull(node.hostname, "hostname is null"); - - NodeOwner owner = null; - if (node.owner != null) { - owner = new NodeOwner(node.owner.tenant, node.owner.application, node.owner.instance); - } - - NodeMembership membership = null; - if (node.membership != null) { - membership = new NodeMembership(node.membership.clusterType, node.membership.clusterId, - node.membership.group, node.membership.index, node.membership.retired); - } - NodeReports reports = NodeReports.fromMap(node.reports == null ? Collections.emptyMap() : node.reports); + Optional<NodeMembership> membership = Optional.ofNullable(node.membership) + .map(m -> new NodeMembership(m.clusterType, m.clusterId, m.group, m.index, m.retired)); + NodeReports reports = NodeReports.fromMap(Optional.ofNullable(node.reports).orElseGet(Map::of)); return new NodeSpec( - hostName, + node.hostname, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString), nodeState, @@ -185,8 +161,8 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.currentOsVersion).map(Version::fromString), Optional.ofNullable(node.allowedToBeDown), Optional.ofNullable(node.wantToDeprovision), - Optional.ofNullable(owner), - Optional.ofNullable(membership), + Optional.ofNullable(node.owner).map(o -> new NodeOwner(o.tenant, o.application, o.instance)), + membership, Optional.ofNullable(node.restartGeneration), Optional.ofNullable(node.currentRestartGeneration), node.rebootGeneration, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java index fe19da0c41c..8575bf7f655 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; + @SuppressWarnings("serial") -public class OrchestratorException extends RuntimeException { +public class OrchestratorException extends ConvergenceException { public OrchestratorException(String message) { super(message); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java index 64a67aa612a..6124e1bdc0e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java @@ -40,8 +40,7 @@ public class OrchestratorImpl implements Orchestrator { } catch (HttpException.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found"); } catch (HttpException e) { - throw new OrchestratorException("Failed to suspend " + hostName + ": " + - e.toString()); + throw new OrchestratorException("Failed to suspend " + hostName + ": " + e.toString()); } catch (RuntimeException e) { throw new RuntimeException("Got error on suspend", e); } @@ -60,9 +59,8 @@ public class OrchestratorImpl implements Orchestrator { parentHostName, params); batchOperationResult = configServerApi.put(url, Optional.empty(), BatchOperationResult.class); } catch (HttpException e) { - throw new OrchestratorException("Failed to batch suspend for " + - parentHostName + ": " + e.toString()); - } catch (Exception e) { + throw new OrchestratorException("Failed to batch suspend for " + parentHostName + ": " + e.toString()); + } catch (RuntimeException e) { throw new RuntimeException("Got error on batch suspend for " + parentHostName + ", with nodes " + hostNames, e); } @@ -80,9 +78,8 @@ public class OrchestratorImpl implements Orchestrator { } catch (HttpException.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found"); } catch (HttpException e) { - throw new OrchestratorException("Failed to suspend " + hostName + ": " + - e.toString()); - } catch (Exception e) { + throw new OrchestratorException("Failed to suspend " + hostName + ": " + e.toString()); + } catch (RuntimeException e) { throw new RuntimeException("Got error on resume", e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java index efeb3039379..2fe8d4b4792 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.state; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; /** @@ -16,26 +17,12 @@ public class StateImpl implements State { @Override public HealthCode getHealth() { - HealthResponse response; try { - response = configServerApi.get("/state/v1/health", HealthResponse.class); - } catch (RuntimeException e) { - if (causedByConnectionRefused(e)) { - return HealthCode.DOWN; - } - - throw e; + HealthResponse response = configServerApi.get("/state/v1/health", HealthResponse.class); + return HealthCode.fromString(response.status.code); + } catch (HttpException e) { + return HealthCode.DOWN; } - return HealthCode.fromString(response.status.code); } - private static boolean causedByConnectionRefused(Throwable throwable) { - for (Throwable cause = throwable; cause != null; cause = cause.getCause()) { - if (cause instanceof java.net.ConnectException) { - return true; - } - } - - return false; - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index 00ec985ba0c..7de2aae77c8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -16,11 +16,8 @@ public interface NodeAdmin { /** Start/stop NodeAgents and schedule next NodeAgent ticks with the given NodeAgentContexts */ void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts); - /** Gather node agent and its docker container metrics and forward them to the {@code MetricReceiverWrapper} */ - void updateNodeAgentMetrics(); - - /** Gather node admin metrics and forward them to the {@code MetricReceiverWrapper} */ - void updateNodeAdminMetrics(); + /** Update node admin metrics */ + void updateMetrics(); /** * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 0d520241ac8..cb10eac9e6c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Counter; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Gauge; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextManager; @@ -39,28 +39,26 @@ public class NodeAdminImpl implements NodeAdmin { private boolean previousWantFrozen; private boolean isFrozen; private Instant startOfFreezeConvergence; - private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>(); - private final GaugeWrapper numberOfContainersInLoadImageState; - private final GaugeWrapper jvmHeapUsed; - private final GaugeWrapper jvmHeapFree; - private final GaugeWrapper jvmHeapTotal; - private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; + private final Gauge jvmHeapUsed; + private final Gauge jvmHeapFree; + private final Gauge jvmHeapTotal; + private final Counter numberOfUnhandledExceptions; - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, MetricReceiverWrapper metricReceiver, Clock clock) { + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock) { this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metricReceiver, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); + metrics, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); } - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, MetricReceiverWrapper metricReceiver, + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metricReceiver, clock, freezeTimeout, spread); + metrics, clock, freezeTimeout, spread); } NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, - MetricReceiverWrapper metricReceiver, Clock clock, Duration freezeTimeout, Duration spread) { + Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; this.clock = clock; @@ -70,13 +68,12 @@ public class NodeAdminImpl implements NodeAdmin { this.isFrozen = true; this.startOfFreezeConvergence = clock.instant(); - Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); - this.numberOfContainersInLoadImageState = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.image.loading"); - this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.unhandled_exceptions"); + this.numberOfUnhandledExceptions = metrics.declareCounter("unhandled_exceptions", + new Dimensions(Map.of("src", "node-agents"))); - this.jvmHeapUsed = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.used"); - this.jvmHeapFree = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.free"); - this.jvmHeapTotal = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.total"); + this.jvmHeapUsed = metrics.declareGauge("mem.heap.used"); + this.jvmHeapFree = metrics.declareGauge("mem.heap.free"); + this.jvmHeapTotal = metrics.declareGauge("mem.heap.total"); } @Override @@ -105,22 +102,12 @@ public class NodeAdminImpl implements NodeAdmin { } @Override - public void updateNodeAgentMetrics() { - int numberContainersWaitingImage = 0; - int numberOfNewUnhandledExceptions = 0; - + public void updateMetrics() { for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) { - if (nodeAgentWithScheduler.isDownloadingImage()) numberContainersWaitingImage++; - numberOfNewUnhandledExceptions += nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions(); + numberOfUnhandledExceptions.add(nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions()); nodeAgentWithScheduler.updateContainerNodeMetrics(); } - numberOfContainersInLoadImageState.sample(numberContainersWaitingImage); - numberOfUnhandledExceptionsInNodeAgent.add(numberOfNewUnhandledExceptions); - } - - @Override - public void updateNodeAdminMetrics() { Runtime runtime = Runtime.getRuntime(); long freeMemory = runtime.freeMemory(); long totalMemory = runtime.totalMemory(); @@ -208,7 +195,6 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void stopForHostSuspension() { nodeAgent.stopForHostSuspension(); } @Override public void stopForRemoval() { nodeAgent.stopForRemoval(); } @Override public void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(); } - @Override public boolean isDownloadingImage() { return nodeAgent.isDownloadingImage(); } @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 2cd15a3ebe4..f3302bd2359 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -80,8 +80,7 @@ public class NodeAdminStateUpdater { metricsScheduler.scheduleAtFixedRate(() -> { try { if (suspendedStates.contains(currentState)) return; - nodeAdmin.updateNodeAgentMetrics(); - nodeAdmin.updateNodeAdminMetrics(); + nodeAdmin.updateMetrics(); } catch (Throwable e) { log.log(Level.WARNING, "Metric fetcher scheduler failed", e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index d62cb8e45d9..de5ee1b69a4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -33,11 +33,6 @@ public interface NodeAgent { void updateContainerNodeMetrics(); /** - * Returns true if NodeAgent is waiting for an image download to finish - */ - boolean isDownloadingImage(); - - /** * Returns and resets number of unhandled exceptions */ int getAndResetNumberOfUnhandledExceptions(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 8c38b1bbd84..2716cc8cc59 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; @@ -40,7 +40,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; -import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT; @@ -73,8 +72,6 @@ public class NodeAgentImpl implements NodeAgent { private final DoubleFlag containerCpuCap; private int numberOfUnhandledException = 0; - private DockerImage imageBeingDownloaded = null; - private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -378,20 +375,16 @@ public class NodeAgentImpl implements NodeAgent { || (zone.getSystemName().isCd() && zone.getEnvironment() != Environment.prod); } - private void scheduleDownLoadIfNeeded(NodeSpec node, Optional<Container> container) { - if (node.getWantedDockerImage().equals(container.map(c -> c.image))) return; + private boolean downloadImageIfNeeded(NodeSpec node, Optional<Container> container) { + if (node.getWantedDockerImage().equals(container.map(c -> c.image))) return false; - if (dockerOperations.pullImageAsyncIfNeeded(node.getWantedDockerImage().get())) { - imageBeingDownloaded = node.getWantedDockerImage().get(); - } else if (imageBeingDownloaded != null) { // Image was downloading, but now it's ready - imageBeingDownloaded = null; - } + return node.getWantedDockerImage().map(dockerOperations::pullImageAsyncIfNeeded).orElse(false); } public void converge(NodeAgentContext context) { try { doConverge(context); - } catch (OrchestratorException | ConvergenceException e) { + } catch (ConvergenceException e) { context.log(logger, e.getMessage()); } catch (ContainerNotFoundException e) { containerState = ABSENT; @@ -436,6 +429,7 @@ public class NodeAgentImpl implements NodeAgent { case reserved: case parked: case failed: + case inactive: removeContainerIfNeededUpdateContainerState(context, container); updateNodeRepoWithCurrentAttributes(context); break; @@ -447,9 +441,8 @@ public class NodeAgentImpl implements NodeAgent { .filter(diskUtil -> diskUtil >= 0.8) .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context)); - scheduleDownLoadIfNeeded(node, container); - if (isDownloadingImage()) { - context.log(logger, "Waiting for image to download " + imageBeingDownloaded.asString()); + if (downloadImageIfNeeded(node, container)) { + context.log(logger, "Waiting for image to download " + context.node().getWantedDockerImage().get().asString()); return; } container = removeContainerIfNeededUpdateContainerState(context, container); @@ -481,10 +474,6 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Call resume against Orchestrator"); orchestrator.resume(context.hostname().value()); break; - case inactive: - removeContainerIfNeededUpdateContainerState(context, container); - updateNodeRepoWithCurrentAttributes(context); - break; case provisioned: nodeRepository.setNodeState(context.hostname().value(), NodeState.dirty); break; @@ -497,7 +486,7 @@ public class NodeAgentImpl implements NodeAgent { nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); break; default: - throw new RuntimeException("UNKNOWN STATE " + node.getState().name()); + throw new ConvergenceException("UNKNOWN STATE " + node.getState().name()); } } @@ -543,7 +532,7 @@ public class NodeAgentImpl implements NodeAgent { Dimensions dimensions = dimensionsBuilder.build(); ContainerStats stats = containerStats.get(); - final String APP = MetricReceiverWrapper.APPLICATION_NODE; + final String APP = Metrics.APPLICATION_NODE; final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus(); final long cpuContainerKernelTime = stats.getCpuStats().getUsageInKernelMode(); final long cpuContainerTotalTime = stats.getCpuStats().getTotalUsage(); @@ -604,27 +593,15 @@ public class NodeAgentImpl implements NodeAgent { for (DimensionMetrics dimensionMetrics : metrics) { params.append(dimensionMetrics.toSecretAgentReport()); } - } catch (JsonProcessingException e) { - // TODO: wrap everything into one try-block (to avoid 'return') when old metrics proxy is discontinued - context.log(logger, LogLevel.WARNING, "Failed to wrap metrics in secret agent report", e); - return; - } - String wrappedMetrics = "s:" + params.toString(); - - // Push metrics to the metrics proxy in each container - runPushMetricsCommand(context, wrappedMetrics, true); - runPushMetricsCommand(context, wrappedMetrics, false); - } + String wrappedMetrics = "s:" + params.toString(); - // TODO: Clean up and inline method when old metrics proxy has been discontinued. - private void runPushMetricsCommand(NodeAgentContext context, String wrappedMetrics, boolean newMetricsProxy) { - int port = newMetricsProxy ? 19095 : 19091; - String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:" + port, "setExtraMetrics", wrappedMetrics}; - try { + // Push metrics to the metrics proxy in each container. + // TODO Remove port selection logic when all hosted apps have upgraded to Vespa 7. + int port = context.node().getVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095; + String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:" + port, "setExtraMetrics", wrappedMetrics}; dockerOperations.executeCommandInContainerAsRoot(context, 5L, command); - } catch (DockerExecTimeoutException e) { - Level level = newMetricsProxy ? LogLevel.DEBUG : LogLevel.WARNING; - context.log(logger, level, "Failed to push metrics to container", e); + } catch (JsonProcessingException | DockerExecTimeoutException e) { + context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e); } } @@ -637,11 +614,6 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public boolean isDownloadingImage() { - return imageBeingDownloaded != null; - } - - @Override public int getAndResetNumberOfUnhandledExceptions() { int temp = numberOfUnhandledException; numberOfUnhandledException = 0; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java new file mode 100644 index 00000000000..5400c19d63e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Path; +import java.util.logging.Logger; + +/** + * Class wrapping a boolean stored on disk. + * + * <p>The implementation is compatible with {@link StoredInteger} when absence or 0 means false. + * + * @author hakonhall + */ +public class StoredBoolean { + private static Logger logger = Logger.getLogger(StoredBoolean.class.getName()); + + private final UnixPath path; + + /** The parent directory must exist. Value is false by default. */ + public StoredBoolean(Path path) { + this.path = new UnixPath(path); + } + + public boolean value() { + return path.readUtf8FileIfExists().map(String::trim).map(s -> !"0".equals(s)).orElse(false); + } + + /** Sets value to true. */ + public void set(TaskContext context) { + if (!value()) { + context.log(logger, "Writes " + path); + path.writeUtf8File("1"); + } + } + + public void set(TaskContext context, boolean value) { + if (value) { + set(context); + } else { + clear(context); + } + } + + /** Sets value to false. */ + public void clear(TaskContext context) { + if (value()) { + context.log(logger, "Deletes " + path); + path.deleteIfExists(); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 1d0299ae272..376fda1d2dc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -50,10 +50,24 @@ public class UnixPath { return path; } + public boolean exists() { + return Files.exists(path); + } + public String readUtf8File() { return new String(readBytes(), StandardCharsets.UTF_8); } + public Optional<String> readUtf8FileIfExists() { + try { + return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + } catch (NoSuchFileException ignored) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public byte[] readBytes() { return uncheck(() -> Files.readAllBytes(path)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PrefixLogger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PrefixLogger.java deleted file mode 100644 index f4d85a19f6d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PrefixLogger.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; - -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; - -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class PrefixLogger { - private final String prefix; - private final Logger logger; - - private <T> PrefixLogger(Class<T> clazz, String prefix) { - this.logger = Logger.getLogger(clazz.getName()); - this.prefix = prefix + ": "; - } - - public static <T> PrefixLogger getNodeAdminLogger(Class<T> clazz) { - return new PrefixLogger(clazz, "NodeAdmin"); - } - - public static <T> PrefixLogger getNodeAgentLogger(Class<T> clazz, ContainerName containerName) { - return new PrefixLogger(clazz, "NodeAgent-" + containerName.asString()); - } - - private void log(Level level, String message, Throwable thrown) { - logger.log(level, prefix + message, thrown); - } - - private void log(Level level, String message) { - logger.log(level, prefix + message); - } - - - public void debug(String message) { - log(LogLevel.DEBUG, message); - } - - public void info(String message) { - log(LogLevel.INFO, message); - } - - public void info(String message, Throwable thrown) { - log(LogLevel.INFO, message, thrown); - } - - public void error(String message) { - log(LogLevel.ERROR, message); - } - - public void error(String message, Throwable thrown) { - log(LogLevel.ERROR, message, thrown); - } - - public void warning(String message) { - log(LogLevel.WARNING, message); - } - - public void warning(String message, Throwable thrown) { - log(LogLevel.WARNING, message, thrown); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java index 2604aa05367..14755ebf9cc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.state; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; import org.junit.Test; @@ -28,7 +29,7 @@ public class StateImplTest { @Test public void connectException() { - RuntimeException exception = new RuntimeException(new ConnectException("connection refused")); + RuntimeException exception = HttpException.handleException("Error: ", new ConnectException("connection refused")); when(api.get(any(), any())).thenThrow(exception); HealthCode code = state.getHealth(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index f9524d32c81..7f0f3fd37f6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.collections.Pair; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; @@ -92,11 +91,11 @@ public class DockerTester implements AutoCloseable { FileSystem fileSystem = TestFileSystem.create(); DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, ipAddresses); - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); + Metrics metrics = new Metrics(); NodeAgentFactory nodeAgentFactory = contextSupplier -> new NodeAgentImpl( contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, Optional.empty(), Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, mr, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index ce2e9f6d7a2..6e645e6c70f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.config.provision.NodeType; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import org.junit.Test; import org.mockito.InOrder; @@ -40,11 +38,10 @@ import static org.mockito.Mockito.when; public class NodeAdminImplTest { private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); - private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); private final ManualClock clock = new ManualClock(); private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock, Duration.ZERO, Duration.ZERO); + new Metrics(), clock, Duration.ZERO, Duration.ZERO); @Test public void nodeAgentsAreProperlyLifeCycleManaged() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index c43750684f6..f754d1798ec 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Container; @@ -13,7 +12,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; @@ -34,10 +33,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Map; +import java.util.List; import java.util.Optional; -import java.util.Set; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -79,7 +76,7 @@ public class NodeAgentImplTest { private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); + private final Metrics metrics = new Metrics(); private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); @@ -688,9 +685,6 @@ public class NodeAgentImplTest { .replaceAll("\"timestamp\":\\d+", "\"timestamp\":0") .replaceAll("([0-9]+\\.[0-9]{1,3})([0-9]*)", "$1"); // Only keep the first 3 decimals - // TODO: Remove when old metrics proxy is discontinued. - calledCommand[3] = calledCommand[3].replaceFirst("19091", "19095"); - assertEquals(context, calledContainerName); assertEquals(5L, calledTimeout); assertArrayEquals(expectedCommand, calledCommand); @@ -713,8 +707,7 @@ public class NodeAgentImplTest { nodeAgent.updateContainerNodeMetrics(); - Set<Map<String, Object>> actualMetrics = metricReceiver.getDefaultMetricsRaw(); - assertEquals(Collections.emptySet(), actualMetrics); + assertEquals(List.of(), metrics.getDefaultMetrics()); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java new file mode 100644 index 00000000000..bc5487f740c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +public class StoredBooleanTest { + private final TaskContext context = mock(TaskContext.class); + private final FileSystem fileSystem = TestFileSystem.create(); + private final Path path = fileSystem.getPath("/foo"); + private final StoredBoolean storedBoolean = new StoredBoolean(path); + + @Test + public void storedBoolean() { + assertFalse(storedBoolean.value()); + storedBoolean.set(context); + assertTrue(storedBoolean.value()); + storedBoolean.clear(context); + assertFalse(storedBoolean.value()); + } + + @Test + public void testCompatibility() throws IOException { + StoredInteger storedInteger = new StoredInteger(path); + assertFalse(storedBoolean.value()); + + storedInteger.write(context, 1); + assertTrue(storedBoolean.value()); + + storedInteger.write(context, 2); + assertTrue(storedBoolean.value()); + + storedInteger.write(context, 0); + assertFalse(storedBoolean.value()); + + Files.delete(path); + assertFalse(storedBoolean.value()); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/resources/expected.container.system.metrics.txt b/node-admin/src/test/resources/expected.container.system.metrics.txt index c44d72b395e..ec750798c98 100644 --- a/node-admin/src/test/resources/expected.container.system.metrics.txt +++ b/node-admin/src/test/resources/expected.container.system.metrics.txt @@ -1,83 +1,80 @@ s: { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "mem.limit": 4294967296, - "mem.used": 1073741824, - "mem_total.util": 40.808, - "mem_total.used": 1752707072, - "disk.used": 39625000000, "cpu.sys.util": 3.402, - "disk.util": 15.85, - "cpu.vcpus": 2.0, "cpu.util": 5.4, + "cpu.vcpus": 2.0, + "disk.limit": 250000000000, + "disk.used": 39625000000, + "disk.util": 15.85, + "mem.limit": 4294967296, + "mem.used": 1073741824, "mem.util": 25.0, - "disk.limit": 250000000000 + "mem_total.used": 1752707072, + "mem_total.util": 40.808 }, - "dimensions": { - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "role": "tenants", - "state": "active", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 } { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "interface": "eth0", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "net.out.bytes": 20303455, + "net.in.bytes": 19499270, "net.in.dropped": 4, + "net.in.errors": 55, + "net.out.bytes": 20303455, "net.out.dropped": 13, - "net.in.bytes": 19499270, - "net.out.errors": 3, - "net.in.errors": 55 + "net.out.errors": 3 }, - "dimensions": { - "role": "tenants", - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "state": "active", - "interface": "eth0", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 } { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "interface": "eth1", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "net.out.bytes": 54246745, + "net.in.bytes": 3245766, "net.in.dropped": 0, + "net.in.errors": 0, + "net.out.bytes": 54246745, "net.out.dropped": 0, - "net.in.bytes": 3245766, - "net.out.errors": 0, - "net.in.errors": 0 + "net.out.errors": 0 }, - "dimensions": { - "role": "tenants", - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "state": "active", - "interface": "eth1", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 }
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5de621634ea..a79ac908b9e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -3,25 +3,22 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.time.Duration; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +27,8 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; + /** * @author oyving */ @@ -56,8 +55,7 @@ public class MetricsReporter extends Maintainer { @Override public void maintain() { - LockedNodeList nodes = nodeRepository().list(() -> {}); // Ignore locking for the purposes of reporting metrics - + NodeList nodes = nodeRepository().list(); Map<HostName, List<ServiceInstance>> servicesByHost = serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName(); @@ -122,12 +120,8 @@ public class MetricsReporter extends Maintainer { metric.set("wantToRetire", node.status().wantToRetire() ? 1 : 0, context); metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); - metric.set("hardwareFailure", - node.status().hardwareFailureDescription().isPresent() ? 1 : 0, - context); - metric.set("hardwareDivergence", - node.status().hardwareDivergence().isPresent() ? 1 : 0, - context); + metric.set("hardwareFailure", node.status().hardwareFailureDescription().isPresent() ? 1 : 0, context); + metric.set("hardwareDivergence", node.status().hardwareDivergence().isPresent() ? 1 : 0, context); orchestrator.apply(new HostName(node.hostname())) .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0) @@ -183,63 +177,57 @@ public class MetricsReporter extends Maintainer { } private Metric.Context getContextAt(String... point) { - if (point.length % 2 != 0) { + if (point.length % 2 != 0) throw new IllegalArgumentException("Dimension specification comes in pairs"); - } Map<String, String> dimensions = new HashMap<>(); for (int i = 0; i < point.length; i += 2) { dimensions.put(point[i], point[i + 1]); } - Metric.Context context = contextMap.get(dimensions); - if (context != null) { - return context; - } - - context = metric.createContext(dimensions); - contextMap.put(dimensions, context); - return context; + return contextMap.computeIfAbsent(dimensions, metric::createContext); } - private void updateStateMetrics(LockedNodeList nodes) { - Map<Node.State, List<Node>> nodesByState = nodes.asList().stream() + private void updateStateMetrics(NodeList nodes) { + Map<Node.State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() .collect(Collectors.groupingBy(Node::state)); // Metrics pr state for (Node.State state : Node.State.values()) { - List<Node> nodesInState = nodesByState.getOrDefault(state, new ArrayList<>()); - long size = nodesInState.stream().filter(node -> node.type() == NodeType.tenant).count(); - metric.set("hostedVespa." + state.name() + "Hosts", size, null); + List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); } } - private void updateDockerMetrics(LockedNodeList nodes) { - // Capacity flavors for docker - DockerHostCapacity capacity = new DockerHostCapacity(nodes); - metric.set("hostedVespa.docker.totalCapacityCpu", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).vcpu(), null); - metric.set("hostedVespa.docker.totalCapacityMem", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).memoryGb(), null); - metric.set("hostedVespa.docker.totalCapacityDisk", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).diskGb(), null); - metric.set("hostedVespa.docker.freeCapacityCpu", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).vcpu(), null); - metric.set("hostedVespa.docker.freeCapacityMem", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).memoryGb(), null); - metric.set("hostedVespa.docker.freeCapacityDisk", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).diskGb(), null); - - List<Flavor> dockerFlavors = nodeRepository().getAvailableFlavors().getFlavors().stream() - .filter(f -> f.getType().equals(Flavor.Type.DOCKER_CONTAINER)) - .collect(Collectors.toList()); - for (Flavor flavor : dockerFlavors) { - Metric.Context context = getContextAt("flavor", flavor.name()); - metric.set("hostedVespa.docker.freeCapacityFlavor", - capacity.freeCapacityInFlavorEquivalence(flavor), context); - metric.set("hostedVespa.docker.hostsAvailableFlavor", - capacity.getNofHostsAvailableFor(flavor), context); - } + private void updateDockerMetrics(NodeList nodes) { + NodeResources totalCapacity = getCapacityTotal(nodes); + metric.set("hostedVespa.docker.totalCapacityCpu", totalCapacity.vcpu(), null); + metric.set("hostedVespa.docker.totalCapacityMem", totalCapacity.memoryGb(), null); + metric.set("hostedVespa.docker.totalCapacityDisk", totalCapacity.diskGb(), null); + + NodeResources totalFreeCapacity = getFreeCapacityTotal(nodes); + metric.set("hostedVespa.docker.freeCapacityCpu", totalFreeCapacity.vcpu(), null); + metric.set("hostedVespa.docker.freeCapacityMem", totalFreeCapacity.memoryGb(), null); + metric.set("hostedVespa.docker.freeCapacityDisk", totalFreeCapacity.diskGb(), null); + } + + private static NodeResources getCapacityTotal(NodeList nodes) { + return nodes.nodeType(NodeType.host).asList().stream() + .map(host -> host.flavor().resources()) + .map(resources -> resources.withDiskSpeed(any)) + .reduce(new NodeResources(0, 0, 0, any), NodeResources::add); } + private static NodeResources getFreeCapacityTotal(NodeList nodes) { + return nodes.nodeType(NodeType.host).asList().stream() + .map(n -> freeCapacityOf(nodes, n)) + .map(resources -> resources.withDiskSpeed(any)) + .reduce(new NodeResources(0, 0, 0, any), NodeResources::add); + } + + private static NodeResources freeCapacityOf(NodeList nodes, Node dockerHost) { + return nodes.childrenOf(dockerHost).asList().stream() + .map(node -> node.flavor().resources().withDiskSpeed(any)) + .reduce(dockerHost.flavor().resources().withDiskSpeed(any), NodeResources::subtract); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index a4b915a6128..fd2294c1b5d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -27,6 +27,13 @@ import java.util.function.Function; */ public class LoadBalancerSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String idField = "id"; private static final String hostnameField = "hostname"; private static final String dnsZoneField = "dnsZone"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index d38a6e5031c..424889caf72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -44,12 +44,12 @@ import java.util.function.UnaryOperator; */ public class NodeSerializer { - // WARNING: Since there are multiple config servers in a cluster and they upgrade one by one - // (and rewrite all nodes on startup), - // changes to the serialized format must be made such that what is serialized on version N+1 - // can be read by version N: + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: // - ADDING FIELDS: Always ok // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. /** The configured node flavors */ private final NodeFlavors flavors; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index feccfb430e6..a5969dc69cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -20,25 +19,15 @@ import java.util.Objects; public class DockerHostCapacity { private final LockedNodeList allNodes; + private final HostResourcesCalculator hostResourcesCalculator; - public DockerHostCapacity(LockedNodeList allNodes) { + DockerHostCapacity(LockedNodeList allNodes, HostResourcesCalculator hostResourcesCalculator) { this.allNodes = Objects.requireNonNull(allNodes, "allNodes must be non-null"); - } - - /** - * Compare hosts on free capacity. - * Used in prioritizing hosts for allocation in <b>descending</b> order. - */ - int compare(Node hostA, Node hostB) { - int result = compare(freeCapacityOf(hostB, false), freeCapacityOf(hostA, false)); - if (result != 0) return result; - - // If resources are equal we want to assign to the one with the most IPaddresses free - return freeIPs(hostB) - freeIPs(hostA); + this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } int compareWithoutInactive(Node hostA, Node hostB) { - int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); + int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; // If resources are equal we want to assign to the one with the most IPaddresses free @@ -64,79 +53,29 @@ public class DockerHostCapacity { return dockerHost.ipAddressPool().findUnused(allNodes).size(); } - /** Return total free capacity for a given disk speed (or for any disk speed) */ - public NodeResources getFreeCapacityTotal(NodeResources.DiskSpeed speed) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> speed == NodeResources.DiskSpeed.any || n.flavor().resources().diskSpeed() == speed) - .map(n -> freeCapacityOf(n, false)) - .map(resources -> resources.withDiskSpeed(speed)) // Set speed to 'any' if necessary - .reduce(new NodeResources(0, 0, 0, speed), NodeResources::add); - } - - /** Return total capacity for a given disk speed (or for any disk speed) */ - public NodeResources getCapacityTotal(NodeResources.DiskSpeed speed) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> speed == NodeResources.DiskSpeed.any || n.flavor().resources().diskSpeed() == speed) - .map(host -> host.flavor().resources()) - .map(resources -> resources.withDiskSpeed(speed)) // Set speed to 'any' if necessary - .reduce(new NodeResources(0, 0, 0, speed), NodeResources::add); - } - - public int freeCapacityInFlavorEquivalence(Flavor flavor) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .map(n -> canFitNumberOf(n, flavor)) - .reduce(0, (a, b) -> a + b); - } - - public long getNofHostsAvailableFor(Flavor flavor) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> hasCapacity(n, flavor.resources())) - .count(); - } - - private int canFitNumberOf(Node node, Flavor flavor) { - NodeResources freeCapacity = freeCapacityOf(node, false); - int capacityFactor = freeCapacityInFlavorEquivalence(freeCapacity, flavor); - int ips = freeIPs(node); - return Math.min(capacityFactor, ips); - } - - int freeCapacityInFlavorEquivalence(NodeResources freeCapacity, Flavor flavor) { - if ( ! freeCapacity.satisfies(flavor.resources())) return 0; - - double cpuFactor = Math.floor(freeCapacity.vcpu() / flavor.getMinCpuCores()); - double memoryFactor = Math.floor(freeCapacity.memoryGb() / flavor.getMinMainMemoryAvailableGb()); - double diskFactor = Math.floor(freeCapacity.diskGb() / flavor.getMinDiskAvailableGb()); - - return (int) Math.min(Math.min(memoryFactor, cpuFactor), diskFactor); - } - /** * Calculate the remaining capacity for the dockerHost. * * @param dockerHost The host to find free capacity of. * @return A default (empty) capacity if not a docker host, otherwise the free/unallocated/rest capacity */ - public NodeResources freeCapacityOf(Node dockerHost, boolean includeInactive) { + NodeResources freeCapacityOf(Node dockerHost, boolean excludeInactive) { // Only hosts have free capacity - if ( ! dockerHost.type().equals(NodeType.host)) return new NodeResources(0, 0, 0); + if (dockerHost.type() != NodeType.host) return new NodeResources(0, 0, 0); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(dockerHost.flavor().resources()); // Subtract used resources without taking disk speed into account since existing allocations grandfathered in // may not use reflect the actual disk speed (as of May 2019). This (the 3 diskSpeed assignments below) // can be removed when all node allocations accurately reflect the true host disk speed return allNodes.childrenOf(dockerHost).asList().stream() - .filter(node -> !(includeInactive && isInactiveOrRetired(node))) + .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) .map(node -> node.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any)) - .reduce(dockerHost.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any), NodeResources::subtract) + .reduce(hostResources.withDiskSpeed(NodeResources.DiskSpeed.any), NodeResources::subtract) .withDiskSpeed(dockerHost.flavor().resources().diskSpeed()); } - private boolean isInactiveOrRetired(Node node) { - if (node.state().equals(Node.State.inactive)) return true; + private static boolean isInactiveOrRetired(Node node) { + if (node.state() == Node.State.inactive) return true; if (node.allocation().isPresent() && node.allocation().get().membership().retired()) return true; return false; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index eb72e1a2081..05915b82bae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -9,6 +10,8 @@ import java.util.Optional; * @author freva */ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { + private final HostResourcesCalculator hostResourcesCalculator = new NoopHostResourcesCalculator(); + @Override public Optional<LoadBalancerService> getLoadBalancerService() { return Optional.empty(); @@ -18,4 +21,17 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public Optional<HostProvisioner> getHostProvisioner() { return Optional.empty(); } + + @Override + public HostResourcesCalculator getHostResourcesCalculator() { + return hostResourcesCalculator; + } + + public static class NoopHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources availableCapacityOf(NodeResources hostResources) { + return hostResources; + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index ff412c6c99b..30d39a4717f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -25,12 +25,14 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabledFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, - BooleanFlag dynamicProvisioningEnabledFlag) { + HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabledFlag) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabledFlag = dynamicProvisioningEnabledFlag; } @@ -65,7 +67,8 @@ public class GroupPreparer { LockedNodeList nodeList = nodeRepository.list(allocationLock); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, spareCount, nodeRepository.nameResolver(), - nodeRepository.getAvailableFlavors()); + nodeRepository.getAvailableFlavors(), + hostResourcesCalculator); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java new file mode 100644 index 00000000000..50aeb464e9f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.NodeResources; + +/** + * @author freva + */ +public interface HostResourcesCalculator { + + /** Calculates the resources that are reserved for host level processes and returns the remainder. */ + NodeResources availableCapacityOf(NodeResources hostResources); +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 294ba12497a..6b27662448c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -51,9 +51,9 @@ class NodePrioritizer { private final Set<Node> spareHosts; NodePrioritizer(LockedNodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int spares, NameResolver nameResolver, NodeFlavors flavors) { + int spares, NameResolver nameResolver, NodeFlavors flavors, HostResourcesCalculator hostResourcesCalculator) { this.allNodes = allNodes; - this.capacity = new DockerHostCapacity(allNodes); + this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.appId = appId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index c3281253b6b..21bfc1b6886 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -6,10 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -64,8 +63,9 @@ public class NodeRepositoryProvisioner implements Provisioner { this.capacityPolicies = new CapacityPolicies(zone, flavors); this.zone = zone; this.preparer = new Preparer(nodeRepository, - zone.environment().equals(Environment.prod) ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, + zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, provisionServiceProvider.getHostProvisioner(), + provisionServiceProvider.getHostResourcesCalculator(), Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource)); this.activator = new Activator(nodeRepository); this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> @@ -140,7 +140,7 @@ public class NodeRepositoryProvisioner implements Provisioner { log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), - Collections.emptyList(), + List.of(), Optional.of(node.flavor()), Optional.of(nodeAllocation.membership()), node.status().vespaVersion(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 75960e4416a..ca958f15c69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -27,10 +27,10 @@ class Preparer { private final int spareCount; public Preparer(NodeRepository nodeRepository, int spareCount, Optional<HostProvisioner> hostProvisioner, - BooleanFlag dynamicProvisioningEnabled) { + HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabled) { this.nodeRepository = nodeRepository; this.spareCount = spareCount; - this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, dynamicProvisioningEnabled); + this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, hostResourcesCalculator, dynamicProvisioningEnabled); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index ca9e629e1ea..a86bd581516 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -15,4 +15,6 @@ public interface ProvisionServiceProvider { Optional<LoadBalancerService> getLoadBalancerService(); Optional<HostProvisioner> getHostProvisioner(); + + HostResourcesCalculator getHostResourcesCalculator(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index ea48d8a6b47..0d5950fe33a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; +import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.util.Optional; @@ -16,6 +18,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { private final Optional<LoadBalancerService> loadBalancerService; private final Optional<HostProvisioner> hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; @Inject public MockProvisionServiceProvider() { @@ -23,8 +26,14 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner) { + this(loadBalancerService, hostProvisioner, new EmptyProvisionServiceProvider.NoopHostResourcesCalculator()); + } + + public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator) { this.loadBalancerService = Optional.ofNullable(loadBalancerService); this.hostProvisioner = Optional.ofNullable(hostProvisioner); + this.hostResourcesCalculator = hostResourcesCalculator; } @Override @@ -36,4 +45,9 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { public Optional<HostProvisioner> getHostProvisioner() { return hostProvisioner; } + + @Override + public HostResourcesCalculator getHostResourcesCalculator() { + return hostResourcesCalculator; + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index e1a0f478b86..4eba95cb817 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -30,7 +30,6 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,19 +55,19 @@ public class MetricsReporterTest { DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); - nodeRepository.addNodes(Collections.singletonList(node)); + nodeRepository.addNodes(List.of(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); - nodeRepository.addNodes(Collections.singletonList(hostNode)); + nodeRepository.addNodes(List.of(hostNode)); Map<String, Number> expectedMetrics = new HashMap<>(); - expectedMetrics.put("hostedVespa.provisionedHosts", 1L); - expectedMetrics.put("hostedVespa.parkedHosts", 0L); - expectedMetrics.put("hostedVespa.readyHosts", 0L); - expectedMetrics.put("hostedVespa.reservedHosts", 0L); - expectedMetrics.put("hostedVespa.activeHosts", 0L); - expectedMetrics.put("hostedVespa.inactiveHosts", 0L); - expectedMetrics.put("hostedVespa.dirtyHosts", 0L); - expectedMetrics.put("hostedVespa.failedHosts", 0L); + expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.parkedHosts", 0); + expectedMetrics.put("hostedVespa.readyHosts", 0); + expectedMetrics.put("hostedVespa.reservedHosts", 0); + expectedMetrics.put("hostedVespa.activeHosts", 0); + expectedMetrics.put("hostedVespa.inactiveHosts", 0); + expectedMetrics.put("hostedVespa.dirtyHosts", 0); + expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -92,7 +91,7 @@ public class MetricsReporterTest { when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap()); + when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -120,18 +119,18 @@ public class MetricsReporterTest { // Allow 4 containers Set<String> ipAddressPool = ImmutableSet.of("::2", "::3", "::4", "::5"); - Node dockerHost = Node.create("openStackId1", Collections.singleton("::1"), ipAddressPool, "dockerHost", + Node dockerHost = Node.create("openStackId1", Set.of("::1"), ipAddressPool, "dockerHost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - nodeRepository.addNodes(Collections.singletonList(dockerHost)); + nodeRepository.addNodes(List.of(dockerHost)); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); - Node container1 = Node.createDockerNode(Collections.singleton("::2"), Collections.emptySet(), "container1", + Node container1 = Node.createDockerNode(Set.of("::2"), Set.of(), "container1", Optional.of("dockerHost"), new NodeResources(1, 3, 2), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1")).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation())); - Node container2 = Node.createDockerNode(Collections.singleton("::3"), Collections.emptySet(), "container2", + Node container2 = Node.createDockerNode(Set.of("::3"), Set.of(), "container2", Optional.of("dockerHost"), new NodeResources(2, 4, 4), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2")).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation())); @@ -141,7 +140,7 @@ public class MetricsReporterTest { when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap()); + when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -154,8 +153,8 @@ public class MetricsReporterTest { ); metricsReporter.maintain(); - assertEquals(0L, metric.values.get("hostedVespa.readyHosts")); /** Only tenants counts **/ - assertEquals(2L, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); assertEquals(12.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(10.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); @@ -164,27 +163,6 @@ public class MetricsReporterTest { assertEquals(6.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); assertEquals(3.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); assertEquals(4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); - - assertContext(metric, "hostedVespa.docker.freeCapacityFlavor", 1, 0); - assertContext(metric, "hostedVespa.docker.hostsAvailableFlavor", 1l, 0l); - } - - private void assertContext(TestMetric metric, String key, Number dockerValue, Number docker2Value) { - List<Metric.Context> freeCapacityFlavor = metric.context.get(key); - assertEquals(freeCapacityFlavor.size(), 2); - - // Get the value for the two flavors - TestMetric.TestContext contextFlavorDocker = (TestMetric.TestContext)freeCapacityFlavor.get(0); - TestMetric.TestContext contextFlavorDocker2 = (TestMetric.TestContext)freeCapacityFlavor.get(1); - if (!contextFlavorDocker.properties.containsValue("docker")) { - TestMetric.TestContext temp = contextFlavorDocker; - contextFlavorDocker = contextFlavorDocker2; - contextFlavorDocker2 = temp; - } - - assertEquals(dockerValue, contextFlavorDocker.value); - assertEquals(docker2Value, contextFlavorDocker2.value); - } private ApplicationId app(String tenant) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index d5d4bcc954d..60a31f1c804 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -11,7 +11,6 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -20,12 +19,16 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; /** * @author smorgrav */ public class DockerHostCapacityTest { + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private DockerHostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; @@ -33,41 +36,32 @@ public class DockerHostCapacityTest { @Before public void setup() { + doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); flavorDocker = nodeFlavors.getFlavorOrThrow("docker"); flavorDocker2 = nodeFlavors.getFlavorOrThrow("docker2"); // Create three docker hosts - host1 = Node.create("host1", Collections.singleton("::1"), generateIPs(2, 4), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host2 = Node.create("host2", Collections.singleton("::11"), generateIPs(12, 3), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host3 = Node.create("host3", Collections.singleton("::21"), generateIPs(22, 1), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host1 = Node.create("host1", Set.of("::1"), generateIPs(2, 4), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host2 = Node.create("host2", Set.of("::11"), generateIPs(12, 3), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host3 = Node.create("host3", Set.of("::21"), generateIPs(22, 1), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); // Add two containers to host1 - var nodeA = Node.create("nodeA", Collections.singleton("::2"), Collections.emptySet(), "nodeA", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); - var nodeB = Node.create("nodeB", Collections.singleton("::3"), Collections.emptySet(), "nodeB", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeA = Node.create("nodeA", Set.of("::2"), Set.of(), "nodeA", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeB = Node.create("nodeB", Set.of("::3"), Set.of(), "nodeB", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); // Add two containers to host 2 (same as host 1) - var nodeC = Node.create("nodeC", Collections.singleton("::12"), Collections.emptySet(), "nodeC", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); - var nodeD = Node.create("nodeD", Collections.singleton("::13"), Collections.emptySet(), "nodeD", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeC = Node.create("nodeC", Set.of("::12"), Set.of(), "nodeC", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeD = Node.create("nodeD", Set.of("::13"), Set.of(), "nodeD", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); // Add a larger container to host3 - var nodeE = Node.create("nodeE", Collections.singleton("::22"), Collections.emptySet(), "nodeE", Optional.of("host3"), Optional.empty(), flavorDocker2, NodeType.tenant); + var nodeE = Node.create("nodeE", Set.of("::22"), Set.of(), "nodeE", Optional.of("host3"), Optional.empty(), flavorDocker2, NodeType.tenant); // init docker host capacity - nodes = new ArrayList<>(); - Collections.addAll(nodes, host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {})); - } - - @Test - public void compare_used_to_sort_in_decending_order() { - assertEquals(host1, nodes.get(0)); // Make sure it is unsorted here - - Collections.sort(nodes, capacity::compare); - assertEquals(host3, nodes.get(0)); - assertEquals(host1, nodes.get(1)); - assertEquals(host2, nodes.get(2)); + nodes = new ArrayList<>(List.of(host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE)); + capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); } @Test @@ -80,10 +74,10 @@ public class DockerHostCapacityTest { assertFalse(capacity.hasCapacity(host3, flavorDocker2.resources())); // No ip available // Add a new node to host1 to deplete the memory resource - Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), + Node nodeF = Node.create("nodeF", Set.of("::6"), Set.of(), "nodeF", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); nodes.add(nodeF); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {})); + capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); assertFalse(capacity.hasCapacity(host1, flavorDocker.resources())); assertFalse(capacity.hasCapacity(host1, flavorDocker2.resources())); } @@ -96,31 +90,17 @@ public class DockerHostCapacityTest { } @Test - public void getCapacityTotal() { - NodeResources total = capacity.getCapacityTotal(NodeResources.DiskSpeed.any); - assertEquals(21.0, total.vcpu(), 0.1); - assertEquals(30.0, total.memoryGb(), 0.1); - assertEquals(36.0, total.diskGb(), 0.1); - } - - @Test - public void getFreeCapacityTotal() { - NodeResources totalFree = capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any); - assertEquals(15.0, totalFree.vcpu(), 0.1); - assertEquals(14.0, totalFree.memoryGb(), 0.1); - assertEquals(24.0, totalFree.diskGb(), 0.1); - } + public void freeCapacityOf() { + assertEquals(new NodeResources(5, 4, 8), capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(5, 6, 8), capacity.freeCapacityOf(host3, false)); - @Test - public void freeCapacityInFlavorEquivalence() { - assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker)); - assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker2)); - } + doAnswer(invocation -> { + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + return totalHostResources.subtract(new NodeResources(1, 2, 3, NodeResources.DiskSpeed.any)); + }).when(hostResourcesCalculator).availableCapacityOf(any()); - @Test - public void getNofHostsAvailableFor() { - assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker)); - assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker2)); + assertEquals(new NodeResources(4, 2, 5), capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(4, 4, 5), capacity.freeCapacityOf(host3, false)); } private Set<String> generateIPs(int start, int count) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 664430a2de8..50e19e15da5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -37,7 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -221,17 +219,13 @@ public class DynamicDockerAllocationTest { List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); tester.activate(application1, ImmutableSet.copyOf(hosts)); - DockerHostCapacity capacity = new DockerHostCapacity(new LockedNodeList(tester.nodeRepository().getNodes(Node.State.values()), () -> {})); - assertThat(capacity.freeCapacityInFlavorEquivalence(new Flavor(flavor)), greaterThan(0)); - List<Node> initialSpareCapacity = findSpareCapacity(tester); assertThat(initialSpareCapacity.size(), is(2)); try { hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); - } catch (OutOfCapacityException e) { - } + } catch (OutOfCapacityException ignored) { } tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); diff --git a/parent/pom.xml b/parent/pom.xml index d8ae9a35261..1855553bc20 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -92,7 +92,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> - <version>3.1.0</version> + <version>3.1.1</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -441,9 +441,14 @@ <version>${asm.version}</version> </dependency> <dependency> - <groupId>com.goldmansachs</groupId> - <artifactId>gs-collections</artifactId> - <version>6.1.0</version> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections</artifactId> + <version>9.2.0</version> + </dependency> + <dependency> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections-api</artifactId> + <version>9.2.0</version> </dependency> <dependency> <groupId>com.infradna.tool</groupId> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index fa1bee2fe28..778ede93366 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -34,8 +34,12 @@ <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>com.goldmansachs</groupId> - <artifactId>gs-collections</artifactId> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections-api</artifactId> </dependency> <dependency> <groupId>io.airlift</groupId> diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java index 91599da5483..9356e86aa2f 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java @@ -2,9 +2,9 @@ package com.yahoo.search.predicate.index; import com.google.common.collect.MinMaxPriorityQueue; -import com.gs.collections.api.tuple.primitive.ObjectLongPair; -import com.gs.collections.impl.map.mutable.primitive.ObjectIntHashMap; -import com.gs.collections.impl.map.mutable.primitive.ObjectLongHashMap; +import org.eclipse.collections.api.tuple.primitive.ObjectLongPair; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; import java.util.ArrayList; import java.util.Arrays; @@ -119,7 +119,7 @@ public class CachedPostingListCounter { private static class Entry implements Comparable<Entry> { public final int[] docIds; - public final double cost; + final double cost; private Entry(int[] docIds, long frequency) { this.docIds = docIds; diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/SimpleIndex.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/SimpleIndex.java index 64583273e77..3e1ed7ad9e4 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/SimpleIndex.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/SimpleIndex.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.index; -import com.gs.collections.api.map.primitive.LongObjectMap; -import com.gs.collections.api.tuple.primitive.LongObjectPair; -import com.gs.collections.impl.map.mutable.primitive.LongObjectHashMap; import com.yahoo.search.predicate.serialization.SerializationHelper; +import org.eclipse.collections.api.map.primitive.LongObjectMap; +import org.eclipse.collections.api.tuple.primitive.LongObjectPair; +import org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap; import java.io.DataInputStream; import java.io.DataOutputStream; diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndex.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndex.java index 5a100ea9cf5..d062af43f22 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndex.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndex.java @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.index.conjunction; -import com.gs.collections.api.map.primitive.IntObjectMap; -import com.gs.collections.api.map.primitive.LongObjectMap; -import com.gs.collections.api.tuple.primitive.IntObjectPair; -import com.gs.collections.api.tuple.primitive.LongObjectPair; -import com.gs.collections.impl.map.mutable.primitive.IntObjectHashMap; -import com.gs.collections.impl.map.mutable.primitive.LongObjectHashMap; import com.yahoo.document.predicate.FeatureConjunction; import com.yahoo.search.predicate.PredicateQuery; import com.yahoo.search.predicate.SubqueryBitmap; import com.yahoo.search.predicate.serialization.SerializationHelper; import com.yahoo.search.predicate.utils.PrimitiveArraySorter; +import org.eclipse.collections.api.map.primitive.IntObjectMap; +import org.eclipse.collections.api.map.primitive.LongObjectMap; +import org.eclipse.collections.api.tuple.primitive.IntObjectPair; +import org.eclipse.collections.api.tuple.primitive.LongObjectPair; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap; import java.io.DataInputStream; import java.io.DataOutputStream; diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexBuilder.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexBuilder.java index a6a03177018..8e3261a4cf8 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexBuilder.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexBuilder.java @@ -3,9 +3,9 @@ package com.yahoo.search.predicate.index.conjunction; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; -import com.gs.collections.api.map.primitive.IntObjectMap; -import com.gs.collections.impl.map.mutable.primitive.IntObjectHashMap; -import com.gs.collections.impl.map.mutable.primitive.LongObjectHashMap; +import org.eclipse.collections.api.map.primitive.IntObjectMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap; import java.util.ArrayList; import java.util.HashMap; diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java index a3dfd00149c..31777959704 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.index; -import com.gs.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.apache.commons.lang.ArrayUtils; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.junit.Test; import java.util.ArrayList; diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 2a29847a634..967d8bfd0aa 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -33,6 +33,9 @@ #include <vespa/searchcore/proton/matching/match_params.h> #include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/searchcore/proton/matching/match_context.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP("matching_test"); @@ -55,6 +58,10 @@ using search::index::schema::DataType; using storage::spi::Timestamp; using search::fef::indexproperties::hitcollector::HeapSize; +using vespalib::nbostream; +using vespalib::eval::TensorSpec; +using vespalib::tensor::DefaultTensorEngine; + void inject_match_phase_limiting(Properties &setup, const vespalib::string &attribute, size_t max_hits, bool descending) { Properties cfg; @@ -134,7 +141,10 @@ struct MyWorld { config.add(indexproperties::hitcollector::HeapSize::NAME, (vespalib::asciistream() << heapSize).str()); config.add(indexproperties::hitcollector::ArraySize::NAME, (vespalib::asciistream() << arraySize).str()); config.add(indexproperties::summary::Feature::NAME, "attribute(a1)"); + config.add(indexproperties::summary::Feature::NAME, "rankingExpression(\"reduce(tensor(x[3])(x),sum)\")"); + config.add(indexproperties::summary::Feature::NAME, "rankingExpression(\"tensor(x[3])(x)\")"); config.add(indexproperties::summary::Feature::NAME, "value(100)"); + config.add(indexproperties::dump::IgnoreDefaultFeatures::NAME, "true"); config.add(indexproperties::dump::Feature::NAME, "attribute(a2)"); @@ -630,21 +640,35 @@ TEST("require that summary features are filled") { world.basicResults(); DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo"); FeatureSet::SP fs = world.getSummaryFeatures(req); - const feature_t * f = NULL; - EXPECT_EQUAL(2u, fs->numFeatures()); + const FeatureSet::Value * f = NULL; + EXPECT_EQUAL(4u, fs->numFeatures()); EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("value(100)", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]); + EXPECT_EQUAL("value(100)", fs->getNames()[3]); EXPECT_EQUAL(2u, fs->numDocs()); f = fs->getFeaturesByDocId(10); EXPECT_TRUE(f != NULL); - EXPECT_EQUAL(10, f[0]); - EXPECT_EQUAL(100, f[1]); + EXPECT_EQUAL(10, f[0].as_double()); + EXPECT_EQUAL(100, f[3].as_double()); f = fs->getFeaturesByDocId(15); EXPECT_TRUE(f == NULL); f = fs->getFeaturesByDocId(30); EXPECT_TRUE(f != NULL); - EXPECT_EQUAL(30, f[0]); - EXPECT_EQUAL(100, f[1]); + EXPECT_EQUAL(30, f[0].as_double()); + EXPECT_EQUAL(100, f[3].as_double()); + EXPECT_TRUE(f[1].is_double()); + EXPECT_TRUE(!f[1].is_data()); + EXPECT_EQUAL(f[1].as_double(), 3.0); // 0 + 1 + 2 + EXPECT_TRUE(!f[2].is_double()); + EXPECT_TRUE(f[2].is_data()); + { + auto &engine = DefaultTensorEngine::ref(); + nbostream buf(f[2].as_data().data, f[2].as_data().size); + auto actual = engine.to_spec(*engine.decode(buf)); + auto expect = TensorSpec("tensor(x[3])").add({{"x", 0}}, 0).add({{"x", 1}}, 1).add({{"x", 2}}, 2); + EXPECT_EQUAL(actual, expect); + } } TEST("require that rank features are filled") { @@ -653,18 +677,18 @@ TEST("require that rank features are filled") { world.basicResults(); DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo"); FeatureSet::SP fs = world.getRankFeatures(req); - const feature_t * f = NULL; + const FeatureSet::Value * f = NULL; EXPECT_EQUAL(1u, fs->numFeatures()); EXPECT_EQUAL("attribute(a2)", fs->getNames()[0]); EXPECT_EQUAL(2u, fs->numDocs()); f = fs->getFeaturesByDocId(10); EXPECT_TRUE(f != NULL); - EXPECT_EQUAL(20, f[0]); + EXPECT_EQUAL(20, f[0].as_double()); f = fs->getFeaturesByDocId(15); EXPECT_TRUE(f == NULL); f = fs->getFeaturesByDocId(30); EXPECT_TRUE(f != NULL); - EXPECT_EQUAL(60, f[0]); + EXPECT_EQUAL(60, f[0].as_double()); } TEST("require that search session can be cached") { @@ -699,25 +723,29 @@ TEST("require that getSummaryFeatures can use cached query setup") { docsum_request->hits.back().docid = 30; FeatureSet::SP fs = world.getSummaryFeatures(docsum_request); - ASSERT_EQUAL(2u, fs->numFeatures()); + ASSERT_EQUAL(4u, fs->numFeatures()); EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("value(100)", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]); + EXPECT_EQUAL("value(100)", fs->getNames()[3]); ASSERT_EQUAL(1u, fs->numDocs()); - const feature_t *f = fs->getFeaturesByDocId(30); + const auto *f = fs->getFeaturesByDocId(30); ASSERT_TRUE(f); - EXPECT_EQUAL(30, f[0]); - EXPECT_EQUAL(100, f[1]); + EXPECT_EQUAL(30, f[0].as_double()); + EXPECT_EQUAL(100, f[3].as_double()); // getSummaryFeatures can be called multiple times. fs = world.getSummaryFeatures(docsum_request); - ASSERT_EQUAL(2u, fs->numFeatures()); + ASSERT_EQUAL(4u, fs->numFeatures()); EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("value(100)", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]); + EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]); + EXPECT_EQUAL("value(100)", fs->getNames()[3]); ASSERT_EQUAL(1u, fs->numDocs()); f = fs->getFeaturesByDocId(30); ASSERT_TRUE(f); - EXPECT_EQUAL(30, f[0]); - EXPECT_EQUAL(100, f[1]); + EXPECT_EQUAL(30, f[0].as_double()); + EXPECT_EQUAL(100, f[3].as_double()); } TEST("require that getSummaryFeatures prefers cached query setup") { @@ -733,7 +761,7 @@ TEST("require that getSummaryFeatures prefers cached query setup") { req->sessionId = request->sessionId; req->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true"); FeatureSet::SP fs = world.getSummaryFeatures(req); - EXPECT_EQUAL(2u, fs->numFeatures()); + EXPECT_EQUAL(4u, fs->numFeatures()); ASSERT_EQUAL(0u, fs->numDocs()); // "spread" has no hits // Empty cache @@ -742,7 +770,7 @@ TEST("require that getSummaryFeatures prefers cached query setup") { world.sessionManager->pruneTimedOutSessions(pruneTime); fs = world.getSummaryFeatures(req); - EXPECT_EQUAL(2u, fs->numFeatures()); + EXPECT_EQUAL(4u, fs->numFeatures()); ASSERT_EQUAL(2u, fs->numDocs()); // "foo" has two hits } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index d37072f5da2..9b535be19b7 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -1079,8 +1079,12 @@ DocumentMetaStore::foreach(const search::IGidToLidMapperVisitor &visitor) const } // namespace proton -template class search::btree:: -BTreeIterator<proton::DocumentMetaStore::DocId, - search::btree::BTreeNoLeafData, - search::btree::NoAggregated, - const proton::DocumentMetaStore::KeyComp &>; +namespace search::btree { + +template class BTreeIteratorBase<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS, BTreeDefaultTraits::PATH_SIZE>; + +template class BTreeConstIterator<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, const proton::DocumentMetaStore::KeyComp &>; + +template class BTreeIterator<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, const proton::DocumentMetaStore::KeyComp &>; + +} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index efac1158cfb..27c1c97556c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -268,8 +268,12 @@ public: } -extern template class search::btree:: -BTreeIterator<proton::DocumentMetaStore::DocId, - search::btree::BTreeNoLeafData, - search::btree::NoAggregated, - const proton::DocumentMetaStore::KeyComp &>; +namespace search::btree { + +extern template class BTreeIteratorBase<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS, BTreeDefaultTraits::PATH_SIZE>; + +extern template class BTreeConstIterator<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, const proton::DocumentMetaStore::KeyComp &>; + +extern template class BTreeIterator<proton::DocumentMetaStore::DocId, BTreeNoLeafData, NoAggregated, const proton::DocumentMetaStore::KeyComp &>; + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index ca159228f25..c2262cc51e5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -11,6 +11,9 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/data/slime/inject.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/eval/eval/tensor.h> +#include <vespa/eval/eval/tensor_engine.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_master"); @@ -127,7 +130,7 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &mtf, RankProgram &rankProgram = matchTools->rank_program(); std::vector<vespalib::string> featureNames; - FeatureResolver resolver(rankProgram.get_seeds()); + FeatureResolver resolver(rankProgram.get_seeds(false)); featureNames.reserve(resolver.num_features()); for (size_t i = 0; i < resolver.num_features(); ++i) { featureNames.emplace_back(resolver.name_of(i)); @@ -144,9 +147,20 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &mtf, if (search.seek(docs[i])) { uint32_t docId = search.getDocId(); search.unpack(docId); - search::feature_t * f = fs.getFeaturesByIndex(fs.addDocId(docId)); + auto * f = fs.getFeaturesByIndex(fs.addDocId(docId)); for (uint32_t j = 0; j < featureNames.size(); ++j) { - f[j] = resolver.resolve(j).as_number(docId); + if (resolver.is_object(j)) { + auto obj = resolver.resolve(j).as_object(docId); + if (const auto *tensor = obj.get().as_tensor()) { + vespalib::nbostream buf; + tensor->engine().encode(*tensor, buf); + f[j].set_data(vespalib::Memory(buf.peek(), buf.size())); + } else { + f[j].set_double(obj.get().as_double()); + } + } else { + f[j].set_double(resolver.resolve(j).as_number(docId)); + } } } else { LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]); diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 6b6abd956fc..5476f0f8e66 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("enumstore_test"); #include <vespa/vespalib/testkit/testapp.h> -//#define LOG_ENUM_STORE #include <vespa/searchlib/attribute/enumstore.hpp> #include <limits> #include <string> #include <iostream> +#include <vespa/log/log.h> +LOG_SETUP("enumstore_test"); + namespace search { size_t enumStoreAlign(size_t size) diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp index 0dc43898441..646b2b818b3 100644 --- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp +++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp @@ -5,6 +5,7 @@ LOG_SETUP("summaryfeatures_test"); #include <vespa/searchlib/common/featureset.h> using namespace search; +using vespalib::Memory; TEST_SETUP(Test); @@ -43,34 +44,34 @@ Test::Main() EXPECT_EQUAL(sf.addDocId(40), 3u); EXPECT_EQUAL(sf.addDocId(50), 4u); EXPECT_EQUAL(sf.numDocs(), 5u); - feature_t *f; - const feature_t *cf; + FeatureSet::Value *f; + const FeatureSet::Value *cf; f = sf.getFeaturesByIndex(0); ASSERT_TRUE(f != 0); - f[0] = 11.0; - f[1] = 12.0; - f[2] = 13.0; + f[0].set_double(11.0); + f[1].set_double(12.0); + f[2].set_double(13.0); f = sf.getFeaturesByIndex(1); ASSERT_TRUE(f != 0); - f[0] = 21.0; - f[1] = 22.0; - f[2] = 23.0; + f[0].set_double(21.0); + f[1].set_double(22.0); + f[2].set_double(23.0); f = sf.getFeaturesByIndex(2); ASSERT_TRUE(f != 0); - f[0] = 31.0; - f[1] = 32.0; - f[2] = 33.0; + f[0].set_double(31.0); + f[1].set_double(32.0); + f[2].set_double(33.0); f = sf.getFeaturesByIndex(3); ASSERT_TRUE(f != 0); - f[0] = 41.0; - f[1] = 42.0; - f[2] = 43.0; + f[0].set_double(41.0); + f[1].set_data(Memory("test", 4)); + f[2].set_double(43.0); f = sf.getFeaturesByIndex(4); ASSERT_TRUE(f != 0); - f[0] = 51.0; - f[1] = 52.0; - f[2] = 53.0; - EXPECT_TRUE(sf.getFeaturesByIndex(5) == 0); + f[0].set_double(51.0); + f[1].set_double(52.0); + f[2].set_double(53.0); + EXPECT_TRUE(sf.getFeaturesByIndex(5) == nullptr); { std::vector<uint32_t> docs; EXPECT_TRUE(sf.contains(docs)); @@ -107,45 +108,49 @@ Test::Main() } { cf = sf.getFeaturesByDocId(10); - ASSERT_TRUE(cf != 0); - EXPECT_APPROX(cf[0], 11.0, 10e-6); - EXPECT_APPROX(cf[1], 12.0, 10e-6); - EXPECT_APPROX(cf[2], 13.0, 10e-6); + ASSERT_TRUE(cf != nullptr); + EXPECT_APPROX(cf[0].as_double(), 11.0, 10e-6); + EXPECT_APPROX(cf[1].as_double(), 12.0, 10e-6); + EXPECT_APPROX(cf[2].as_double(), 13.0, 10e-6); } { cf = sf.getFeaturesByDocId(20); - ASSERT_TRUE(cf != 0); - EXPECT_APPROX(cf[0], 21.0, 10e-6); - EXPECT_APPROX(cf[1], 22.0, 10e-6); - EXPECT_APPROX(cf[2], 23.0, 10e-6); + ASSERT_TRUE(cf != nullptr); + EXPECT_APPROX(cf[0].as_double(), 21.0, 10e-6); + EXPECT_APPROX(cf[1].as_double(), 22.0, 10e-6); + EXPECT_APPROX(cf[2].as_double(), 23.0, 10e-6); } { cf = sf.getFeaturesByDocId(30); - ASSERT_TRUE(cf != 0); - EXPECT_APPROX(cf[0], 31.0, 10e-6); - EXPECT_APPROX(cf[1], 32.0, 10e-6); - EXPECT_APPROX(cf[2], 33.0, 10e-6); + ASSERT_TRUE(cf != nullptr); + EXPECT_APPROX(cf[0].as_double(), 31.0, 10e-6); + EXPECT_APPROX(cf[1].as_double(), 32.0, 10e-6); + EXPECT_APPROX(cf[2].as_double(), 33.0, 10e-6); } { cf = sf.getFeaturesByDocId(40); - ASSERT_TRUE(cf != 0); - EXPECT_APPROX(cf[0], 41.0, 10e-6); - EXPECT_APPROX(cf[1], 42.0, 10e-6); - EXPECT_APPROX(cf[2], 43.0, 10e-6); + ASSERT_TRUE(cf != nullptr); + EXPECT_TRUE(cf[0].is_double()); + EXPECT_TRUE(!cf[0].is_data()); + EXPECT_EQUAL(cf[0].as_double(), 41.0); + EXPECT_TRUE(!cf[1].is_double()); + EXPECT_TRUE(cf[1].is_data()); + EXPECT_EQUAL(cf[1].as_data(), Memory("test", 4)); + EXPECT_EQUAL(cf[2].as_double(), 43.0); } { cf = sf.getFeaturesByDocId(50); - ASSERT_TRUE(cf != 0); - EXPECT_APPROX(cf[0], 51.0, 10e-6); - EXPECT_APPROX(cf[1], 52.0, 10e-6); - EXPECT_APPROX(cf[2], 53.0, 10e-6); + ASSERT_TRUE(cf != nullptr); + EXPECT_APPROX(cf[0].as_double(), 51.0, 10e-6); + EXPECT_APPROX(cf[1].as_double(), 52.0, 10e-6); + EXPECT_APPROX(cf[2].as_double(), 53.0, 10e-6); } - EXPECT_TRUE(sf.getFeaturesByDocId(5) == 0); - EXPECT_TRUE(sf.getFeaturesByDocId(15) == 0); - EXPECT_TRUE(sf.getFeaturesByDocId(25) == 0); - EXPECT_TRUE(sf.getFeaturesByDocId(35) == 0); - EXPECT_TRUE(sf.getFeaturesByDocId(45) == 0); - EXPECT_TRUE(sf.getFeaturesByDocId(55) == 0); + EXPECT_TRUE(sf.getFeaturesByDocId(5) == nullptr); + EXPECT_TRUE(sf.getFeaturesByDocId(15) == nullptr); + EXPECT_TRUE(sf.getFeaturesByDocId(25) == nullptr); + EXPECT_TRUE(sf.getFeaturesByDocId(35) == nullptr); + EXPECT_TRUE(sf.getFeaturesByDocId(45) == nullptr); + EXPECT_TRUE(sf.getFeaturesByDocId(55) == nullptr); } TEST_DONE(); } diff --git a/searchlib/src/tests/features/bm25/bm25_test.cpp b/searchlib/src/tests/features/bm25/bm25_test.cpp index eb2f46650a6..55c9caa6c0f 100644 --- a/searchlib/src/tests/features/bm25/bm25_test.cpp +++ b/searchlib/src/tests/features/bm25/bm25_test.cpp @@ -135,6 +135,7 @@ struct Bm25ExecutorTest : public ::testing::Test { void add_query_term(const vespalib::string& field_name, uint32_t matching_doc_count) { auto* term = test.getQueryEnv().getBuilder().addIndexNode({field_name}); term->field(0).setDocFreq(matching_doc_count, total_doc_count); + term->setUniqueId(test.getQueryEnv().getNumTerms() - 1); } void setup() { EXPECT_TRUE(test.setup()); @@ -236,4 +237,12 @@ TEST_F(Bm25ExecutorTest, b_param_can_be_overriden) EXPECT_TRUE(execute(score(3.0, 20, idf(25)))); } +TEST_F(Bm25ExecutorTest, inverse_document_frequency_can_be_overriden_with_significance) +{ + test.getQueryEnv().getProperties().add("vespa.term.0.significance", "0.35"); + setup(); + prepare_term(0, 0, 3, 20); + EXPECT_TRUE(execute(score(3.0, 20, 0.35))); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp index c7c3447a4cc..251040ecfa7 100644 --- a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp +++ b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp @@ -26,6 +26,8 @@ struct DummyExpression : IntrinsicExpression { DummyExpression(const FeatureType &type_in) : type(type_in) {} vespalib::string describe_self() const override { return "dummy"; } const FeatureType &result_type() const override { return type; } + void prepare_shared_state(const QueryEnv &, IObjectStore &) const override { + } FeatureExecutor &create_executor(const QueryEnv &, vespalib::Stash &stash) const override { return stash.create<DummyExecutor>(); } @@ -81,7 +83,7 @@ SetupResult::SetupResult(const TypeMap &object_inputs, setup_ok = rank.setup(index_env, {}); EXPECT_TRUE(!deps.accept_type_mismatch); } -SetupResult::~SetupResult() {} +SetupResult::~SetupResult() = default; void verify_output_type(const TypeMap &object_inputs, const vespalib::string &expression, const FeatureType &expect) diff --git a/searchlib/src/tests/features/util/util_test.cpp b/searchlib/src/tests/features/util/util_test.cpp index 6b166682346..208c290cff3 100644 --- a/searchlib/src/tests/features/util/util_test.cpp +++ b/searchlib/src/tests/features/util/util_test.cpp @@ -8,6 +8,7 @@ using namespace search; using namespace search::fef; using namespace search::fef::test; using namespace search::features; +using namespace search::features::util; SimpleTermData make_term(uint32_t uid) { SimpleTermData term; @@ -37,4 +38,23 @@ TEST_F("require that label can be mapped to term", TermLabelFixture) { EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "unknown")); } +template <typename T> +void verifyStrToNum() { + EXPECT_EQUAL(-17, static_cast<long>(strToNum<T>("-17"))); + EXPECT_EQUAL(-1, static_cast<long>(strToNum<T>("-1"))); + EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0"))); + EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("1"))); + EXPECT_EQUAL(17, static_cast<long>(strToNum<T>("17"))); + EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0x0"))); + EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("0x1"))); + EXPECT_EQUAL(27, static_cast<long>(strToNum<T>("0x1b"))); +} + +TEST("verify str2Num") { + verifyStrToNum<int8_t>(); + verifyStrToNum<int16_t>(); + verifyStrToNum<int32_t>(); + verifyStrToNum<int64_t>(); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 05c905cdc32..0f1c966ad5d 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -171,7 +171,7 @@ assertPostingList(const std::string &exp, uint32_t docId = itr.getKey(); ss << docId; if (store != nullptr) { // consider features as well - EntryRef ref(itr.getData()); + EntryRef ref(itr.getData().get_features()); store->setupForField(0, decoder); store->setupForUnpackFeatures(ref, decoder); decoder.unpackFeatures(matchData, docId); diff --git a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h index 4cd446352d0..255d0bead9f 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h +++ b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h @@ -27,13 +27,18 @@ public: /** * Creates a comparator using the given enum store. **/ - EnumStoreComparatorT(const EnumStoreType & enumStore); + EnumStoreComparatorT(const EnumStoreType & enumStore) + : _enumStore(enumStore), + _value() + {} /** * Creates a comparator using the given enum store and that uses the * given value during compare if the enum index is invalid. **/ - EnumStoreComparatorT(const EnumStoreType & enumStore, - EntryValue value); + EnumStoreComparatorT(const EnumStoreType & enumStore, EntryValue value) + : _enumStore(enumStore), + _value(value) + {} static int compare(EntryValue lhs, EntryValue rhs) { if (lhs < rhs) { @@ -60,7 +65,7 @@ private: typedef typename ParentType::EnumIndex EnumIndex; typedef typename ParentType::EntryValue EntryValue; using ParentType::getValue; - bool _prefix; + bool _prefix; size_t _prefixLen; public: /** @@ -90,22 +95,6 @@ public: } }; - -template <typename EntryType> -EnumStoreComparatorT<EntryType>::EnumStoreComparatorT(const EnumStoreType & enumStore) : - _enumStore(enumStore), - _value() -{ -} - -template <typename EntryType> -EnumStoreComparatorT<EntryType>::EnumStoreComparatorT(const EnumStoreType & enumStore, - EntryValue value) : - _enumStore(enumStore), - _value(value) -{ -} - template <> int EnumStoreComparatorT<NumericEntryType<float> >::compare(EntryValue lhs, EntryValue rhs); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 94c431368cb..decb4152d8d 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -704,6 +704,10 @@ template class btree::BTreeIteratorBase<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; +template class btree::BTreeConstIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; + +template class btree::BTreeConstIterator<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; + template class btree::BTreeIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index 48bf4a56874..d8604a5a85e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -467,6 +467,10 @@ extern template class btree::BTreeIteratorBase<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; +extern template class btree::BTreeConstIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; + +extern template class btree::BTreeConstIterator<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; + extern template class btree::BTreeIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, const EnumStoreComparatorWrapper, EnumTreeTraits>; diff --git a/searchlib/src/vespa/searchlib/common/featureset.cpp b/searchlib/src/vespa/searchlib/common/featureset.cpp index 07b04f2f675..adf24196200 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.cpp +++ b/searchlib/src/vespa/searchlib/common/featureset.cpp @@ -59,7 +59,7 @@ FeatureSet::contains(const std::vector<uint32_t> &docIds) const return true; } -feature_t * +FeatureSet::Value * FeatureSet::getFeaturesByIndex(uint32_t idx) { if (idx >= _docIds.size()) { @@ -68,7 +68,7 @@ FeatureSet::getFeaturesByIndex(uint32_t idx) return &(_values[idx * _names.size()]); } -const feature_t * +const FeatureSet::Value * FeatureSet::getFeaturesByDocId(uint32_t docId) const { uint32_t low = 0; diff --git a/searchlib/src/vespa/searchlib/common/featureset.h b/searchlib/src/vespa/searchlib/common/featureset.h index f57cf918ba5..1ec662685a7 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.h +++ b/searchlib/src/vespa/searchlib/common/featureset.h @@ -4,6 +4,7 @@ #include "feature.h" #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/memory.h> #include <map> #include <vector> @@ -16,12 +17,34 @@ namespace search { class FeatureSet { public: + class Value { + private: + std::vector<char> _data; + double _value; + public: + bool operator==(const Value &rhs) const { + return ((_data == rhs._data) && (_value == rhs._value)); + } + bool is_double() const { return _data.empty(); } + bool is_data() const { return !_data.empty(); } + double as_double() const { return _value; } + vespalib::Memory as_data() const { return vespalib::Memory(&_data[0], _data.size()); } + void set_double(double value) { + _data.clear(); + _value = value; + } + void set_data(vespalib::Memory data) { + _data.assign(data.data, data.data + data.size); + _value = 0.0; + } + }; + typedef vespalib::string string; typedef std::vector<string> StringVector; private: StringVector _names; - std::vector<uint32_t> _docIds; - std::vector<feature_t> _values; + std::vector<uint32_t> _docIds; + std::vector<Value> _values; FeatureSet(const FeatureSet &); FeatureSet & operator=(const FeatureSet &); @@ -112,7 +135,7 @@ public: * @return pointer to features * @param idx index into docid array **/ - feature_t *getFeaturesByIndex(uint32_t idx); + Value *getFeaturesByIndex(uint32_t idx); /** * Obtain the feature values belonging to a document based on the @@ -122,7 +145,7 @@ public: * @return pointer to features * @param docId docid value **/ - const feature_t *getFeaturesByDocId(uint32_t docId) const; + const Value *getFeaturesByDocId(uint32_t docId) const; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index e89655a75bb..f2114e4705d 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bm25_feature.h" +#include "utils.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/itermfielddata.h> #include <vespa/searchlib/fef/objectstore.h> @@ -23,6 +24,21 @@ using fef::ITermFieldData; using fef::MatchDataDetails; using fef::objectstore::as_value; +namespace { + +double +get_inverse_document_frequency(const ITermFieldData& term_field, + const fef::IQueryEnvironment& env, + const ITermData& term) + +{ + double fallback = Bm25Executor::calculate_inverse_document_frequency(term_field.get_matching_doc_count(), + term_field.get_total_doc_count()); + return util::lookupSignificance(env, term, fallback); +} + +} + Bm25Executor::Bm25Executor(const fef::FieldInfo& field, const fef::IQueryEnvironment& env, double avg_field_length, @@ -31,18 +47,17 @@ Bm25Executor::Bm25Executor(const fef::FieldInfo& field, : FeatureExecutor(), _terms(), _avg_field_length(avg_field_length), - _k1_param(k1_param), - _b_param(b_param) + _k1_mul_b(k1_param * b_param), + _k1_mul_one_minus_b(k1_param * (1 - b_param)) { for (size_t i = 0; i < env.getNumTerms(); ++i) { const ITermData* term = env.getTerm(i); for (size_t j = 0; j < term->numFields(); ++j) { const ITermFieldData& term_field = term->field(j); if (field.id() == term_field.getFieldId()) { - // TODO: Add support for using significance instead of default idf if specified in the query _terms.emplace_back(term_field.getHandle(MatchDataDetails::Cheap), - calculate_inverse_document_frequency(term_field.get_matching_doc_count(), - term_field.get_total_doc_count())); + get_inverse_document_frequency(term_field, env, *term), + k1_param); } } } @@ -72,8 +87,8 @@ Bm25Executor::execute(uint32_t doc_id) feature_t num_occs = term.tfmd->getNumOccs(); feature_t norm_field_length = ((feature_t)term.tfmd->getFieldLength()) / _avg_field_length; - feature_t numerator = term.inverse_doc_freq * num_occs * (_k1_param + 1); - feature_t denominator = num_occs + (_k1_param * (1 - _b_param + (_b_param * norm_field_length))); + feature_t numerator = num_occs * term.idf_mul_k1_plus_one; + feature_t denominator = num_occs + (_k1_mul_one_minus_b + _k1_mul_b * norm_field_length); score += numerator / denominator; } @@ -114,7 +129,7 @@ Bm25Blueprint::visitDumpFeatures(const fef::IIndexEnvironment& env, fef::IDumpFe { (void) env; (void) visitor; - // TODO: Implement + // TODO: Implement when feature is supported end-2-end with both memory and disk index. } fef::Blueprint::UP diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index 533c7487a2f..0afd14e7ac8 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -13,11 +13,11 @@ private: struct QueryTerm { fef::TermFieldHandle handle; const fef::TermFieldMatchData* tfmd; - double inverse_doc_freq; - QueryTerm(fef::TermFieldHandle handle_, double inverse_doc_freq_) + double idf_mul_k1_plus_one; + QueryTerm(fef::TermFieldHandle handle_, double inverse_doc_freq, double k1_param) : handle(handle_), tfmd(nullptr), - inverse_doc_freq(inverse_doc_freq_) + idf_mul_k1_plus_one(inverse_doc_freq * (k1_param + 1)) {} }; @@ -25,8 +25,11 @@ private: QueryTermVector _terms; double _avg_field_length; - double _k1_param; // Determines term frequency saturation characteristics. - double _b_param; // Adjusts the effects of the field length of the document matched compared to the average field length. + + // The 'k1' param determines term frequency saturation characteristics. + // The 'b' param adjusts the effects of the field length of the document matched compared to the average field length. + double _k1_mul_b; + double _k1_mul_one_minus_b; public: Bm25Executor(const fef::FieldInfo& field, diff --git a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp b/searchlib/src/vespa/searchlib/features/documenttestutils.cpp index e13699576cd..1768eb0a216 100644 --- a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp +++ b/searchlib/src/vespa/searchlib/features/documenttestutils.cpp @@ -18,7 +18,8 @@ using namespace search::fef; namespace search::features::util { -feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, uint32_t termId, feature_t fallback) +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) { if (termId == 0) { return fallback; // no previous term @@ -26,14 +27,15 @@ feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, uint32 const ITermData * data = env.getTerm(termId); const ITermData * prev = env.getTerm(termId - 1); - if (data == NULL || prev == NULL) { + if (data == nullptr || prev == nullptr) { return fallback; // default value } return lookupConnectedness(env, data->getUniqueId(), prev->getUniqueId(), fallback); } -feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, - uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, + uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) { // Connectedness of 0.5 between term with unique id 2 and term with unique id 1 is represented as: // [vespa.term.2.connexity: "1", vespa.term.2.connexity: "0.5"] @@ -49,33 +51,40 @@ feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, return fallback; } -feature_t lookupSignificance(const search::fef::IQueryEnvironment & env, uint32_t termId, feature_t fallback) +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& term, feature_t fallback) { - const ITermData * data = env.getTerm(termId); - if (data == NULL) { - return fallback; - } - // Significance of 0.5 for term with unique id 1 is represented as: // [vespa.term.1.significance: "0.5"] vespalib::asciistream os; - os << "vespa.term." << data->getUniqueId() << ".significance"; + os << "vespa.term." << term.getUniqueId() << ".significance"; Property p = env.getProperties().lookup(os.str()); if (p.found()) { return strToNum<feature_t>(p.get()); } - return fallback; } -double getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +{ + const ITermData* term = env.getTerm(termId); + if (term == nullptr) { + return fallback; + } + return lookupSignificance(env, *term, fallback); +} + +double +getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) { return std::log((docsInCorpus - docCount + 0.5)/(docCount + 0.5)); } static const double N = 1000000.0; -feature_t getSignificance(double docFreq) +feature_t +getSignificance(double docFreq) { if (docFreq < (1.0/N)) { docFreq = 1.0/N; @@ -95,7 +104,8 @@ feature_t getSignificance(double docFreq) #endif } -feature_t getSignificance(const search::fef::ITermData &termData) +feature_t +getSignificance(const search::fef::ITermData& termData) { typedef search::fef::ITermFieldRangeAdapter FRA; double df = 0; @@ -115,7 +125,7 @@ lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & vespalib::string tn1 = env.getProperties().lookup(featureName, table).get(fallback); vespalib::string tn2 = env.getProperties().lookup(featureName, table, fieldName).get(tn1); const search::fef::Table * retval = env.getTableManager().getTable(tn2); - if (retval == NULL) { + if (retval == nullptr) { LOG(warning, "Could not find the %s '%s' to be used for field '%s' in feature '%s'", table.c_str(), tn2.c_str(), fieldName.c_str(), featureName.c_str()); } diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp index 018da0e7bcd..4ff9d2f4e30 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp @@ -29,8 +29,11 @@ struct IntrinsicBlueprint : IntrinsicExpression { : blueprint(std::move(blueprint_in)), type(type_in) {} vespalib::string describe_self() const override { return blueprint->getName(); } const FeatureType &result_type() const override { return type; } - FeatureExecutor &create_executor(const QueryEnv &queryEnv, vespalib::Stash &stash) const override { - return blueprint->createExecutor(queryEnv, stash); + void prepare_shared_state(const QueryEnv & env, fef::IObjectStore & store) const override { + blueprint->prepareSharedState(env, store); + } + FeatureExecutor &create_executor(const QueryEnv &env, vespalib::Stash &stash) const override { + return blueprint->createExecutor(env, stash); } }; diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_expression.h b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_expression.h index 34c4f34b03f..79cb3f9035b 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_expression.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_expression.h @@ -11,6 +11,7 @@ namespace search::fef { class FeatureType; class FeatureExecutor; class IQueryEnvironment; +class IObjectStore; } namespace search::features::rankingexpression { @@ -26,8 +27,8 @@ struct IntrinsicExpression { using UP = std::unique_ptr<IntrinsicExpression>; virtual vespalib::string describe_self() const = 0; virtual const FeatureType &result_type() const = 0; - virtual FeatureExecutor &create_executor(const QueryEnv &queryEnv, - vespalib::Stash &stash) const = 0; + virtual void prepare_shared_state(const QueryEnv & env, fef::IObjectStore & store) const = 0; + virtual FeatureExecutor &create_executor(const QueryEnv &queryEnv, vespalib::Stash &stash) const = 0; virtual ~IntrinsicExpression(); }; diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index b2c8c64d55a..2733ec62105 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -134,10 +134,14 @@ CompiledRankingExpressionExecutor::execute(uint32_t) //----------------------------------------------------------------------------- +namespace { + using Context = fef::FeatureExecutor::Inputs; double resolve_input(void *ctx, size_t idx) { return ((const Context *)(ctx))->get_number(idx); } Context *make_ctx(const Context &inputs) { return const_cast<Context *>(&inputs); } +} + LazyCompiledRankingExpressionExecutor::LazyCompiledRankingExpressionExecutor(const CompiledFunction &compiled_function) : _ranking_function(compiled_function.get_lazy_function()) { @@ -278,6 +282,14 @@ RankingExpressionBlueprint::createInstance() const return std::make_unique<RankingExpressionBlueprint>(_expression_replacer); } +void +RankingExpressionBlueprint::prepareSharedState(const fef::IQueryEnvironment & env, fef::IObjectStore & store) const +{ + if (_intrinsic_expression) { + return _intrinsic_expression->prepare_shared_state(env, store); + } +} + fef::FeatureExecutor & RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const { diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h index 8d5144206ea..104e8d63a70 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h @@ -26,7 +26,7 @@ private: public: RankingExpressionBlueprint(); RankingExpressionBlueprint(rankingexpression::ExpressionReplacer::SP replacer); - ~RankingExpressionBlueprint(); + ~RankingExpressionBlueprint() override; void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &visitor) const override; fef::Blueprint::UP createInstance() const override; @@ -37,6 +37,7 @@ public: } bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + void prepareSharedState(const fef::IQueryEnvironment & queryEnv, fef::IObjectStore & objectStore) const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/utils.cpp b/searchlib/src/vespa/searchlib/features/utils.cpp index 3f68e69ff25..7bfea67b7f6 100644 --- a/searchlib/src/vespa/searchlib/features/utils.cpp +++ b/searchlib/src/vespa/searchlib/features/utils.cpp @@ -1,28 +1,43 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "utils.hpp" +#include <charconv> namespace search::features::util { +template <typename T> +T strToInt(vespalib::stringref str) +{ + T retval = 0; + if ((str.size() > 2) && (str[0] == '0') && ((str[1] | 0x20) == 'x')) { + std::from_chars(str.data()+2, str.data()+str.size(), retval, 16); + } else { + std::from_chars(str.data(), str.data()+str.size(), retval, 10); + } + + return retval; +} + template <> uint8_t strToNum<uint8_t>(vespalib::stringref str) { - return strToNum<uint16_t>(str); + return strToInt<uint16_t>(str); } template <> int8_t strToNum<int8_t>(vespalib::stringref str) { - return strToNum<int16_t>(str); + return strToInt<int16_t>(str); } template double strToNum<double>(vespalib::stringref str); template float strToNum<float>(vespalib::stringref str); -template uint16_t strToNum<uint16_t>(vespalib::stringref str); -template uint32_t strToNum<uint32_t>(vespalib::stringref str); -template uint64_t strToNum<uint64_t>(vespalib::stringref str); -template int16_t strToNum<int16_t>(vespalib::stringref str); -template int32_t strToNum<int32_t>(vespalib::stringref str); -template int64_t strToNum<int64_t>(vespalib::stringref str); + +template <> uint16_t strToNum<uint16_t>(vespalib::stringref str) { return strToInt<uint16_t>(str); } +template <> uint32_t strToNum<uint32_t>(vespalib::stringref str) { return strToInt<uint32_t>(str); } +template <> uint64_t strToNum<uint64_t>(vespalib::stringref str) { return strToInt<uint64_t>(str); } +template <> int16_t strToNum<int16_t>(vespalib::stringref str) { return strToInt<int16_t>(str); } +template <> int32_t strToNum<int32_t>(vespalib::stringref str) { return strToInt<int32_t>(str); } +template <> int64_t strToNum<int64_t>(vespalib::stringref str) { return strToInt<int64_t>(str); } } diff --git a/searchlib/src/vespa/searchlib/features/utils.h b/searchlib/src/vespa/searchlib/features/utils.h index 859c66af66a..890c7cc5225 100644 --- a/searchlib/src/vespa/searchlib/features/utils.h +++ b/searchlib/src/vespa/searchlib/features/utils.h @@ -116,6 +116,17 @@ feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, * Uses the property map of the query environment to lookup this data. * * @param env The query environment. + * @param term The term data. + * @param fallback The value to return if the significance was not found in the property map. + * @return The significance. + */ +feature_t lookupSignificance(const search::fef::IQueryEnvironment& env, const search::fef::ITermData& term, feature_t fallback); + +/** + * Returns the significance of the given term. + * Uses the property map of the query environment to lookup this data. + * + * @param env The query environment. * @param termId The term id. * @param fallback The value to return if the significance was not found in the property map. * @return The significance. diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index e2e1c99a9b9..8daf87e1899 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -127,36 +127,35 @@ FieldIndex::compactFeatures() const PostingList *tree = _postingListStore.getTreeEntry(pidx); auto pitr = tree->begin(_postingListStore.getAllocator()); for (; pitr.valid(); ++pitr) { - EntryRef oldFeatures(pitr.getData()); + const PostingListEntry &posting_entry(pitr.getData()); // Filter on which buffers to move features from when // performing incremental compaction. - EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, oldFeatures); + EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, posting_entry.get_features()); // Features must be written before reference is updated. std::atomic_thread_fence(std::memory_order_release); - // Ugly, ugly due to const_cast in iterator - pitr.writeData(newFeatures.ref()); + // Reference the moved data + posting_entry.update_features(newFeatures); } } else { const PostingListKeyDataType *shortArray = _postingListStore.getKeyDataEntry(pidx, clusterSize); const PostingListKeyDataType *ite = shortArray + clusterSize; for (const PostingListKeyDataType *it = shortArray; it < ite; ++it) { - EntryRef oldFeatures(it->getData()); + const PostingListEntry &posting_entry(it->getData()); // Filter on which buffers to move features from when // performing incremental compaction. - EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, oldFeatures); + EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, posting_entry.get_features()); // Features must be written before reference is updated. std::atomic_thread_fence(std::memory_order_release); - // Ugly, ugly due to const_cast, but new data is - // semantically equal to old data - const_cast<PostingListKeyDataType *>(it)->setData(newFeatures.ref()); + // Reference the moved data + posting_entry.update_features(newFeatures); } } } @@ -189,7 +188,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) assert(pitr.valid()); for (; pitr.valid(); ++pitr) { uint32_t docId = pitr.getKey(); - EntryRef featureRef(pitr.getData()); + EntryRef featureRef(pitr.getData().get_features()); _featureStore.setupForReadFeatures(featureRef, decoder); decoder.readFeatures(features); features.set_doc_id(docId); @@ -202,7 +201,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) const PostingListKeyDataType *kde = kd + clusterSize; for (; kd != kde; ++kd) { uint32_t docId = kd->_key; - EntryRef featureRef(kd->getData()); + EntryRef featureRef(kd->getData().get_features()); _featureStore.setupForReadFeatures(featureRef, decoder); decoder.readFeatures(features); features.set_doc_id(docId); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index dba57f553b5..d5df2fa49c8 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -5,6 +5,7 @@ #include "feature_store.h" #include "field_index_remover.h" #include "word_store.h" +#include "posting_list_entry.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/field_length_calculator.h> #include <vespa/searchlib/index/indexbuilder.h> @@ -35,8 +36,8 @@ class OrderedFieldIndexInserter; class FieldIndex { public: // Mapping from docid -> feature ref - using PostingList = btree::BTreeRoot<uint32_t, uint32_t, search::btree::NoAggregated>; - using PostingListStore = btree::BTreeStore<uint32_t, uint32_t, + using PostingList = btree::BTreeRoot<uint32_t, PostingListEntry, search::btree::NoAggregated>; + using PostingListStore = btree::BTreeStore<uint32_t, PostingListEntry, search::btree::NoAggregated, std::less<uint32_t>, btree::BTreeDefaultTraits>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 637a13d67be..1d38e88b747 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -111,7 +111,7 @@ OrderedFieldIndexInserter::add(uint32_t docId, assert(_prevDocId == noDocId || _prevDocId < docId || (_prevDocId == docId && !_prevAdd)); datastore::EntryRef featureRef = _fieldIndex.addFeatures(features); - _adds.push_back(PostingListKeyDataType(docId, featureRef.ref())); + _adds.push_back(PostingListKeyDataType(docId, featureRef)); _listener.insert(_dItr.getKey()._wordRef, docId); _prevDocId = docId; _prevAdd = true; diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index 63040aab66f..290aa16dfe4 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.memoryindex.posting_iterator"); @@ -62,7 +63,7 @@ PostingIterator::doUnpack(uint32_t docId) assert(docId == getDocId()); assert(_itr.valid()); assert(docId == _itr.getKey()); - datastore::EntryRef featureRef(_itr.getData()); + datastore::EntryRef featureRef(_itr.getData().get_features()); _featureStore.setupForUnpackFeatures(featureRef, _featureDecoder); _featureDecoder.unpackFeatures(_matchData, docId); setUnpacked(); @@ -70,3 +71,59 @@ PostingIterator::doUnpack(uint32_t docId) } +namespace search::btree { + +template class BTreeNodeTT<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::INTERNAL_SLOTS>; + +template class BTreeLeafNode<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::LEAF_SLOTS>; + +template class BTreeNodeStore<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::INTERNAL_SLOTS, + BTreeDefaultTraits::LEAF_SLOTS>; + +template class BTreeIteratorBase<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::INTERNAL_SLOTS, + BTreeDefaultTraits::LEAF_SLOTS, + BTreeDefaultTraits::PATH_SIZE>; + +template class BTreeIterator<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + std::less<uint32_t>, + BTreeDefaultTraits>; + +template class BTree<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + std::less<uint32_t>, + BTreeDefaultTraits>; + +template class BTreeRoot<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + std::less<uint32_t>, + BTreeDefaultTraits>; + +template class BTreeRootBase<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::INTERNAL_SLOTS, + BTreeDefaultTraits::LEAF_SLOTS>; + +template class BTreeNodeAllocator<uint32_t, + search::memoryindex::PostingListEntry, + search::btree::NoAggregated, + BTreeDefaultTraits::INTERNAL_SLOTS, + BTreeDefaultTraits::LEAF_SLOTS>; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h new file mode 100644 index 00000000000..b28cd87736c --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h @@ -0,0 +1,36 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# pragma once + +#include <vespa/vespalib/datastore/entryref.h> + +namespace search::memoryindex { + +/** + * Entry per document in memory index posting list. + */ +class PostingListEntry { + mutable datastore::EntryRef _features; // reference to compressed features + +public: + PostingListEntry(datastore::EntryRef features) + : _features(features) + { + } + + PostingListEntry() + : _features() + { + } + + datastore::EntryRef get_features() const { return _features; } + + /* + * Reference moved features (used when compacting FeatureStore). + * The moved features must have the same content as the original + * features. + */ + void update_features(datastore::EntryRef features) const { _features = features; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 54c0aa866b4..f5300430bea 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -261,13 +261,13 @@ FakeMemTreeOccMgr::flush() lastWord = wordIdx; if (i->getRemove()) { if (itr.valid() && itr.getKey() == docId) { - uint64_t bits = _featureStore.bitSize(fw->getPackedIndex(), EntryRef(itr.getData())); + uint64_t bits = _featureStore.bitSize(fw->getPackedIndex(), EntryRef(itr.getData().get_features())); _featureSizes[wordIdx] -= RefType::align((bits + 7) / 8) * 8; tree.remove(itr); } } else { if (!itr.valid() || docId < itr.getKey()) { - tree.insert(itr, docId, i->getFeatureRef().ref()); + tree.insert(itr, docId, i->getFeatureRef()); } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index a1c96bb3e5b..71febe10e59 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -30,12 +30,16 @@ RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, } } const FeatureSet::StringVector & names = state->_rankFeatures->getNames(); - const feature_t * values = state->_rankFeatures->getFeaturesByDocId(docid); + const FeatureSet::Value * values = state->_rankFeatures->getFeaturesByDocId(docid); if (type == RES_FEATUREDATA && values != nullptr) { vespalib::slime::Cursor& obj = target.insertObject(); for (uint32_t i = 0; i < names.size(); ++i) { vespalib::Memory name(names[i].c_str(), names[i].size()); - obj.setDouble(name, values[i]); + if (values[i].is_data()) { + obj.setData(name, values[i].as_data()); + } else { + obj.setDouble(name, values[i].as_double()); + } } return; } @@ -44,7 +48,7 @@ RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, json.clear(); json.beginObject(); for (uint32_t i = 0; i < names.size(); ++i) { - featureDump(json, names[i], values[i]); + featureDump(json, names[i], values[i].as_double()); } json.endObject(); vespalib::Memory value(json.toString().data(), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 9992d782d6e..c5b027a372d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -37,12 +37,16 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType } } const FeatureSet::StringVector &names = state->_summaryFeatures->getNames(); - const feature_t *values = state->_summaryFeatures->getFeaturesByDocId(docid); + const FeatureSet::Value *values = state->_summaryFeatures->getFeaturesByDocId(docid); if (type == RES_FEATUREDATA && values != nullptr) { vespalib::slime::Cursor& obj = target.insertObject(); for (uint32_t i = 0; i < names.size(); ++i) { vespalib::Memory name(names[i].c_str(), names[i].size()); - obj.setDouble(name, values[i]); + if (values[i].is_data()) { + obj.setData(name, values[i].as_data()); + } else { + obj.setDouble(name, values[i].as_double()); + } } if (state->_summaryFeaturesCached) { obj.setDouble(_M_cached, 1.0); @@ -56,7 +60,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType json.clear(); json.beginObject(); for (uint32_t i = 0; i < names.size(); ++i) { - featureDump(json, names[i], values[i]); + featureDump(json, names[i], values[i].as_double()); } json.appendKey(_G_cached); if (state->_summaryFeaturesCached) { diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index 53113ea0eb1..ae55c80c148 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -6,8 +6,6 @@ vespa_add_executable(storage_testrunner_app TEST SOURCES testrunner.cpp DEPENDS - storage_teststorageserver - storage_testvisiting storage_testcommon storage_testhostreporter storage_testdistributor diff --git a/storage/src/tests/common/testhelper.h b/storage/src/tests/common/testhelper.h index 5a42732ae43..1bcc53dfe12 100644 --- a/storage/src/tests/common/testhelper.h +++ b/storage/src/tests/common/testhelper.h @@ -19,17 +19,6 @@ CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ (dummylink).getNumReplies()); \ } -#define ASSERT_COMMAND_COUNT(count, dummylink) \ - { \ - std::ostringstream msgost; \ - if ((dummylink).getNumCommands() != count) { \ - for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \ - msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \ - } \ - } \ - CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ - (dummylink).getNumCommands()); \ - } namespace storage { diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index aa1059e6d79..9b63f5054c0 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -1,6 +1,14 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_teststorageserver TEST SOURCES + testvisitormessagesession.cpp + DEPENDS + storage_storageserver + storage_testcommon +) + +vespa_add_executable(storage_storageserver_gtest_runner_app TEST + SOURCES bouncertest.cpp bucketintegritycheckertest.cpp changedbucketownershiphandlertest.cpp @@ -13,18 +21,11 @@ vespa_add_library(storage_teststorageserver TEST service_layer_error_listener_test.cpp statemanagertest.cpp statereportertest.cpp - testvisitormessagesession.cpp - DEPENDS - storage_storageserver - storage_testcommon -) - -vespa_add_executable(storage_storageserver_gtest_runner_app TEST - SOURCES gtest_runner.cpp DEPENDS storage_storageserver storage_testcommon + storage_teststorageserver gtest ) diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index 371c24accbc..35b752fedfd 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cppunit/extensions/HelperMacros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/stat.h> @@ -13,12 +12,14 @@ #include <vespa/document/test/make_document_bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; +using namespace ::testing; namespace storage { -struct BouncerTest : public CppUnit::TestFixture { +struct BouncerTest : public Test { std::unique_ptr<TestStorageApp> _node; std::unique_ptr<DummyStorageLink> _upper; Bouncer* _manager; @@ -26,41 +27,11 @@ struct BouncerTest : public CppUnit::TestFixture { BouncerTest(); - void setUp() override; - void tearDown() override; + void SetUp() override; + void TearDown() override; void setUpAsNode(const lib::NodeType& type); - void testFutureTimestamp(); - void testAllowNotifyBucketChangeEvenWhenDistributorDown(); - void rejectLowerPrioritizedFeedMessagesWhenConfigured(); - void doNotRejectHigherPrioritizedFeedMessagesThanConfigured(); - void rejectionThresholdIsExclusive(); - void onlyRejectFeedMessagesWhenConfigured(); - void rejectionIsDisabledByDefaultInConfig(); - void readOnlyOperationsAreNotRejected(); - void internalOperationsAreNotRejected(); - void outOfBoundsConfigValuesThrowException(); - void abort_request_when_derived_bucket_space_node_state_is_marked_down(); - void client_operations_are_allowed_through_on_cluster_state_down_distributor(); - void cluster_state_activation_commands_are_not_bounced(); - - CPPUNIT_TEST_SUITE(BouncerTest); - CPPUNIT_TEST(testFutureTimestamp); - CPPUNIT_TEST(testAllowNotifyBucketChangeEvenWhenDistributorDown); - CPPUNIT_TEST(rejectLowerPrioritizedFeedMessagesWhenConfigured); - CPPUNIT_TEST(doNotRejectHigherPrioritizedFeedMessagesThanConfigured); - CPPUNIT_TEST(rejectionThresholdIsExclusive); - CPPUNIT_TEST(onlyRejectFeedMessagesWhenConfigured); - CPPUNIT_TEST(rejectionIsDisabledByDefaultInConfig); - CPPUNIT_TEST(readOnlyOperationsAreNotRejected); - CPPUNIT_TEST(internalOperationsAreNotRejected); - CPPUNIT_TEST(outOfBoundsConfigValuesThrowException); - CPPUNIT_TEST(abort_request_when_derived_bucket_space_node_state_is_marked_down); - CPPUNIT_TEST(client_operations_are_allowed_through_on_cluster_state_down_distributor); - CPPUNIT_TEST(cluster_state_activation_commands_are_not_bounced); - CPPUNIT_TEST_SUITE_END(); - using Priority = api::StorageMessage::Priority; static constexpr int RejectionDisabledConfigValue = -1; @@ -77,13 +48,11 @@ struct BouncerTest : public CppUnit::TestFixture { api::Timestamp timestamp, document::BucketSpace bucketSpace); - void assertMessageBouncedWithRejection(); - void assertMessageBouncedWithAbort(); - void assertMessageNotBounced(); + void expectMessageBouncedWithRejection(); + void expectMessageBouncedWithAbort(); + void expectMessageNotBounced(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(BouncerTest); - BouncerTest::BouncerTest() : _node(), _upper(), @@ -109,12 +78,12 @@ void BouncerTest::setUpAsNode(const lib::NodeType& type) { } void -BouncerTest::setUp() { +BouncerTest::SetUp() { setUpAsNode(lib::NodeType::STORAGE); } void -BouncerTest::tearDown() { +BouncerTest::TearDown() { _manager = nullptr; _lower = nullptr; if (_upper) { @@ -149,30 +118,27 @@ BouncerTest::createDummyFeedMessage(api::Timestamp timestamp, return cmd; } -void -BouncerTest::testFutureTimestamp() -{ - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _manager->metrics().clock_skew_aborts.getValue()); +TEST_F(BouncerTest, future_timestamp) { + EXPECT_EQ(0, _manager->metrics().clock_skew_aborts.getValue()); // Fail when future timestamps (more than 5 seconds) are received. { _upper->sendDown(createDummyFeedMessage(16 * 1000000)); - CPPUNIT_ASSERT_EQUAL(1, (int)_upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(0, (int)_upper->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::REJECTED, - static_cast<api::RemoveReply&>(*_upper->getReply(0)). - getResult().getResult()); + ASSERT_EQ(1, _upper->getNumReplies()); + EXPECT_EQ(0, _upper->getNumCommands()); + EXPECT_EQ(api::ReturnCode::REJECTED, + dynamic_cast<api::RemoveReply&>(*_upper->getReply(0)).getResult().getResult()); _upper->reset(); } - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().clock_skew_aborts.getValue()); + EXPECT_EQ(1, _manager->metrics().clock_skew_aborts.getValue()); // Verify that 1 second clock skew is OK { _upper->sendDown(createDummyFeedMessage(11 * 1000000)); - CPPUNIT_ASSERT_EQUAL(0, (int)_upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(1, (int)_lower->getNumCommands()); + EXPECT_EQ(0, _upper->getNumReplies()); + EXPECT_EQ(1, _lower->getNumCommands()); _lower->reset(); } @@ -180,15 +146,13 @@ BouncerTest::testFutureTimestamp() { _upper->sendDown(createDummyFeedMessage(5 * 1000000)); - CPPUNIT_ASSERT_EQUAL(1, (int)_lower->getNumCommands()); + EXPECT_EQ(1, _lower->getNumCommands()); } - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().clock_skew_aborts.getValue()); + EXPECT_EQ(1, _manager->metrics().clock_skew_aborts.getValue()); } -void -BouncerTest::testAllowNotifyBucketChangeEvenWhenDistributorDown() -{ +TEST_F(BouncerTest, allow_notify_bucket_change_even_when_distributor_down) { lib::NodeState state(lib::NodeType::DISTRIBUTOR, lib::State::DOWN); _node->getNodeStateUpdater().setReportedNodeState(state); // Trigger Bouncer state update @@ -202,39 +166,38 @@ BouncerTest::testAllowNotifyBucketChangeEvenWhenDistributorDown() auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(makeDocumentBucket(bucket), info); _upper->sendDown(cmd); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(1), _lower->getNumCommands()); + EXPECT_EQ(0, _upper->getNumReplies()); + EXPECT_EQ(1, _lower->getNumCommands()); } void -BouncerTest::assertMessageBouncedWithRejection() +BouncerTest::expectMessageBouncedWithRejection() { - CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::REJECTED, - static_cast<api::RemoveReply&>(*_upper->getReply(0)). - getResult().getResult()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumCommands()); + ASSERT_EQ(1, _upper->getNumReplies()); + EXPECT_EQ(0, _upper->getNumCommands()); + EXPECT_EQ(api::ReturnCode::REJECTED, + dynamic_cast<api::RemoveReply&>(*_upper->getReply(0)).getResult().getResult()); + EXPECT_EQ(size_t(0), _lower->getNumCommands()); } void -BouncerTest::assertMessageBouncedWithAbort() +BouncerTest::expectMessageBouncedWithAbort() { - CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumCommands()); + ASSERT_EQ(1, _upper->getNumReplies()); + EXPECT_EQ(0, _upper->getNumCommands()); auto& reply = dynamic_cast<api::StorageReply&>(*_upper->getReply(0)); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::ABORTED, - "We don't allow command of type MessageType(12, Remove) " - "when node is in state Down (on storage.2)"), - reply.getResult()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumCommands()); + EXPECT_EQ(api::ReturnCode(api::ReturnCode::ABORTED, + "We don't allow command of type MessageType(12, Remove) " + "when node is in state Down (on storage.2)"), + reply.getResult()); + EXPECT_EQ(0, _lower->getNumCommands()); } void -BouncerTest::assertMessageNotBounced() +BouncerTest::expectMessageNotBounced() { - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(1), _lower->getNumCommands()); + EXPECT_EQ(size_t(0), _upper->getNumReplies()); + EXPECT_EQ(size_t(1), _lower->getNumCommands()); } void @@ -246,49 +209,37 @@ BouncerTest::configureRejectionThreshold(int newThreshold) _manager->configure(std::move(config)); } -void -BouncerTest::rejectLowerPrioritizedFeedMessagesWhenConfigured() -{ +TEST_F(BouncerTest, reject_lower_prioritized_feed_messages_when_configured) { configureRejectionThreshold(Priority(120)); _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(121))); - assertMessageBouncedWithRejection(); + expectMessageBouncedWithRejection(); } -void -BouncerTest::doNotRejectHigherPrioritizedFeedMessagesThanConfigured() -{ +TEST_F(BouncerTest, do_not_reject_higher_prioritized_feed_messages_than_configured) { configureRejectionThreshold(Priority(120)); _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(119))); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::rejectionThresholdIsExclusive() -{ +TEST_F(BouncerTest, rejection_threshold_is_exclusive) { configureRejectionThreshold(Priority(120)); _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(120))); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::onlyRejectFeedMessagesWhenConfigured() -{ +TEST_F(BouncerTest, only_reject_feed_messages_when_configured) { configureRejectionThreshold(RejectionDisabledConfigValue); // A message with even the lowest priority should not be rejected. _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(255))); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::rejectionIsDisabledByDefaultInConfig() -{ +TEST_F(BouncerTest, rejection_is_disabled_by_default_in_config) { _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(255))); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::readOnlyOperationsAreNotRejected() -{ +TEST_F(BouncerTest, read_only_operations_are_not_rejected) { configureRejectionThreshold(Priority(1)); // StatBucket is an external operation, but it's not a mutating operation // and should therefore not be blocked. @@ -296,33 +247,22 @@ BouncerTest::readOnlyOperationsAreNotRejected() makeDocumentBucket(document::BucketId(16, 5)), ""); cmd->setPriority(Priority(2)); _upper->sendDown(cmd); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::internalOperationsAreNotRejected() -{ +TEST_F(BouncerTest, internal_operations_are_not_rejected) { configureRejectionThreshold(Priority(1)); document::BucketId bucket(16, 1234); api::BucketInfo info(0x1, 0x2, 0x3); auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(makeDocumentBucket(bucket), info); cmd->setPriority(Priority(2)); _upper->sendDown(cmd); - assertMessageNotBounced(); + expectMessageNotBounced(); } -void -BouncerTest::outOfBoundsConfigValuesThrowException() -{ - try { - configureRejectionThreshold(256); - CPPUNIT_FAIL("Upper bound violation not caught"); - } catch (config::InvalidConfigException &) {} - - try { - configureRejectionThreshold(-2); - CPPUNIT_FAIL("Lower bound violation not caught"); - } catch (config::InvalidConfigException &) {} +TEST_F(BouncerTest, out_of_bounds_config_values_throw_exception) { + EXPECT_THROW(configureRejectionThreshold(256), config::InvalidConfigException); + EXPECT_THROW(configureRejectionThreshold(-2), config::InvalidConfigException); } @@ -340,25 +280,23 @@ makeClusterStateBundle(const vespalib::string &baselineState, const std::map<doc } -void -BouncerTest::abort_request_when_derived_bucket_space_node_state_is_marked_down() -{ - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _manager->metrics().unavailable_node_aborts.getValue()); +TEST_F(BouncerTest, abort_request_when_derived_bucket_space_node_state_is_marked_down) { + EXPECT_EQ(0, _manager->metrics().unavailable_node_aborts.getValue()); auto state = makeClusterStateBundle("distributor:3 storage:3", {{ document::FixedBucketSpaces::default_space(), "distributor:3 storage:3 .2.s:d" }}); _node->getNodeStateUpdater().setClusterStateBundle(state); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); - assertMessageBouncedWithAbort(); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().unavailable_node_aborts.getValue()); + expectMessageBouncedWithAbort(); + EXPECT_EQ(1, _manager->metrics().unavailable_node_aborts.getValue()); _upper->reset(); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::global_space())); - assertMessageNotBounced(); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _manager->metrics().unavailable_node_aborts.getValue()); + expectMessageNotBounced(); + EXPECT_EQ(1, _manager->metrics().unavailable_node_aborts.getValue()); } -void BouncerTest::client_operations_are_allowed_through_on_cluster_state_down_distributor() { - tearDown(); +TEST_F(BouncerTest, client_operations_are_allowed_through_on_cluster_state_down_distributor) { + TearDown(); setUpAsNode(lib::NodeType::DISTRIBUTOR); // Distributor states never vary across bucket spaces, so not necessary to test with @@ -366,12 +304,12 @@ void BouncerTest::client_operations_are_allowed_through_on_cluster_state_down_di auto state = makeClusterStateBundle("distributor:3 .2.s:d storage:3", {}); _node->getNodeStateUpdater().setClusterStateBundle(state); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); - assertMessageNotBounced(); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _manager->metrics().unavailable_node_aborts.getValue()); + expectMessageNotBounced(); + EXPECT_EQ(0, _manager->metrics().unavailable_node_aborts.getValue()); } -void BouncerTest::cluster_state_activation_commands_are_not_bounced() { - tearDown(); +TEST_F(BouncerTest, cluster_state_activation_commands_are_not_bounced) { + TearDown(); setUpAsNode(lib::NodeType::DISTRIBUTOR); auto state = makeClusterStateBundle("version:10 distributor:3 .2.s:d storage:3", {}); // Our index (2) is down @@ -379,7 +317,7 @@ void BouncerTest::cluster_state_activation_commands_are_not_bounced() { auto activate_cmd = std::make_shared<api::ActivateClusterStateVersionCommand>(11); _upper->sendDown(activate_cmd); - assertMessageNotBounced(); + expectMessageNotBounced(); } } // storage diff --git a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp index a4a6cbae9cf..ae466f04734 100644 --- a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp +++ b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <boost/lexical_cast.hpp> -#include <cppunit/extensions/HelperMacros.h> -#include <vespa/log/log.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/storageserver/bucketintegritychecker.h> @@ -11,43 +8,26 @@ #include <tests/common/dummystoragelink.h> #include <vespa/vespalib/io/fileutil.h> #include <tests/common/teststorageapp.h> +#include <vespa/vespalib/gtest/gtest.h> -LOG_SETUP(".test.bucketintegritychecker"); +using namespace ::testing; namespace storage { -struct BucketIntegrityCheckerTest : public CppUnit::TestFixture { +struct BucketIntegrityCheckerTest : public Test { std::unique_ptr<vdstestlib::DirConfig> _config; std::unique_ptr<TestServiceLayerApp> _node; int _timeout; // Timeout in seconds before aborting - void setUp() override { + void SetUp() override { _timeout = 60*2; - _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); - _node.reset(new TestServiceLayerApp(DiskCount(256), - NodeIndex(0), - _config->getConfigId())); + _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true)); + _node = std::make_unique<TestServiceLayerApp>( + DiskCount(256), NodeIndex(0), _config->getConfigId()); } - - void tearDown() override { - LOG(info, "Finished test"); - } - - void testConfig(); - void testBasicFunctionality(); - void testTiming(); - - CPPUNIT_TEST_SUITE(BucketIntegrityCheckerTest); - CPPUNIT_TEST(testConfig); - CPPUNIT_TEST(testBasicFunctionality); - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(BucketIntegrityCheckerTest); - -void BucketIntegrityCheckerTest::testConfig() -{ - +TEST_F(BucketIntegrityCheckerTest, config) { // Verify that config is read correctly. Given config should not use // any default values. vdstestlib::DirConfig::Config& config( @@ -63,18 +43,18 @@ void BucketIntegrityCheckerTest::testConfig() _node->getComponentRegister()); checker.setMaxThreadWaitTime(framework::MilliSecTime(10)); SchedulingOptions& opt(checker.getSchedulingOptions()); - CPPUNIT_ASSERT_EQUAL(60u, opt._dailyCycleStart); - CPPUNIT_ASSERT_EQUAL(360u, opt._dailyCycleStop); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[0]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_CHEAP, opt._dailyStates[1]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_FULL, opt._dailyStates[2]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[3]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::DONT_RUN, opt._dailyStates[4]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_CHEAP, opt._dailyStates[5]); - CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[6]); - CPPUNIT_ASSERT_EQUAL(2u, opt._maxPendingCount); - CPPUNIT_ASSERT_EQUAL(framework::SecondTime(7200), opt._minCycleTime); - CPPUNIT_ASSERT_EQUAL(framework::SecondTime(5), opt._requestDelay); + EXPECT_EQ(60u, opt._dailyCycleStart); + EXPECT_EQ(360u, opt._dailyCycleStop); + EXPECT_EQ(SchedulingOptions::CONTINUE, opt._dailyStates[0]); + EXPECT_EQ(SchedulingOptions::RUN_CHEAP, opt._dailyStates[1]); + EXPECT_EQ(SchedulingOptions::RUN_FULL, opt._dailyStates[2]); + EXPECT_EQ(SchedulingOptions::CONTINUE, opt._dailyStates[3]); + EXPECT_EQ(SchedulingOptions::DONT_RUN, opt._dailyStates[4]); + EXPECT_EQ(SchedulingOptions::RUN_CHEAP, opt._dailyStates[5]); + EXPECT_EQ(SchedulingOptions::CONTINUE, opt._dailyStates[6]); + EXPECT_EQ(2u, opt._maxPendingCount); + EXPECT_EQ(framework::SecondTime(7200), opt._minCycleTime); + EXPECT_EQ(framework::SecondTime(5), opt._requestDelay); } namespace { @@ -115,11 +95,11 @@ namespace { mytime.tm_min = 0; mytime.tm_sec = 0; time_t startTime = timegm(&mytime); - CPPUNIT_ASSERT(gmtime_r(&startTime, &mytime)); + assert(gmtime_r(&startTime, &mytime)); while (mytime.tm_wday != 0) { ++mytime.tm_mday; startTime = timegm(&mytime); - CPPUNIT_ASSERT(gmtime_r(&startTime, &mytime)); + assert(gmtime_r(&startTime, &mytime)); } // Add the wanted values to the start time time_t resultTime = startTime; @@ -166,8 +146,18 @@ namespace { } } -void BucketIntegrityCheckerTest::testBasicFunctionality() -{ +#define ASSERT_COMMAND_COUNT(count, dummylink) \ + { \ + std::ostringstream msgost; \ + if ((dummylink).getNumCommands() != count) { \ + for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \ + msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \ + } \ + } \ + ASSERT_EQ(size_t(count), (dummylink).getNumCommands()) << msgost.str(); \ + } + +TEST_F(BucketIntegrityCheckerTest, basic_functionality) { _node->getClock().setAbsoluteTimeInSeconds(getDate("week1 sun 00:00:00")); addBucketsToDatabase(*_node, false); DummyStorageLink* dummyLink = 0; @@ -193,91 +183,70 @@ void BucketIntegrityCheckerTest::testBasicFunctionality() dummyLink->waitForMessages(4, _timeout); FastOS_Thread::Sleep(10); // Give 5th message chance to come ASSERT_COMMAND_COUNT(4, *dummyLink); - RepairBucketCommand *cmd1 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(0).get()); - CPPUNIT_ASSERT_EQUAL(230, (int)cmd1->getPriority()); - CPPUNIT_ASSERT(cmd1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x234), - cmd1->getBucketId()); - RepairBucketCommand *cmd2 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(1).get()); - CPPUNIT_ASSERT(cmd2); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x456), - cmd2->getBucketId()); - RepairBucketCommand *cmd3 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(2).get()); - CPPUNIT_ASSERT(cmd3); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x567), - cmd3->getBucketId()); - RepairBucketCommand *cmd4 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(3).get()); - CPPUNIT_ASSERT(cmd4); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x987), - cmd4->getBucketId()); + auto* cmd1 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(0).get()); + EXPECT_EQ(230, cmd1->getPriority()); + ASSERT_TRUE(cmd1); + EXPECT_EQ(document::BucketId(16, 0x234), cmd1->getBucketId()); + auto* cmd2 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(1).get()); + ASSERT_TRUE(cmd2); + EXPECT_EQ(document::BucketId(16, 0x456), cmd2->getBucketId()); + auto* cmd3 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(2).get()); + ASSERT_TRUE(cmd3); + EXPECT_EQ(document::BucketId(16, 0x567), cmd3->getBucketId()); + auto* cmd4 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(3).get()); + ASSERT_TRUE(cmd4); + EXPECT_EQ(document::BucketId(16, 0x987), cmd4->getBucketId()); // Answering a message on disk with no more buckets does not trigger new - std::shared_ptr<RepairBucketReply> reply1( - new RepairBucketReply(*cmd3)); - CPPUNIT_ASSERT(checker.onUp(reply1)); + auto reply1 = std::make_shared<RepairBucketReply>(*cmd3); + ASSERT_TRUE(checker.onUp(reply1)); FastOS_Thread::Sleep(10); // Give next message chance to come ASSERT_COMMAND_COUNT(4, *dummyLink); // Answering a message on disk with more buckets trigger new repair - std::shared_ptr<RepairBucketReply> reply2( - new RepairBucketReply(*cmd2)); - CPPUNIT_ASSERT(checker.onUp(reply2)); + auto reply2 = std::make_shared<RepairBucketReply>(*cmd2); + ASSERT_TRUE(checker.onUp(reply2)); dummyLink->waitForMessages(5, _timeout); FastOS_Thread::Sleep(10); // Give 6th message chance to come ASSERT_COMMAND_COUNT(5, *dummyLink); - RepairBucketCommand *cmd5 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(4).get()); - CPPUNIT_ASSERT(cmd5); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x345), - cmd5->getBucketId()); + auto* cmd5 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(4).get()); + ASSERT_TRUE(cmd5); + EXPECT_EQ(document::BucketId(16, 0x345), cmd5->getBucketId()); // Fail a repair, causing it to be resent later, but first continue // with other bucket. - std::shared_ptr<RepairBucketReply> reply3( - new RepairBucketReply(*cmd1)); + auto reply3 = std::make_shared<RepairBucketReply>(*cmd1); reply3->setResult(api::ReturnCode(api::ReturnCode::IGNORED)); - CPPUNIT_ASSERT(checker.onUp(reply3)); + ASSERT_TRUE(checker.onUp(reply3)); dummyLink->waitForMessages(6, _timeout); FastOS_Thread::Sleep(10); // Give 7th message chance to come ASSERT_COMMAND_COUNT(6, *dummyLink); - RepairBucketCommand *cmd6 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(5).get()); - CPPUNIT_ASSERT(cmd6); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x123), - cmd6->getBucketId()); + auto* cmd6 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(5).get()); + ASSERT_TRUE(cmd6); + EXPECT_EQ(document::BucketId(16, 0x123), cmd6->getBucketId()); // Fail a repair with not found. That is an acceptable return code. // (No more requests as this was last for that disk) - std::shared_ptr<RepairBucketReply> reply4( - new RepairBucketReply(*cmd4)); + auto reply4 = std::make_shared<RepairBucketReply>(*cmd4); reply3->setResult(api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND)); - CPPUNIT_ASSERT(checker.onUp(reply4)); + ASSERT_TRUE(checker.onUp(reply4)); FastOS_Thread::Sleep(10); // Give 7th message chance to come ASSERT_COMMAND_COUNT(6, *dummyLink); // Send a repair reply that actually have corrected the bucket. api::BucketInfo newInfo(0x3456, 4, 8192); - std::shared_ptr<RepairBucketReply> reply5( - new RepairBucketReply(*cmd5, newInfo)); + auto reply5 = std::make_shared<RepairBucketReply>(*cmd5, newInfo); reply5->setAltered(true); - CPPUNIT_ASSERT(checker.onUp(reply5)); + ASSERT_TRUE(checker.onUp(reply5)); // Finish run. New iteration should not start yet as min // cycle time has not passed - std::shared_ptr<RepairBucketReply> reply6( - new RepairBucketReply(*cmd6)); - CPPUNIT_ASSERT(checker.onUp(reply6)); + auto reply6 = std::make_shared<RepairBucketReply>(*cmd6); + ASSERT_TRUE(checker.onUp(reply6)); dummyLink->waitForMessages(7, _timeout); ASSERT_COMMAND_COUNT(7, *dummyLink); - RepairBucketCommand *cmd7 = dynamic_cast<RepairBucketCommand*>( - dummyLink->getCommand(6).get()); - CPPUNIT_ASSERT(cmd7); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x234), - cmd7->getBucketId()); - std::shared_ptr<RepairBucketReply> reply7( - new RepairBucketReply(*cmd7)); - CPPUNIT_ASSERT(checker.onUp(reply7)); + auto* cmd7 = dynamic_cast<RepairBucketCommand*>(dummyLink->getCommand(6).get()); + ASSERT_TRUE(cmd7); + EXPECT_EQ(document::BucketId(16, 0x234), cmd7->getBucketId()); + auto reply7 = std::make_shared<RepairBucketReply>(*cmd7); + ASSERT_TRUE(checker.onUp(reply7)); FastOS_Thread::Sleep(10); // Give 8th message chance to come ASSERT_COMMAND_COUNT(7, *dummyLink); diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 98ad8761736..2f091572ed2 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/state.h> @@ -16,46 +15,20 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/storageserver/changedbucketownershiphandler.h> +#include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; +using namespace ::testing; namespace storage { -class ChangedBucketOwnershipHandlerTest : public CppUnit::TestFixture -{ +struct ChangedBucketOwnershipHandlerTest : Test { std::unique_ptr<TestServiceLayerApp> _app; std::unique_ptr<DummyStorageLink> _top; ChangedBucketOwnershipHandler* _handler; DummyStorageLink* _bottom; document::TestDocMan _testDocRepo; - CPPUNIT_TEST_SUITE(ChangedBucketOwnershipHandlerTest); - CPPUNIT_TEST(testEnumerateBucketsBelongingOnChangedNodes); - CPPUNIT_TEST(testNoPreExistingClusterState); - CPPUNIT_TEST(testNoAvailableDistributorsInCurrentState); - CPPUNIT_TEST(testNoAvailableDistributorsInCurrentAndNewState); - CPPUNIT_TEST(testDownEdgeToNoAvailableDistributors); - CPPUNIT_TEST(testOwnershipChangedOnDistributorUpEdge); - CPPUNIT_TEST(testDistributionConfigChangeUpdatesOwnership); - CPPUNIT_TEST(testAbortOpsWhenNoClusterStateSet); - CPPUNIT_TEST(testAbortOutdatedSplit); - CPPUNIT_TEST(testAbortOutdatedJoin); - CPPUNIT_TEST(testAbortOutdatedSetBucketState); - CPPUNIT_TEST(testAbortOutdatedCreateBucket); - CPPUNIT_TEST(testAbortOutdatedDeleteBucket); - CPPUNIT_TEST(testAbortOutdatedMergeBucket); - CPPUNIT_TEST(testAbortOutdatedRemoveLocation); - CPPUNIT_TEST(testIdealStateAbortsAreConfigurable); - CPPUNIT_TEST(testAbortOutdatedPutOperation); - CPPUNIT_TEST(testAbortOutdatedUpdateCommand); - CPPUNIT_TEST(testAbortOutdatedRemoveCommand); - CPPUNIT_TEST(testAbortOutdatedRevertCommand); - CPPUNIT_TEST(testIdealStateAbortUpdatesMetric); - CPPUNIT_TEST(testExternalLoadOpAbortUpdatesMetric); - CPPUNIT_TEST(testExternalLoadOpAbortsAreConfigurable); - CPPUNIT_TEST(testAbortCommandsWhenStorageNodeIsDown); - CPPUNIT_TEST_SUITE_END(); - // TODO test: down edge triggered on cluster state with cluster down? std::vector<document::BucketId> insertBuckets( @@ -89,10 +62,10 @@ class ChangedBucketOwnershipHandlerTest : public CppUnit::TestFixture void sendAndExpectAbortedCreateBucket(uint16_t fromDistributorIndex); template <typename MsgType, typename... MsgParams> - bool changeAbortsMessage(MsgParams&&... params); + void expectChangeAbortsMessage(bool expected, MsgParams&& ... params); template <typename MsgType, typename... MsgParams> - bool downAbortsMessage(MsgParams&&... params); + void expectDownAbortsMessage(bool expected, MsgParams&& ... params); lib::ClusterState getDefaultTestClusterState() const { return lib::ClusterState("distributor:4 storage:1"); @@ -101,36 +74,9 @@ class ChangedBucketOwnershipHandlerTest : public CppUnit::TestFixture lib::ClusterState getStorageDownTestClusterState() const { return lib::ClusterState("distributor:4 storage:1 .0.s:d"); } -public: - void testEnumerateBucketsBelongingOnChangedNodes(); - void testNoPreExistingClusterState(); - void testNoAvailableDistributorsInCurrentState(); - void testNoAvailableDistributorsInCurrentAndNewState(); - void testDownEdgeToNoAvailableDistributors(); - void testOwnershipChangedOnDistributorUpEdge(); - void testDistributionConfigChangeUpdatesOwnership(); - void testAbortOpsWhenNoClusterStateSet(); - void testAbortOutdatedSplit(); - void testAbortOutdatedJoin(); - void testAbortOutdatedSetBucketState(); - void testAbortOutdatedCreateBucket(); - void testAbortOutdatedDeleteBucket(); - void testAbortOutdatedMergeBucket(); - void testAbortOutdatedRemoveLocation(); - void testIdealStateAbortsAreConfigurable(); - void testAbortOutdatedPutOperation(); - void testAbortOutdatedUpdateCommand(); - void testAbortOutdatedRemoveCommand(); - void testAbortOutdatedRevertCommand(); - void testIdealStateAbortUpdatesMetric(); - void testExternalLoadOpAbortUpdatesMetric(); - void testExternalLoadOpAbortsAreConfigurable(); - void testAbortCommandsWhenStorageNodeIsDown(); - - void setUp() override; -}; -CPPUNIT_TEST_SUITE_REGISTRATION(ChangedBucketOwnershipHandlerTest); + void SetUp() override; +}; document::BucketId ChangedBucketOwnershipHandlerTest::nextOwnedBucket( @@ -171,7 +117,7 @@ ChangedBucketOwnershipHandlerTest::insertBuckets(uint32_t numBuckets, } void -ChangedBucketOwnershipHandlerTest::setUp() +ChangedBucketOwnershipHandlerTest::SetUp() { vdstestlib::DirConfig config(getStandardConfig(true)); @@ -185,8 +131,7 @@ ChangedBucketOwnershipHandlerTest::setUp() _top->open(); // Ensure we're not dependent on config schema default values. - std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> pconfig( - new vespa::config::content::PersistenceConfigBuilder); + auto pconfig = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); pconfig->abortOutdatedMutatingIdealStateOps = true; pconfig->abortOutdatedMutatingExternalLoadOps = true; _handler->configure(std::move(pconfig)); @@ -253,9 +198,7 @@ ChangedBucketOwnershipHandlerTest::applyClusterState( _handler->reloadClusterState(); } -void -ChangedBucketOwnershipHandlerTest::testEnumerateBucketsBelongingOnChangedNodes() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, enumerate_buckets_belonging_on_changed_nodes) { lib::ClusterState stateBefore("distributor:4 storage:1"); applyDistribution(Redundancy(1), NodeCount(4)); applyClusterState(stateBefore); @@ -267,25 +210,21 @@ ChangedBucketOwnershipHandlerTest::testEnumerateBucketsBelongingOnChangedNodes() _top->sendDown(createStateCmd("distributor:4 .1.s:d .3.s:d storage:1")); // TODO: refactor into own function - CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); - AbortBucketOperationsCommand::SP cmd( - std::dynamic_pointer_cast<AbortBucketOperationsCommand>( - _bottom->getCommand(0))); - CPPUNIT_ASSERT(cmd.get() != 0); + ASSERT_EQ(2, _bottom->getNumCommands()); + auto cmd = std::dynamic_pointer_cast<AbortBucketOperationsCommand>(_bottom->getCommand(0)); + ASSERT_TRUE(cmd.get() != 0); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node3Buckets)); - CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node0Buckets)); - CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node2Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node1Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node3Buckets)); + EXPECT_TRUE(hasAbortedNoneOf(cmd, node0Buckets)); + EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); - CPPUNIT_ASSERT_EQUAL(size_t(0), _top->getNumReplies()); + EXPECT_EQ(size_t(0), _top->getNumReplies()); } -void -ChangedBucketOwnershipHandlerTest::testNoPreExistingClusterState() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, no_pre_existing_cluster_state) { applyDistribution(Redundancy(1), NodeCount(4)); lib::ClusterState stateBefore("distributor:4 storage:1"); insertBuckets(2, 1, stateBefore); @@ -293,7 +232,7 @@ ChangedBucketOwnershipHandlerTest::testNoPreExistingClusterState() insertBuckets(2, 2, stateBefore); _top->sendDown(createStateCmd("distributor:4 .1.s:d .3.s:d storage:1")); - CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); + EXPECT_TRUE(hasOnlySetSystemStateCmdQueued(*_bottom)); } /** @@ -301,9 +240,7 @@ ChangedBucketOwnershipHandlerTest::testNoPreExistingClusterState() * more distributors, we do not send any abort messages since this should * already have been done on the down-edge. */ -void -ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentState() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, no_available_distributors_in_current_state) { applyDistribution(Redundancy(1), NodeCount(3)); lib::ClusterState insertedState("distributor:3 storage:1"); insertBuckets(2, 0, insertedState); @@ -313,12 +250,10 @@ ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentState() _app->setClusterState(downState); _top->sendDown(createStateCmd("distributor:3 .1.s:d storage:1")); - CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); + EXPECT_TRUE(hasOnlySetSystemStateCmdQueued(*_bottom)); } -void -ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentAndNewState() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, no_available_distributors_in_current_and_new_state) { applyDistribution(Redundancy(1), NodeCount(3)); lib::ClusterState insertedState("distributor:3 storage:1"); insertBuckets(2, 0, insertedState); @@ -329,12 +264,10 @@ ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentAndNewSta lib::ClusterState downState("distributor:3 .0.s:d .1.s:d .2.s:d storage:1"); _top->sendDown(createStateCmd(downState)); - CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); + EXPECT_TRUE(hasOnlySetSystemStateCmdQueued(*_bottom)); } -void -ChangedBucketOwnershipHandlerTest::testDownEdgeToNoAvailableDistributors() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, down_edge_to_no_available_distributors) { lib::ClusterState insertedState("distributor:3 storage:1"); applyDistribution(Redundancy(1), NodeCount(3)); applyClusterState(insertedState); @@ -345,20 +278,16 @@ ChangedBucketOwnershipHandlerTest::testDownEdgeToNoAvailableDistributors() _top->sendDown(createStateCmd(downState)); // TODO: refactor into own function - CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); - AbortBucketOperationsCommand::SP cmd( - std::dynamic_pointer_cast<AbortBucketOperationsCommand>( - _bottom->getCommand(0))); - CPPUNIT_ASSERT(cmd.get() != 0); + ASSERT_EQ(2, _bottom->getNumCommands()); + auto cmd = std::dynamic_pointer_cast<AbortBucketOperationsCommand>(_bottom->getCommand(0)); + ASSERT_TRUE(cmd.get() != 0); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node0Buckets)); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node2Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node0Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node1Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node2Buckets)); } -void -ChangedBucketOwnershipHandlerTest::testOwnershipChangedOnDistributorUpEdge() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, ownership_changed_on_distributor_up_edge) { lib::ClusterState stateBefore( "version:10 distributor:4 .1.s:d storage:4 .1.s:d"); lib::ClusterState stateAfter( @@ -373,19 +302,17 @@ ChangedBucketOwnershipHandlerTest::testOwnershipChangedOnDistributorUpEdge() _top->sendDown(createStateCmd(stateAfter)); // TODO: refactor into own function - CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); - AbortBucketOperationsCommand::SP cmd( - std::dynamic_pointer_cast<AbortBucketOperationsCommand>( - _bottom->getCommand(0))); - CPPUNIT_ASSERT(cmd.get() != 0); + ASSERT_EQ(2, _bottom->getNumCommands()); + auto cmd = std::dynamic_pointer_cast<AbortBucketOperationsCommand>(_bottom->getCommand(0)); + ASSERT_TRUE(cmd.get() != 0); - CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); - CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node0Buckets)); - CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node2Buckets)); + EXPECT_TRUE(hasAbortedAllOf(cmd, node1Buckets)); + EXPECT_TRUE(hasAbortedNoneOf(cmd, node0Buckets)); + EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); - CPPUNIT_ASSERT_EQUAL(size_t(0), _top->getNumReplies()); + EXPECT_EQ(0, _top->getNumReplies()); } void @@ -398,21 +325,16 @@ ChangedBucketOwnershipHandlerTest::sendAndExpectAbortedCreateBucket( _top->sendDown(msg); std::vector<api::StorageMessage::SP> replies(_top->getRepliesOnce()); - CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); - api::StorageReply& reply(dynamic_cast<api::StorageReply&>(*replies[0])); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, - reply.getResult().getResult()); + ASSERT_EQ(1, replies.size()); + auto& reply(dynamic_cast<api::StorageReply&>(*replies[0])); + EXPECT_EQ(api::ReturnCode::ABORTED, reply.getResult().getResult()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOpsWhenNoClusterStateSet() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_ops_when_no_cluster_state_set) { sendAndExpectAbortedCreateBucket(1); } -void -ChangedBucketOwnershipHandlerTest::testDistributionConfigChangeUpdatesOwnership() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, distribution_config_change_updates_ownership) { lib::ClusterState insertedState("distributor:3 storage:1"); applyClusterState(insertedState); applyDistribution(Redundancy(1), NodeCount(3)); @@ -431,8 +353,8 @@ ChangedBucketOwnershipHandlerTest::testDistributionConfigChangeUpdatesOwnership( * owned by distributor 1 in this state to trigger an abort. */ template <typename MsgType, typename... MsgParams> -bool -ChangedBucketOwnershipHandlerTest::changeAbortsMessage(MsgParams&&... params) +void +ChangedBucketOwnershipHandlerTest::expectChangeAbortsMessage(bool expected, MsgParams&&... params) { auto msg = std::make_shared<MsgType>(std::forward<MsgParams>(params)...); msg->setSourceIndex(1); @@ -444,15 +366,14 @@ ChangedBucketOwnershipHandlerTest::changeAbortsMessage(MsgParams&&... params) std::vector<api::StorageMessage::SP> replies(_top->getRepliesOnce()); // Test is single-threaded, no need to do any waiting. if (replies.empty()) { - return false; + EXPECT_FALSE(expected); } else { - CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + ASSERT_EQ(replies.size(), 1); // Make sure the message was actually aborted and not bounced with // some other arbitrary failure code. - api::StorageReply& reply(dynamic_cast<api::StorageReply&>(*replies[0])); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, - reply.getResult().getResult()); - return true; + auto& reply(dynamic_cast<api::StorageReply&>(*replies[0])); + ASSERT_EQ(reply.getResult().getResult(), api::ReturnCode::ABORTED); + EXPECT_TRUE(expected); } } @@ -463,21 +384,21 @@ ChangedBucketOwnershipHandlerTest::changeAbortsMessage(MsgParams&&... params) * node is down. This means that any abortable message will trigger an abort. */ template <typename MsgType, typename... MsgParams> -bool -ChangedBucketOwnershipHandlerTest::downAbortsMessage(MsgParams&&... params) +void +ChangedBucketOwnershipHandlerTest::expectDownAbortsMessage(bool expected, MsgParams&&... params) { (void) _top->getRepliesOnce(); (void) _bottom->getCommandsOnce(); - CPPUNIT_ASSERT((!changeAbortsMessage<MsgType, MsgParams...>(std::forward<MsgParams>(params) ...))); + ASSERT_NO_FATAL_FAILURE((expectChangeAbortsMessage<MsgType, MsgParams...>(false, std::forward<MsgParams>(params)...))); _top->sendDown(createStateCmd(getStorageDownTestClusterState())); - CPPUNIT_ASSERT_EQUAL(size_t(3), _bottom->getNumCommands()); + ASSERT_EQ(_bottom->getNumCommands(), 3); auto setSystemStateCommand = std::dynamic_pointer_cast<api::SetSystemStateCommand>(_bottom->getCommand(2)); - CPPUNIT_ASSERT(setSystemStateCommand); + ASSERT_TRUE(setSystemStateCommand); auto abortBucketOperationsCommand = std::dynamic_pointer_cast<AbortBucketOperationsCommand>(_bottom->getCommand(1)); - CPPUNIT_ASSERT(abortBucketOperationsCommand); + ASSERT_TRUE(abortBucketOperationsCommand); auto testCommand = _bottom->getCommand(0); - CPPUNIT_ASSERT(testCommand); - return abortBucketOperationsCommand->shouldAbort(testCommand->getBucket()); + ASSERT_TRUE(testCommand); + EXPECT_EQ(expected, abortBucketOperationsCommand->shouldAbort(testCommand->getBucket())); } /** @@ -502,59 +423,37 @@ ChangedBucketOwnershipHandlerTest::getBucketToAllow() const return makeDocumentBucket(nextOwnedBucket(1, state, document::BucketId())); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedSplit() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::SplitBucketCommand>( - getBucketToAbort())); - CPPUNIT_ASSERT(!changeAbortsMessage<api::SplitBucketCommand>( - getBucketToAllow())); +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_split) { + expectChangeAbortsMessage<api::SplitBucketCommand>(true, getBucketToAbort()); + expectChangeAbortsMessage<api::SplitBucketCommand>(false, getBucketToAllow()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedJoin() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::JoinBucketsCommand>( - getBucketToAbort())); - CPPUNIT_ASSERT(!changeAbortsMessage<api::JoinBucketsCommand>( - getBucketToAllow())); +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_join) { + expectChangeAbortsMessage<api::JoinBucketsCommand>(true, getBucketToAbort()); + expectChangeAbortsMessage<api::JoinBucketsCommand>(false, getBucketToAllow()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedSetBucketState() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::SetBucketStateCommand>( - getBucketToAbort(), api::SetBucketStateCommand::ACTIVE)); - CPPUNIT_ASSERT(!changeAbortsMessage<api::SetBucketStateCommand>( - getBucketToAllow(), api::SetBucketStateCommand::ACTIVE)); +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_set_bucket_state) { + expectChangeAbortsMessage<api::SetBucketStateCommand>( + true, getBucketToAbort(), api::SetBucketStateCommand::ACTIVE); + expectChangeAbortsMessage<api::SetBucketStateCommand>( + false, getBucketToAllow(), api::SetBucketStateCommand::ACTIVE); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedCreateBucket() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::CreateBucketCommand>( - getBucketToAbort())); - CPPUNIT_ASSERT(!changeAbortsMessage<api::CreateBucketCommand>( - getBucketToAllow())); +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_create_bucket) { + expectChangeAbortsMessage<api::CreateBucketCommand>(true, getBucketToAbort()); + expectChangeAbortsMessage<api::CreateBucketCommand>(false, getBucketToAllow()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedDeleteBucket() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::DeleteBucketCommand>( - getBucketToAbort())); - CPPUNIT_ASSERT(!changeAbortsMessage<api::DeleteBucketCommand>( - getBucketToAllow())); +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_delete_bucket) { + expectChangeAbortsMessage<api::DeleteBucketCommand>(true, getBucketToAbort()); + expectChangeAbortsMessage<api::DeleteBucketCommand>(false, getBucketToAllow()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedMergeBucket() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_merge_bucket) { std::vector<api::MergeBucketCommand::Node> nodes; - CPPUNIT_ASSERT(changeAbortsMessage<api::MergeBucketCommand>( - getBucketToAbort(), nodes, 0)); - CPPUNIT_ASSERT(!changeAbortsMessage<api::MergeBucketCommand>( - getBucketToAllow(), nodes, 0)); + expectChangeAbortsMessage<api::MergeBucketCommand>(true, getBucketToAbort(), nodes, 0); + expectChangeAbortsMessage<api::MergeBucketCommand>(false, getBucketToAllow(), nodes, 0); } /** @@ -562,114 +461,76 @@ ChangedBucketOwnershipHandlerTest::testAbortOutdatedMergeBucket() * used as the backing operation for GC we have to treat it as if it were an * ideal state operation class. */ -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveLocation() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_remove_location) { std::vector<api::MergeBucketCommand::Node> nodes; - CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveLocationCommand>( - "foo", getBucketToAbort())); - CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveLocationCommand>( - "foo", getBucketToAllow())); + expectChangeAbortsMessage<api::RemoveLocationCommand>(true, "foo", getBucketToAbort()); + expectChangeAbortsMessage<api::RemoveLocationCommand>(false, "foo", getBucketToAllow()); } -void -ChangedBucketOwnershipHandlerTest::testIdealStateAbortsAreConfigurable() -{ - std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> config( - new vespa::config::content::PersistenceConfigBuilder); +TEST_F(ChangedBucketOwnershipHandlerTest, ideal_state_aborts_are_configurable) { + auto config = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); config->abortOutdatedMutatingIdealStateOps = false; _handler->configure(std::move(config)); // Should not abort operation, even when ownership has changed. - CPPUNIT_ASSERT(!changeAbortsMessage<api::CreateBucketCommand>( - getBucketToAbort())); + expectChangeAbortsMessage<api::CreateBucketCommand>(false, getBucketToAbort()); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedPutOperation() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_put_operation) { document::Document::SP doc(_testDocRepo.createRandomDocumentAtLocation(1)); - CPPUNIT_ASSERT(changeAbortsMessage<api::PutCommand>( - getBucketToAbort(), doc, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage<api::PutCommand>( - getBucketToAllow(), doc, api::Timestamp(1234))); + expectChangeAbortsMessage<api::PutCommand>(true, getBucketToAbort(), doc, api::Timestamp(1234)); + expectChangeAbortsMessage<api::PutCommand>(false, getBucketToAllow(), doc, api::Timestamp(1234)); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedUpdateCommand() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_update_command) { const document::DocumentType* docType(_testDocRepo.getTypeRepo().getDocumentType("testdoctype1")); document::DocumentId docId("id:foo:testdoctype1::bar"); auto update(std::make_shared<document::DocumentUpdate>(_testDocRepo.getTypeRepo(), *docType, docId)); - CPPUNIT_ASSERT(changeAbortsMessage<api::UpdateCommand>(getBucketToAbort(), update, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage<api::UpdateCommand>(getBucketToAllow(), update, api::Timestamp(1234))); + expectChangeAbortsMessage<api::UpdateCommand>(true, getBucketToAbort(), update, api::Timestamp(1234)); + expectChangeAbortsMessage<api::UpdateCommand>(false, getBucketToAllow(), update, api::Timestamp(1234)); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveCommand() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_remove_command) { document::DocumentId docId("id:foo:testdoctype1::bar"); - CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>(getBucketToAbort(), docId, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>(getBucketToAllow(), docId, api::Timestamp(1234))); + expectChangeAbortsMessage<api::RemoveCommand>(true, getBucketToAbort(), docId, api::Timestamp(1234)); + expectChangeAbortsMessage<api::RemoveCommand>(false, getBucketToAllow(), docId, api::Timestamp(1234)); } -void -ChangedBucketOwnershipHandlerTest::testAbortOutdatedRevertCommand() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_revert_command) { std::vector<api::Timestamp> timestamps; - CPPUNIT_ASSERT(changeAbortsMessage<api::RevertCommand>( - getBucketToAbort(), timestamps)); - CPPUNIT_ASSERT(!changeAbortsMessage<api::RevertCommand>( - getBucketToAllow(), timestamps)); + expectChangeAbortsMessage<api::RevertCommand>(true, getBucketToAbort(), timestamps); + expectChangeAbortsMessage<api::RevertCommand>(false, getBucketToAllow(), timestamps); } -void -ChangedBucketOwnershipHandlerTest::testIdealStateAbortUpdatesMetric() -{ - CPPUNIT_ASSERT(changeAbortsMessage<api::SplitBucketCommand>( - getBucketToAbort())); - CPPUNIT_ASSERT_EQUAL( - uint64_t(1), - _handler->getMetrics().idealStateOpsAborted.getValue()); - CPPUNIT_ASSERT_EQUAL( - uint64_t(0), - _handler->getMetrics().externalLoadOpsAborted.getValue()); +TEST_F(ChangedBucketOwnershipHandlerTest, ideal_state_abort_updates_metric) { + expectChangeAbortsMessage<api::SplitBucketCommand>(true, getBucketToAbort()); + EXPECT_EQ(1, _handler->getMetrics().idealStateOpsAborted.getValue()); + EXPECT_EQ(0, _handler->getMetrics().externalLoadOpsAborted.getValue()); } -void -ChangedBucketOwnershipHandlerTest::testExternalLoadOpAbortUpdatesMetric() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, external_load_op_abort_updates_metric) { document::DocumentId docId("id:foo:testdoctype1::bar"); - CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>( - getBucketToAbort(), docId, api::Timestamp(1234))); - CPPUNIT_ASSERT_EQUAL( - uint64_t(0), - _handler->getMetrics().idealStateOpsAborted.getValue()); - CPPUNIT_ASSERT_EQUAL( - uint64_t(1), - _handler->getMetrics().externalLoadOpsAborted.getValue()); + expectChangeAbortsMessage<api::RemoveCommand>( + true, getBucketToAbort(), docId, api::Timestamp(1234)); + EXPECT_EQ(0, _handler->getMetrics().idealStateOpsAborted.getValue()); + EXPECT_EQ(1, _handler->getMetrics().externalLoadOpsAborted.getValue()); } -void -ChangedBucketOwnershipHandlerTest::testExternalLoadOpAbortsAreConfigurable() -{ - std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> config( - new vespa::config::content::PersistenceConfigBuilder); +TEST_F(ChangedBucketOwnershipHandlerTest, external_load_op_aborts_are_configurable) { + auto config = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); config->abortOutdatedMutatingExternalLoadOps = false; _handler->configure(std::move(config)); // Should not abort operation, even when ownership has changed. document::DocumentId docId("id:foo:testdoctype1::bar"); - CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>( - getBucketToAbort(), docId, api::Timestamp(1234))); + expectChangeAbortsMessage<api::RemoveCommand>( + false, getBucketToAbort(), docId, api::Timestamp(1234)); } -void -ChangedBucketOwnershipHandlerTest::testAbortCommandsWhenStorageNodeIsDown() -{ +TEST_F(ChangedBucketOwnershipHandlerTest, abort_commands_when_storage_node_is_down) { document::Document::SP doc(_testDocRepo.createRandomDocumentAtLocation(1)); - CPPUNIT_ASSERT(downAbortsMessage<api::PutCommand>( - getBucketToAllow(), doc, api::Timestamp(1234))); - CPPUNIT_ASSERT(downAbortsMessage<api::SetBucketStateCommand>( - getBucketToAllow(), api::SetBucketStateCommand::ACTIVE)); + expectDownAbortsMessage<api::PutCommand>( + true, getBucketToAllow(), doc, api::Timestamp(1234)); + expectDownAbortsMessage<api::SetBucketStateCommand>( + true, getBucketToAllow(), api::SetBucketStateCommand::ACTIVE); } } // storage diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 6af2733f3b9..b970e56343e 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -12,24 +12,17 @@ #include <tests/common/testhelper.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; +using namespace ::testing; namespace storage { -struct CommunicationManagerTest : public CppUnit::TestFixture { - void testSimple(); - void testDistPendingLimitConfigsArePropagatedToMessageBus(); - void testStorPendingLimitConfigsArePropagatedToMessageBus(); - void testCommandsAreDequeuedInFifoOrder(); - void testRepliesAreDequeuedInFifoOrder(); - void bucket_space_config_can_be_updated_live(); - void unmapped_bucket_space_documentapi_request_returns_error_reply(); - void unmapped_bucket_space_for_get_documentapi_request_returns_error_reply(); +struct CommunicationManagerTest : Test { static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60; @@ -45,23 +38,9 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { cmd->setPriority(priority); return cmd; } - - CPPUNIT_TEST_SUITE(CommunicationManagerTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testDistPendingLimitConfigsArePropagatedToMessageBus); - CPPUNIT_TEST(testStorPendingLimitConfigsArePropagatedToMessageBus); - CPPUNIT_TEST(testCommandsAreDequeuedInFifoOrder); - CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder); - CPPUNIT_TEST(bucket_space_config_can_be_updated_live); - CPPUNIT_TEST(unmapped_bucket_space_documentapi_request_returns_error_reply); - CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_error_reply); - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(CommunicationManagerTest); - -void CommunicationManagerTest::testSimple() -{ +TEST_F(CommunicationManagerTest, simple) { mbus::Slobrok slobrok; vdstestlib::DirConfig distConfig(getStandardConfig(false)); vdstestlib::DirConfig storConfig(getStandardConfig(true)); @@ -70,8 +49,8 @@ void CommunicationManagerTest::testSimple() addSlobrokConfig(distConfig, slobrok); addSlobrokConfig(storConfig, slobrok); - // Set up a "distributor" and a "storage" node with communication - // managers and a dummy storage link below we can use for testing. + // Set up a "distributor" and a "storage" node with communication + // managers and a dummy storage link below we can use for testing. TestServiceLayerApp storNode(storConfig.getConfigId()); TestDistributorApp distNode(distConfig.getConfigId()); @@ -89,30 +68,22 @@ void CommunicationManagerTest::testSimple() FastOS_Thread::Sleep(1000); // Send a message through from distributor to storage - std::shared_ptr<api::StorageCommand> cmd( - new api::GetCommand( - makeDocumentBucket(document::BucketId(0)), document::DocumentId("doc::mydoc"), "[all]")); - cmd->setAddress(api::StorageMessageAddress( - "storage", lib::NodeType::STORAGE, 1)); + auto cmd = std::make_shared<api::GetCommand>( + makeDocumentBucket(document::BucketId(0)), document::DocumentId("doc::mydoc"), "[all]"); + cmd->setAddress(api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 1)); distributorLink->sendUp(cmd); storageLink->waitForMessages(1, MESSAGE_WAIT_TIME_SEC); - CPPUNIT_ASSERT(storageLink->getNumCommands() > 0); - std::shared_ptr<api::StorageCommand> cmd2( - std::dynamic_pointer_cast<api::StorageCommand>( - storageLink->getCommand(0))); - CPPUNIT_ASSERT_EQUAL( - vespalib::string("doc::mydoc"), - static_cast<api::GetCommand&>(*cmd2).getDocumentId().toString()); - // Reply to the message + ASSERT_GT(storageLink->getNumCommands(), 0); + auto cmd2 = std::dynamic_pointer_cast<api::StorageCommand>(storageLink->getCommand(0)); + EXPECT_EQ("doc::mydoc", dynamic_cast<api::GetCommand&>(*cmd2).getDocumentId().toString()); + // Reply to the message std::shared_ptr<api::StorageReply> reply(cmd2->makeReply().release()); storageLink->sendUp(reply); storageLink->sendUp(reply); distributorLink->waitForMessages(1, MESSAGE_WAIT_TIME_SEC); - CPPUNIT_ASSERT(distributorLink->getNumCommands() > 0); - std::shared_ptr<api::GetReply> reply2( - std::dynamic_pointer_cast<api::GetReply>( - distributorLink->getCommand(0))); - CPPUNIT_ASSERT_EQUAL(false, reply2->wasFound()); + ASSERT_GT(distributorLink->getNumCommands(), 0); + auto reply2 = std::dynamic_pointer_cast<api::GetReply>(distributorLink->getCommand(0)); + EXPECT_FALSE(reply2->wasFound()); } void @@ -144,11 +115,11 @@ CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) // Outer type is RPCMessageBus, which wraps regular MessageBus. auto& mbus = commMgr.getMessageBus().getMessageBus(); if (isContentNode) { - CPPUNIT_ASSERT_EQUAL(uint32_t(12345), mbus.getMaxPendingCount()); - CPPUNIT_ASSERT_EQUAL(uint32_t(555666), mbus.getMaxPendingSize()); + EXPECT_EQ(12345, mbus.getMaxPendingCount()); + EXPECT_EQ(555666, mbus.getMaxPendingSize()); } else { - CPPUNIT_ASSERT_EQUAL(uint32_t(6789), mbus.getMaxPendingCount()); - CPPUNIT_ASSERT_EQUAL(uint32_t(777888), mbus.getMaxPendingSize()); + EXPECT_EQ(6789, mbus.getMaxPendingCount()); + EXPECT_EQ(777888, mbus.getMaxPendingSize()); } // Test live reconfig of limits. @@ -160,27 +131,21 @@ CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) commMgr.configure(std::move(liveCfg)); if (isContentNode) { - CPPUNIT_ASSERT_EQUAL(uint32_t(777777), mbus.getMaxPendingCount()); + EXPECT_EQ(777777, mbus.getMaxPendingCount()); } else { - CPPUNIT_ASSERT_EQUAL(uint32_t(999999), mbus.getMaxPendingCount()); + EXPECT_EQ(999999, mbus.getMaxPendingCount()); } } -void -CommunicationManagerTest::testDistPendingLimitConfigsArePropagatedToMessageBus() -{ +TEST_F(CommunicationManagerTest, dist_pending_limit_configs_are_propagated_to_message_bus) { doTestConfigPropagation(false); } -void -CommunicationManagerTest::testStorPendingLimitConfigsArePropagatedToMessageBus() -{ +TEST_F(CommunicationManagerTest, stor_pending_limit_configs_are_propagated_to_message_bus) { doTestConfigPropagation(true); } -void -CommunicationManagerTest::testCommandsAreDequeuedInFifoOrder() -{ +TEST_F(CommunicationManagerTest, commands_are_dequeued_in_fifo_order) { mbus::Slobrok slobrok; vdstestlib::DirConfig storConfig(getStandardConfig(true)); storConfig.getConfig("stor-server").set("node_index", "1"); @@ -207,15 +172,13 @@ CommunicationManagerTest::testCommandsAreDequeuedInFifoOrder() for (size_t i = 0; i < pris.size(); ++i) { // Casting is just to avoid getting mismatched values printed to the // output verbatim as chars. - CPPUNIT_ASSERT_EQUAL( + EXPECT_EQ( uint32_t(pris[i]), uint32_t(storageLink->getCommand(i)->getPriority())); } } -void -CommunicationManagerTest::testRepliesAreDequeuedInFifoOrder() -{ +TEST_F(CommunicationManagerTest, replies_are_dequeued_in_fifo_order) { mbus::Slobrok slobrok; vdstestlib::DirConfig storConfig(getStandardConfig(true)); storConfig.getConfig("stor-server").set("node_index", "1"); @@ -236,7 +199,7 @@ CommunicationManagerTest::testRepliesAreDequeuedInFifoOrder() // Want FIFO order for replies, not priority-sorted order. for (size_t i = 0; i < pris.size(); ++i) { - CPPUNIT_ASSERT_EQUAL( + EXPECT_EQ( uint32_t(pris[i]), uint32_t(storageLink->getCommand(i)->getPriority())); } @@ -303,7 +266,7 @@ BucketspacesConfigBuilder::Documenttype doc_type(vespalib::stringref name, vespa } -void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { +TEST_F(CommunicationManagerTest, bucket_space_config_can_be_updated_live) { CommunicationManagerFixture f; BucketspacesConfigBuilder config; config.documenttype.emplace_back(doc_type("foo", "default")); @@ -315,10 +278,10 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { f.bottom_link->waitForMessages(2, MESSAGE_WAIT_TIME_SEC); auto cmd1 = f.bottom_link->getCommand(0); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::global_space(), cmd1->getBucket().getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::global_space(), cmd1->getBucket().getBucketSpace()); auto cmd2 = f.bottom_link->getCommand(1); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), cmd2->getBucket().getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), cmd2->getBucket().getBucketSpace()); config.documenttype[1] = doc_type("bar", "default"); f.comm_mgr->updateBucketSpacesConfig(config); @@ -326,30 +289,30 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { f.bottom_link->waitForMessages(3, MESSAGE_WAIT_TIME_SEC); auto cmd3 = f.bottom_link->getCommand(2); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), cmd3->getBucket().getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), cmd3->getBucket().getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); + EXPECT_EQ(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } -void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns_error_reply() { +TEST_F(CommunicationManagerTest, unmapped_bucket_space_documentapi_request_returns_error_reply) { CommunicationManagerFixture f; BucketspacesConfigBuilder config; config.documenttype.emplace_back(doc_type("foo", "default")); f.comm_mgr->updateBucketSpacesConfig(config); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); + EXPECT_EQ(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("fluff")); - CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); + ASSERT_EQ(1, f.reply_handler.replies.size()); auto& reply = *f.reply_handler.replies[0]; - CPPUNIT_ASSERT(reply.hasErrors()); - CPPUNIT_ASSERT_EQUAL(static_cast<uint32_t>(api::ReturnCode::REJECTED), reply.getError(0).getCode()); + ASSERT_TRUE(reply.hasErrors()); + EXPECT_EQ(static_cast<uint32_t>(api::ReturnCode::REJECTED), reply.getError(0).getCode()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); + EXPECT_EQ(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } -void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_error_reply() { +TEST_F(CommunicationManagerTest, unmapped_bucket_space_for_get_documentapi_request_returns_error_reply) { CommunicationManagerFixture f; BucketspacesConfigBuilder config; @@ -357,11 +320,11 @@ void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request f.comm_mgr->updateBucketSpacesConfig(config); f.comm_mgr->handleMessage(f.documentapi_get_message_for_space("fluff")); - CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); + ASSERT_EQ(1, f.reply_handler.replies.size()); auto& reply = *f.reply_handler.replies[0]; - CPPUNIT_ASSERT(reply.hasErrors()); - CPPUNIT_ASSERT_EQUAL(static_cast<uint32_t>(api::ReturnCode::REJECTED), reply.getError(0).getCode()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); + ASSERT_TRUE(reply.hasErrors()); + EXPECT_EQ(static_cast<uint32_t>(api::ReturnCode::REJECTED), reply.getError(0).getCode()); + EXPECT_EQ(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } } // storage diff --git a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp index c10db0a1acd..5797074f892 100644 --- a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp +++ b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp @@ -3,26 +3,14 @@ #include <vespa/storage/storageserver/configurable_bucket_resolver.h> #include <vespa/document/base/documentid.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <cppunit/extensions/HelperMacros.h> +#include <vespa/vespalib/gtest/gtest.h> namespace storage { using document::DocumentId; +using namespace ::testing; -struct ConfigurableBucketResolverTest : CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(ConfigurableBucketResolverTest); - CPPUNIT_TEST(bucket_space_from_name_is_defined_for_default_space); - CPPUNIT_TEST(bucket_space_from_name_is_defined_for_global_space); - CPPUNIT_TEST(bucket_space_from_name_throws_exception_for_unknown_space); - CPPUNIT_TEST(name_from_bucket_space_is_defined_for_default_space); - CPPUNIT_TEST(name_from_bucket_space_is_defined_for_global_space); - CPPUNIT_TEST(name_from_bucket_space_throws_exception_for_unknown_space); - CPPUNIT_TEST(known_bucket_space_is_resolved_from_document_id); - CPPUNIT_TEST(unknown_bucket_space_in_id_throws_exception); - CPPUNIT_TEST(can_create_resolver_from_bucket_space_config); - CPPUNIT_TEST(legacy_document_id_without_document_type_maps_to_default_space); - CPPUNIT_TEST_SUITE_END(); - +struct ConfigurableBucketResolverTest : Test { using BucketSpaceMapping = ConfigurableBucketResolver::BucketSpaceMapping; BucketSpaceMapping create_simple_mapping() { @@ -38,74 +26,50 @@ struct ConfigurableBucketResolverTest : CppUnit::TestFixture { ConfigurableBucketResolver create_simple_resolver() { return ConfigurableBucketResolver(create_simple_mapping()); } - - void bucket_space_from_name_is_defined_for_default_space(); - void bucket_space_from_name_is_defined_for_global_space(); - void bucket_space_from_name_throws_exception_for_unknown_space(); - void name_from_bucket_space_is_defined_for_default_space(); - void name_from_bucket_space_is_defined_for_global_space(); - void name_from_bucket_space_throws_exception_for_unknown_space(); - void known_bucket_space_is_resolved_from_document_id(); - void unknown_bucket_space_in_id_throws_exception(); - void can_create_resolver_from_bucket_space_config(); - void legacy_document_id_without_document_type_maps_to_default_space(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(ConfigurableBucketResolverTest); - // TODO reduce overlap with FixedBucketSpacesTest -void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_default_space() { +TEST_F(ConfigurableBucketResolverTest, bucket_space_from_name_is_defined_for_default_space) { auto space = create_empty_resolver().bucketSpaceFromName("default"); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), space); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), space); } -void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_global_space() { +TEST_F(ConfigurableBucketResolverTest, bucket_space_from_name_is_defined_for_global_space) { auto space = create_empty_resolver().bucketSpaceFromName("global"); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::global_space(), space); + EXPECT_EQ(document::FixedBucketSpaces::global_space(), space); } -void ConfigurableBucketResolverTest::bucket_space_from_name_throws_exception_for_unknown_space() { - try { - create_empty_resolver().bucketSpaceFromName("bjarne"); - CPPUNIT_FAIL("Expected exception on unknown bucket space name"); - } catch (document::UnknownBucketSpaceException& e) { - } +TEST_F(ConfigurableBucketResolverTest, bucket_space_from_name_throws_exception_for_unknown_space) { + EXPECT_THROW(create_empty_resolver().bucketSpaceFromName("bjarne"), + document::UnknownBucketSpaceException); } -void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_default_space() { - CPPUNIT_ASSERT_EQUAL(vespalib::string("default"), - create_empty_resolver().nameFromBucketSpace(document::FixedBucketSpaces::default_space())); +TEST_F(ConfigurableBucketResolverTest, name_from_bucket_space_is_defined_for_default_space) { + EXPECT_EQ("default", create_empty_resolver().nameFromBucketSpace(document::FixedBucketSpaces::default_space())); } -void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_global_space() { - CPPUNIT_ASSERT_EQUAL(vespalib::string("global"), - create_empty_resolver().nameFromBucketSpace(document::FixedBucketSpaces::global_space())); +TEST_F(ConfigurableBucketResolverTest, name_from_bucket_space_is_defined_for_global_space) { + EXPECT_EQ("global", create_empty_resolver().nameFromBucketSpace(document::FixedBucketSpaces::global_space())); } -void ConfigurableBucketResolverTest::name_from_bucket_space_throws_exception_for_unknown_space() { - try { - create_empty_resolver().nameFromBucketSpace(document::BucketSpace(1234)); - CPPUNIT_FAIL("Expected exception on unknown bucket space value"); - } catch (document::UnknownBucketSpaceException& e) { - } +TEST_F(ConfigurableBucketResolverTest, name_from_bucket_space_throws_exception_for_unknown_space) { + EXPECT_THROW(create_empty_resolver().nameFromBucketSpace(document::BucketSpace(1234)), + document::UnknownBucketSpaceException); } -void ConfigurableBucketResolverTest::known_bucket_space_is_resolved_from_document_id() { +TEST_F(ConfigurableBucketResolverTest, known_bucket_space_is_resolved_from_document_id) { auto resolver = create_simple_resolver(); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), - resolver.bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), - resolver.bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::global_space(), - resolver.bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), + resolver.bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), + resolver.bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::global_space(), + resolver.bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); } -void ConfigurableBucketResolverTest::unknown_bucket_space_in_id_throws_exception() { - try { - create_simple_resolver().bucketFromId(DocumentId("id::bjarne::xyz")); - CPPUNIT_FAIL("Expected exception on unknown document type -> bucket space mapping"); - } catch (document::UnknownBucketSpaceException& e) { - } +TEST_F(ConfigurableBucketResolverTest, unknown_bucket_space_in_id_throws_exception) { + EXPECT_THROW(create_simple_resolver().bucketFromId(DocumentId("id::bjarne::xyz")), + document::UnknownBucketSpaceException); } using BucketSpacesConfigBuilder = vespa::config::content::core::BucketspacesConfigBuilder; @@ -121,24 +85,24 @@ BucketSpacesConfigBuilder::Documenttype make_doc_type(vespalib::stringref name, } -void ConfigurableBucketResolverTest::can_create_resolver_from_bucket_space_config() { +TEST_F(ConfigurableBucketResolverTest, can_create_resolver_from_bucket_space_config) { BucketSpacesConfigBuilder builder; builder.documenttype.emplace_back(make_doc_type("foo", "default")); builder.documenttype.emplace_back(make_doc_type("bar", "global")); builder.documenttype.emplace_back(make_doc_type("baz", "global")); auto resolver = ConfigurableBucketResolver::from_config(builder); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), - resolver->bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::global_space(), - resolver->bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::global_space(), - resolver->bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), + resolver->bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::global_space(), + resolver->bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::global_space(), + resolver->bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); } -void ConfigurableBucketResolverTest::legacy_document_id_without_document_type_maps_to_default_space() { +TEST_F(ConfigurableBucketResolverTest, legacy_document_id_without_document_type_maps_to_default_space) { auto resolver = create_simple_resolver(); - CPPUNIT_ASSERT_EQUAL(document::FixedBucketSpaces::default_space(), - resolver.bucketFromId(DocumentId("userdoc:baz:1234:baz")).getBucketSpace()); + EXPECT_EQ(document::FixedBucketSpaces::default_space(), + resolver.bucketFromId(DocumentId("userdoc:baz:1234:baz")).getBucketSpace()); } } diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 7f8b1b8f34a..c879f7d2779 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cppunit/extensions/HelperMacros.h> #include <vespa/config/subscription/configuri.h> #include <vespa/document/base/testdocrepo.h> #include <vespa/document/bucket/bucketidfactory.h> @@ -17,7 +16,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> using document::Bucket; using document::BucketId; @@ -29,6 +28,7 @@ using document::DocumentId; using document::DocumentTypeRepo; using document::readDocumenttypesConfig; using document::test::makeDocumentBucket; +using namespace ::testing; namespace storage { @@ -58,8 +58,7 @@ struct MockBucketResolver : public BucketResolver { } }; -struct DocumentApiConverterTest : public CppUnit::TestFixture -{ +struct DocumentApiConverterTest : Test { std::shared_ptr<MockBucketResolver> _bucketResolver; std::unique_ptr<DocumentApiConverter> _converter; const std::shared_ptr<const DocumentTypeRepo> _repo; @@ -67,20 +66,19 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture DocumentApiConverterTest() : _bucketResolver(std::make_shared<MockBucketResolver>()), - _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig( - TEST_PATH("config-doctypes.cfg")))), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("../config-doctypes.cfg"))), _html_type(*_repo->getDocumentType("text/html")) { } - void setUp() override { - _converter.reset(new DocumentApiConverter("raw:", _bucketResolver)); + void SetUp() override { + _converter = std::make_unique<DocumentApiConverter>("raw:", _bucketResolver); }; template <typename DerivedT, typename BaseT> std::unique_ptr<DerivedT> dynamic_unique_ptr_cast(std::unique_ptr<BaseT> base) { auto derived = dynamic_cast<DerivedT*>(base.get()); - CPPUNIT_ASSERT(derived); + assert(derived); base.release(); return std::unique_ptr<DerivedT>(derived); } @@ -103,162 +101,119 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture auto result = _converter->toDocumentAPI(cmd); return dynamic_unique_ptr_cast<T>(std::move(result)); } - - void testPut(); - void testForwardedPut(); - void testUpdate(); - void testRemove(); - void testGet(); - void testCreateVisitor(); - void testCreateVisitorHighTimeout(); - void testCreateVisitorReplyNotReady(); - void testCreateVisitorReplyLastBucket(); - void testDestroyVisitor(); - void testVisitorInfo(); - void testStatBucket(); - void testGetBucketList(); - void testRemoveLocation(); - void can_replace_bucket_resolver_after_construction(); - - CPPUNIT_TEST_SUITE(DocumentApiConverterTest); - CPPUNIT_TEST(testPut); - CPPUNIT_TEST(testForwardedPut); - CPPUNIT_TEST(testUpdate); - CPPUNIT_TEST(testRemove); - CPPUNIT_TEST(testGet); - CPPUNIT_TEST(testCreateVisitor); - CPPUNIT_TEST(testCreateVisitorHighTimeout); - CPPUNIT_TEST(testCreateVisitorReplyNotReady); - CPPUNIT_TEST(testCreateVisitorReplyLastBucket); - CPPUNIT_TEST(testDestroyVisitor); - CPPUNIT_TEST(testVisitorInfo); - CPPUNIT_TEST(testStatBucket); - CPPUNIT_TEST(testGetBucketList); - CPPUNIT_TEST(testRemoveLocation); - CPPUNIT_TEST(can_replace_bucket_resolver_after_construction); - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(DocumentApiConverterTest); - -void DocumentApiConverterTest::testPut() -{ +TEST_F(DocumentApiConverterTest, put) { auto doc = std::make_shared<Document>(_html_type, defaultDocId); documentapi::PutDocumentMessage putmsg(doc); putmsg.setTimestamp(1234); auto cmd = toStorageAPI<api::PutCommand>(putmsg); - CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); - CPPUNIT_ASSERT(cmd->getDocument().get() == doc.get()); + EXPECT_EQ(defaultBucket, cmd->getBucket()); + ASSERT_EQ(cmd->getDocument().get(), doc.get()); std::unique_ptr<mbus::Reply> reply = putmsg.createReply(); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); toStorageAPI<api::PutReply>(*reply, *cmd); auto mbusPut = toDocumentAPI<documentapi::PutDocumentMessage>(*cmd); - CPPUNIT_ASSERT(mbusPut->getDocumentSP().get() == doc.get()); - CPPUNIT_ASSERT(mbusPut->getTimestamp() == 1234); + ASSERT_EQ(mbusPut->getDocumentSP().get(), doc.get()); + EXPECT_EQ(mbusPut->getTimestamp(), 1234); } -void DocumentApiConverterTest::testForwardedPut() -{ +TEST_F(DocumentApiConverterTest, forwarded_put) { auto doc = std::make_shared<Document>(_html_type, DocumentId(DocIdString("test", "test"))); - documentapi::PutDocumentMessage* putmsg = new documentapi::PutDocumentMessage(doc); - std::unique_ptr<mbus::Reply> reply(((documentapi::DocumentMessage*)putmsg)->createReply()); - reply->setMessage(std::unique_ptr<mbus::Message>(putmsg)); + auto putmsg = std::make_unique<documentapi::PutDocumentMessage>(doc); + auto* putmsg_raw = putmsg.get(); + std::unique_ptr<mbus::Reply> reply(putmsg->createReply()); + reply->setMessage(std::unique_ptr<mbus::Message>(putmsg.release())); - auto cmd = toStorageAPI<api::PutCommand>(*putmsg); + auto cmd = toStorageAPI<api::PutCommand>(*putmsg_raw); cmd->setTimestamp(1234); auto rep = dynamic_unique_ptr_cast<api::PutReply>(cmd->makeReply()); _converter->transferReplyState(*rep, *reply); } -void DocumentApiConverterTest::testUpdate() -{ +TEST_F(DocumentApiConverterTest, update) { auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, defaultDocId); documentapi::UpdateDocumentMessage updateMsg(update); updateMsg.setOldTimestamp(1234); updateMsg.setNewTimestamp(5678); auto updateCmd = toStorageAPI<api::UpdateCommand>(updateMsg); - CPPUNIT_ASSERT_EQUAL(defaultBucket, updateCmd->getBucket()); - CPPUNIT_ASSERT_EQUAL(update.get(), updateCmd->getUpdate().get()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(1234), updateCmd->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(5678), updateCmd->getTimestamp()); + EXPECT_EQ(defaultBucket, updateCmd->getBucket()); + ASSERT_EQ(update.get(), updateCmd->getUpdate().get()); + EXPECT_EQ(api::Timestamp(1234), updateCmd->getOldTimestamp()); + EXPECT_EQ(api::Timestamp(5678), updateCmd->getTimestamp()); auto mbusReply = updateMsg.createReply(); - CPPUNIT_ASSERT(mbusReply.get()); + ASSERT_TRUE(mbusReply.get()); toStorageAPI<api::UpdateReply>(*mbusReply, *updateCmd); auto mbusUpdate = toDocumentAPI<documentapi::UpdateDocumentMessage>(*updateCmd); - CPPUNIT_ASSERT((&mbusUpdate->getDocumentUpdate()) == update.get()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); + ASSERT_EQ((&mbusUpdate->getDocumentUpdate()), update.get()); + EXPECT_EQ(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); + EXPECT_EQ(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); } -void DocumentApiConverterTest::testRemove() -{ +TEST_F(DocumentApiConverterTest, remove) { documentapi::RemoveDocumentMessage removemsg(defaultDocId); auto cmd = toStorageAPI<api::RemoveCommand>(removemsg); - CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); - CPPUNIT_ASSERT_EQUAL(defaultDocId, cmd->getDocumentId()); + EXPECT_EQ(defaultBucket, cmd->getBucket()); + EXPECT_EQ(defaultDocId, cmd->getDocumentId()); std::unique_ptr<mbus::Reply> reply = removemsg.createReply(); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); toStorageAPI<api::RemoveReply>(*reply, *cmd); auto mbusRemove = toDocumentAPI<documentapi::RemoveDocumentMessage>(*cmd); - CPPUNIT_ASSERT_EQUAL(defaultDocId, mbusRemove->getDocumentId()); + EXPECT_EQ(defaultDocId, mbusRemove->getDocumentId()); } -void DocumentApiConverterTest::testGet() -{ +TEST_F(DocumentApiConverterTest, get) { documentapi::GetDocumentMessage getmsg(defaultDocId, "foo bar"); auto cmd = toStorageAPI<api::GetCommand>(getmsg); - CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); - CPPUNIT_ASSERT_EQUAL(defaultDocId, cmd->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo bar"), cmd->getFieldSet()); + EXPECT_EQ(defaultBucket, cmd->getBucket()); + EXPECT_EQ(defaultDocId, cmd->getDocumentId()); + EXPECT_EQ("foo bar", cmd->getFieldSet()); } -void DocumentApiConverterTest::testCreateVisitor() -{ +TEST_F(DocumentApiConverterTest, create_visitor) { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); cv.setBucketSpace(defaultSpaceName); cv.setTimeRemaining(123456); auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); - CPPUNIT_ASSERT_EQUAL(defaultBucketSpace, cmd->getBucket().getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), cmd->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), cmd->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), cmd->getDataDestination()); - CPPUNIT_ASSERT_EQUAL(123456u, cmd->getTimeout()); + EXPECT_EQ(defaultBucketSpace, cmd->getBucket().getBucketSpace()); + EXPECT_EQ("mylib", cmd->getLibraryName()); + EXPECT_EQ("myinstance", cmd->getInstanceId()); + EXPECT_EQ("control-dest", cmd->getControlDestination()); + EXPECT_EQ("data-dest", cmd->getDataDestination()); + EXPECT_EQ(123456u, cmd->getTimeout()); auto msg = toDocumentAPI<documentapi::CreateVisitorMessage>(*cmd); - CPPUNIT_ASSERT_EQUAL(defaultSpaceName, msg->getBucketSpace()); + EXPECT_EQ(defaultSpaceName, msg->getBucketSpace()); } -void DocumentApiConverterTest::testCreateVisitorHighTimeout() -{ +TEST_F(DocumentApiConverterTest, create_visitor_high_timeout) { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); cv.setTimeRemaining((uint64_t)std::numeric_limits<uint32_t>::max() + 1); // Will be INT_MAX auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); - CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), cmd->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), cmd->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), cmd->getDataDestination()); - CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), cmd->getTimeout()); + EXPECT_EQ("mylib", cmd->getLibraryName()); + EXPECT_EQ("myinstance", cmd->getInstanceId()); + EXPECT_EQ("control-dest", cmd->getControlDestination()); + EXPECT_EQ("data-dest", cmd->getDataDestination()); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), cmd->getTimeout()); } -void DocumentApiConverterTest::testCreateVisitorReplyNotReady() -{ +TEST_F(DocumentApiConverterTest, create_visitor_reply_not_ready) { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); @@ -267,14 +222,13 @@ void DocumentApiConverterTest::testCreateVisitorReplyNotReady() std::unique_ptr<documentapi::CreateVisitorReply> reply( dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release())); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); _converter->transferReplyState(cvr, *reply); - CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::ERROR_NODE_NOT_READY, reply->getError(0).getCode()); - CPPUNIT_ASSERT_EQUAL(document::BucketId(std::numeric_limits<int>::max()), reply->getLastBucket()); + EXPECT_EQ(documentapi::DocumentProtocol::ERROR_NODE_NOT_READY, reply->getError(0).getCode()); + EXPECT_EQ(document::BucketId(std::numeric_limits<int>::max()), reply->getLastBucket()); } -void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() -{ +TEST_F(DocumentApiConverterTest, create_visitor_reply_last_bucket) { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); @@ -283,75 +237,66 @@ void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() std::unique_ptr<documentapi::CreateVisitorReply> reply( dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release())); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); _converter->transferReplyState(cvr, *reply); - CPPUNIT_ASSERT_EQUAL(document::BucketId(123), reply->getLastBucket()); + EXPECT_EQ(document::BucketId(123), reply->getLastBucket()); } -void DocumentApiConverterTest::testDestroyVisitor() -{ +TEST_F(DocumentApiConverterTest, destroy_visitor) { documentapi::DestroyVisitorMessage cv("myinstance"); auto cmd = toStorageAPI<api::DestroyVisitorCommand>(cv); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); + EXPECT_EQ("myinstance", cmd->getInstanceId()); } -void -DocumentApiConverterTest::testVisitorInfo() -{ +TEST_F(DocumentApiConverterTest, visitor_info) { api::VisitorInfoCommand vicmd; std::vector<api::VisitorInfoCommand::BucketTimestampPair> bucketsCompleted; - bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 1), 0)); - bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 2), 0)); - bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 4), 0)); + bucketsCompleted.emplace_back(document::BucketId(16, 1), 0); + bucketsCompleted.emplace_back(document::BucketId(16, 2), 0); + bucketsCompleted.emplace_back(document::BucketId(16, 4), 0); vicmd.setBucketsCompleted(bucketsCompleted); auto mbusvi = toDocumentAPI<documentapi::VisitorInfoMessage>(vicmd); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), mbusvi->getFinishedBuckets()[0]); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), mbusvi->getFinishedBuckets()[1]); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 4), mbusvi->getFinishedBuckets()[2]); + EXPECT_EQ(document::BucketId(16, 1), mbusvi->getFinishedBuckets()[0]); + EXPECT_EQ(document::BucketId(16, 2), mbusvi->getFinishedBuckets()[1]); + EXPECT_EQ(document::BucketId(16, 4), mbusvi->getFinishedBuckets()[2]); std::unique_ptr<mbus::Reply> reply = mbusvi->createReply(); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); toStorageAPI<api::VisitorInfoReply>(*reply, vicmd); } -void -DocumentApiConverterTest::testStatBucket() -{ +TEST_F(DocumentApiConverterTest, stat_bucket) { documentapi::StatBucketMessage msg(BucketId(123), ""); msg.setBucketSpace(defaultSpaceName); auto cmd = toStorageAPI<api::StatBucketCommand>(msg); - CPPUNIT_ASSERT_EQUAL(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); + EXPECT_EQ(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); auto mbusMsg = toDocumentAPI<documentapi::StatBucketMessage>(*cmd); - CPPUNIT_ASSERT_EQUAL(BucketId(123), mbusMsg->getBucketId()); - CPPUNIT_ASSERT_EQUAL(defaultSpaceName, mbusMsg->getBucketSpace()); + EXPECT_EQ(BucketId(123), mbusMsg->getBucketId()); + EXPECT_EQ(defaultSpaceName, mbusMsg->getBucketSpace()); } -void -DocumentApiConverterTest::testGetBucketList() -{ +TEST_F(DocumentApiConverterTest, get_bucket_list) { documentapi::GetBucketListMessage msg(BucketId(123)); msg.setBucketSpace(defaultSpaceName); auto cmd = toStorageAPI<api::GetBucketListCommand>(msg); - CPPUNIT_ASSERT_EQUAL(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); + EXPECT_EQ(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); } -void -DocumentApiConverterTest::testRemoveLocation() -{ +TEST_F(DocumentApiConverterTest, remove_location) { document::BucketIdFactory factory; document::select::Parser parser(*_repo, factory); documentapi::RemoveLocationMessage msg(factory, parser, "id.group == \"mygroup\""); msg.setBucketSpace(defaultSpaceName); auto cmd = toStorageAPI<api::RemoveLocationCommand>(msg); - CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); + EXPECT_EQ(defaultBucket, cmd->getBucket()); } namespace { @@ -367,16 +312,16 @@ struct ReplacementMockBucketResolver : public MockBucketResolver { } -void DocumentApiConverterTest::can_replace_bucket_resolver_after_construction() { +TEST_F(DocumentApiConverterTest, can_replace_bucket_resolver_after_construction) { documentapi::GetDocumentMessage get_msg(DocumentId("id::testdoctype1::baz"), "foo bar"); auto cmd = toStorageAPI<api::GetCommand>(get_msg); - CPPUNIT_ASSERT_EQUAL(BucketSpace(0), cmd->getBucket().getBucketSpace()); + EXPECT_EQ(BucketSpace(0), cmd->getBucket().getBucketSpace()); _converter->setBucketResolver(std::make_shared<ReplacementMockBucketResolver>()); cmd = toStorageAPI<api::GetCommand>(get_msg); - CPPUNIT_ASSERT_EQUAL(defaultBucketSpace, cmd->getBucket().getBucketSpace()); + EXPECT_EQ(defaultBucketSpace, cmd->getBucket().getBucketSpace()); } } diff --git a/storage/src/tests/storageserver/fnet_listener_test.cpp b/storage/src/tests/storageserver/fnet_listener_test.cpp index d40b230d725..b9f2ca74df3 100644 --- a/storage/src/tests/storageserver/fnet_listener_test.cpp +++ b/storage/src/tests/storageserver/fnet_listener_test.cpp @@ -8,43 +8,20 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/cppunit/dirconfig.hpp> #include <vespa/messagebus/testlib/slobrok.h> #include <tests/common/testhelper.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vector> namespace storage { using document::FixedBucketSpaces; +using namespace ::testing; -class FNetListenerTest : public CppUnit::TestFixture { -public: - CPPUNIT_TEST_SUITE(FNetListenerTest); - CPPUNIT_TEST(baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle); - CPPUNIT_TEST(set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle); - CPPUNIT_TEST(compressed_bundle_is_transparently_uncompressed); - CPPUNIT_TEST(set_distribution_rpc_is_immediately_failed_if_listener_is_closed); - CPPUNIT_TEST(overly_large_uncompressed_bundle_size_parameter_returns_rpc_error); - CPPUNIT_TEST(mismatching_uncompressed_bundle_size_parameter_returns_rpc_error); - CPPUNIT_TEST(true_deferred_activation_flag_can_be_roundtrip_encoded); - CPPUNIT_TEST(false_deferred_activation_flag_can_be_roundtrip_encoded); - CPPUNIT_TEST(activate_cluster_state_version_rpc_enqueues_command_with_version); - CPPUNIT_TEST_SUITE_END(); - - void baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle(); - void set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle(); - void compressed_bundle_is_transparently_uncompressed(); - void set_distribution_rpc_is_immediately_failed_if_listener_is_closed(); - void overly_large_uncompressed_bundle_size_parameter_returns_rpc_error(); - void mismatching_uncompressed_bundle_size_parameter_returns_rpc_error(); - void true_deferred_activation_flag_can_be_roundtrip_encoded(); - void false_deferred_activation_flag_can_be_roundtrip_encoded(); - void activate_cluster_state_version_rpc_enqueues_command_with_version(); +struct FNetListenerTest : Test { }; -CPPUNIT_TEST_SUITE_REGISTRATION(FNetListenerTest); - namespace { struct MockOperationEnqueuer : MessageEnqueuer { @@ -113,11 +90,11 @@ struct SetStateFixture : FixtureBase { } void assert_enqueued_operation_has_bundle(const lib::ClusterStateBundle& expectedBundle) { - CPPUNIT_ASSERT(bound_request != nullptr); - CPPUNIT_ASSERT(request_is_detached); - CPPUNIT_ASSERT_EQUAL(size_t(1), enqueuer._enqueued.size()); + ASSERT_TRUE(bound_request != nullptr); + ASSERT_TRUE(request_is_detached); + ASSERT_EQ(1, enqueuer._enqueued.size()); auto& state_request = dynamic_cast<const api::SetSystemStateCommand&>(*enqueuer._enqueued[0]); - CPPUNIT_ASSERT_EQUAL(expectedBundle, state_request.getClusterStateBundle()); + ASSERT_EQ(expectedBundle, state_request.getClusterStateBundle()); } void assert_request_received_and_propagated(const lib::ClusterStateBundle& bundle) { @@ -128,9 +105,9 @@ struct SetStateFixture : FixtureBase { void assert_request_returns_error_response(RPCRequestWrapper::ErrorCode error_code) { fnet_listener->RPC_setDistributionStates(bound_request); - CPPUNIT_ASSERT(!request_is_detached); - CPPUNIT_ASSERT(bound_request->IsError()); - CPPUNIT_ASSERT_EQUAL(static_cast<uint32_t>(error_code), bound_request->GetErrorCode()); + ASSERT_FALSE(request_is_detached); + ASSERT_TRUE(bound_request->IsError()); + ASSERT_EQ(static_cast<uint32_t>(error_code), bound_request->GetErrorCode()); } lib::ClusterStateBundle dummy_baseline_bundle() const { @@ -157,14 +134,14 @@ vespalib::string make_compressable_state_string() { } // anon namespace -void FNetListenerTest::baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle() { +TEST_F(FNetListenerTest, baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle) { SetStateFixture f; auto baseline = f.dummy_baseline_bundle(); f.assert_request_received_and_propagated(baseline); } -void FNetListenerTest::set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle() { +TEST_F(FNetListenerTest, set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle) { SetStateFixture f; lib::ClusterStateBundle spaces_bundle( lib::ClusterState("version:123 distributor:3 storage:3"), @@ -174,47 +151,47 @@ void FNetListenerTest::set_distribution_states_rpc_with_derived_enqueues_command f.assert_request_received_and_propagated(spaces_bundle); } -void FNetListenerTest::compressed_bundle_is_transparently_uncompressed() { +TEST_F(FNetListenerTest, compressed_bundle_is_transparently_uncompressed) { SetStateFixture f; auto state_str = make_compressable_state_string(); lib::ClusterStateBundle compressable_bundle{lib::ClusterState(state_str)}; f.create_request(compressable_bundle); // First verify that the bundle is sent in compressed form - CPPUNIT_ASSERT(f.bound_request->GetParams()->GetValue(2)._data._len < state_str.size()); + ASSERT_LT(f.bound_request->GetParams()->GetValue(2)._data._len, state_str.size()); // Ensure we uncompress it to the original form f.fnet_listener->RPC_setDistributionStates(f.bound_request); f.assert_enqueued_operation_has_bundle(compressable_bundle); } -void FNetListenerTest::set_distribution_rpc_is_immediately_failed_if_listener_is_closed() { +TEST_F(FNetListenerTest, set_distribution_rpc_is_immediately_failed_if_listener_is_closed) { SetStateFixture f; f.create_request(f.dummy_baseline_bundle()); f.fnet_listener->close(); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN); } -void FNetListenerTest::overly_large_uncompressed_bundle_size_parameter_returns_rpc_error() { +TEST_F(FNetListenerTest, overly_large_uncompressed_bundle_size_parameter_returns_rpc_error) { SetStateFixture f; auto encoded_bundle = f.codec.encode(f.dummy_baseline_bundle()); f.bind_request_params(encoded_bundle, FNetListener::StateBundleMaxUncompressedSize + 1); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_BAD_REQUEST); } -void FNetListenerTest::mismatching_uncompressed_bundle_size_parameter_returns_rpc_error() { +TEST_F(FNetListenerTest, mismatching_uncompressed_bundle_size_parameter_returns_rpc_error) { SetStateFixture f; auto encoded_bundle = f.codec.encode(f.dummy_baseline_bundle()); f.bind_request_params(encoded_bundle, encoded_bundle._buffer->getDataLen() + 100); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_BAD_REQUEST); } -void FNetListenerTest::true_deferred_activation_flag_can_be_roundtrip_encoded() { +TEST_F(FNetListenerTest, true_deferred_activation_flag_can_be_roundtrip_encoded) { SetStateFixture f; f.assert_request_received_and_propagated(f.dummy_baseline_bundle_with_deferred_activation(true)); } -void FNetListenerTest::false_deferred_activation_flag_can_be_roundtrip_encoded() { +TEST_F(FNetListenerTest, false_deferred_activation_flag_can_be_roundtrip_encoded) { SetStateFixture f; f.assert_request_received_and_propagated(f.dummy_baseline_bundle_with_deferred_activation(false)); } @@ -238,11 +215,11 @@ struct ActivateStateFixture : FixtureBase { } void assert_enqueued_operation_has_activate_version(uint32_t version) { - CPPUNIT_ASSERT(bound_request != nullptr); - CPPUNIT_ASSERT(request_is_detached); - CPPUNIT_ASSERT_EQUAL(size_t(1), enqueuer._enqueued.size()); + ASSERT_TRUE(bound_request != nullptr); + ASSERT_TRUE(request_is_detached); + ASSERT_EQ(1, enqueuer._enqueued.size()); auto& state_request = dynamic_cast<const api::ActivateClusterStateVersionCommand&>(*enqueuer._enqueued[0]); - CPPUNIT_ASSERT_EQUAL(version, state_request.version()); + ASSERT_EQ(version, state_request.version()); } void assert_request_received_and_propagated(uint32_t activate_version) { @@ -252,7 +229,7 @@ struct ActivateStateFixture : FixtureBase { } }; -void FNetListenerTest::activate_cluster_state_version_rpc_enqueues_command_with_version() { +TEST_F(FNetListenerTest, activate_cluster_state_version_rpc_enqueues_command_with_version) { ActivateStateFixture f; f.assert_request_received_and_propagated(1234567); } diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 5815bbd67c8..30ad9b58e9f 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cppunit/extensions/HelperMacros.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <tests/common/testhelper.h> @@ -12,6 +11,7 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/gtest/gtest.h> #include <unordered_set> #include <memory> #include <iterator> @@ -23,6 +23,7 @@ using namespace document; using namespace storage::api; using document::test::makeDocumentBucket; +using namespace ::testing; namespace storage { @@ -100,16 +101,16 @@ struct MergeBuilder { bool source_only = (_source_only.find(node) != _source_only.end()); n.emplace_back(node, source_only); } - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(_bucket), n, _maxTimestamp, - _clusterStateVersion, _chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(_bucket), n, _maxTimestamp, + _clusterStateVersion, _chain); StorageMessageAddress address("storage", lib::NodeType::STORAGE, _nodes[0]); cmd->setAddress(address); return cmd; } }; -MergeBuilder::~MergeBuilder() {} +MergeBuilder::~MergeBuilder() = default; std::shared_ptr<api::SetSystemStateCommand> makeSystemStateCmd(const std::string& state) @@ -120,70 +121,9 @@ makeSystemStateCmd(const std::string& state) } // anon ns -class MergeThrottlerTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(MergeThrottlerTest); - CPPUNIT_TEST(testMergesConfig); - CPPUNIT_TEST(testChain); - CPPUNIT_TEST(testWithSourceOnlyNode); - CPPUNIT_TEST(test42DistributorBehavior); - CPPUNIT_TEST(test42DistributorBehaviorDoesNotTakeOwnership); - CPPUNIT_TEST(testEndOfChainExecutionDoesNotTakeOwnership); - CPPUNIT_TEST(testResendHandling); - CPPUNIT_TEST(testPriorityQueuing); - CPPUNIT_TEST(testCommandInQueueDuplicateOfKnownMerge); - CPPUNIT_TEST(testInvalidReceiverNode); - CPPUNIT_TEST(testForwardQueuedMerge); - CPPUNIT_TEST(testExecuteQueuedMerge); - CPPUNIT_TEST(testFlush); - CPPUNIT_TEST(testUnseenMergeWithNodeInChain); - CPPUNIT_TEST(testMergeWithNewerClusterStateFlushesOutdatedQueued); - CPPUNIT_TEST(testUpdatedClusterStateFlushesOutdatedQueued); - CPPUNIT_TEST(test42MergesDoNotTriggerFlush); - CPPUNIT_TEST(testOutdatedClusterStateMergesAreRejectedOnArrival); - CPPUNIT_TEST(testUnknownMergeWithSelfInChain); - CPPUNIT_TEST(testBusyReturnedOnFullQueue); - CPPUNIT_TEST(testBrokenCycle); - CPPUNIT_TEST(testGetBucketDiffCommandNotInActiveSetIsRejected); - CPPUNIT_TEST(testApplyBucketDiffCommandNotInActiveSetIsRejected); - CPPUNIT_TEST(testNewClusterStateAbortsAllOutdatedActiveMerges); - CPPUNIT_TEST(backpressure_busy_bounces_merges_for_configured_duration); - CPPUNIT_TEST(source_only_merges_are_not_affected_by_backpressure); - CPPUNIT_TEST(backpressure_evicts_all_queued_merges); - CPPUNIT_TEST_SUITE_END(); -public: - void setUp() override; - void tearDown() override; - - void testMergesConfig(); - void testChain(); - void testWithSourceOnlyNode(); - void test42DistributorBehavior(); - void test42DistributorBehaviorDoesNotTakeOwnership(); - void testEndOfChainExecutionDoesNotTakeOwnership(); - void testResendHandling(); - void testPriorityQueuing(); - void testCommandInQueueDuplicateOfKnownMerge(); - void testInvalidReceiverNode(); - void testForwardQueuedMerge(); - void testExecuteQueuedMerge(); - void testFlush(); - void testUnseenMergeWithNodeInChain(); - void testMergeWithNewerClusterStateFlushesOutdatedQueued(); - void testUpdatedClusterStateFlushesOutdatedQueued(); - void test42MergesDoNotTriggerFlush(); - void testOutdatedClusterStateMergesAreRejectedOnArrival(); - void testUnknownMergeWithSelfInChain(); - void testBusyReturnedOnFullQueue(); - void testBrokenCycle(); - void testGetBucketDiffCommandNotInActiveSetIsRejected(); - void testApplyBucketDiffCommandNotInActiveSetIsRejected(); - void testNewClusterStateAbortsAllOutdatedActiveMerges(); - void backpressure_busy_bounces_merges_for_configured_duration(); - void source_only_merges_are_not_affected_by_backpressure(); - void backpressure_evicts_all_queued_merges(); -private: - static const int _storageNodeCount = 3; - static const int _messageWaitTime = 100; +struct MergeThrottlerTest : Test { + static constexpr int _storageNodeCount = 3; + static constexpr int _messageWaitTime = 100; // Using n storage node links and dummy servers std::vector<std::shared_ptr<DummyStorageLink> > _topLinks; @@ -191,6 +131,12 @@ private: std::vector<MergeThrottler*> _throttlers; std::vector<DummyStorageLink*> _bottomLinks; + MergeThrottlerTest(); + ~MergeThrottlerTest(); + + void SetUp() override; + void TearDown() override; + api::MergeBucketCommand::SP sendMerge(const MergeBuilder&); void sendAndExpectReply( @@ -201,13 +147,11 @@ private: void fill_throttler_queue_with_n_commands(uint16_t throttler_index, size_t queued_count); }; -const int MergeThrottlerTest::_storageNodeCount; -const int MergeThrottlerTest::_messageWaitTime; - -CPPUNIT_TEST_SUITE_REGISTRATION(MergeThrottlerTest); +MergeThrottlerTest::MergeThrottlerTest() = default; +MergeThrottlerTest::~MergeThrottlerTest() = default; void -MergeThrottlerTest::setUp() +MergeThrottlerTest::SetUp() { vdstestlib::DirConfig config(getStandardConfig(true)); @@ -234,7 +178,7 @@ MergeThrottlerTest::setUp() } void -MergeThrottlerTest::tearDown() +MergeThrottlerTest::TearDown() { for (std::size_t i = 0; i < _topLinks.size(); ++i) { if (_topLinks[i]->getState() == StorageLink::OPENED) { @@ -293,20 +237,16 @@ void waitUntilMergeQueueIs(MergeThrottler& throttler, std::size_t sz, int timeou // Extremely simple test that just checks that (min|max)_merges_per_node // under the stor-server config gets propagated to all the nodes -void -MergeThrottlerTest::testMergesConfig() -{ +TEST_F(MergeThrottlerTest, merges_config) { for (int i = 0; i < _storageNodeCount; ++i) { - CPPUNIT_ASSERT_EQUAL(uint32_t(25), _throttlers[i]->getThrottlePolicy().getMaxPendingCount()); - CPPUNIT_ASSERT_EQUAL(std::size_t(20), _throttlers[i]->getMaxQueueSize()); + EXPECT_EQ(25, _throttlers[i]->getThrottlePolicy().getMaxPendingCount()); + EXPECT_EQ(20, _throttlers[i]->getMaxQueueSize()); } } // Test that a distributor sending a merge to the lowest-index storage // node correctly invokes a merge forwarding chain and subsequent unwind. -void -MergeThrottlerTest::testChain() -{ +TEST_F(MergeThrottlerTest, chain) { uint16_t indices[_storageNodeCount]; for (int i = 0; i < _storageNodeCount; ++i) { indices[i] = i; @@ -342,31 +282,31 @@ MergeThrottlerTest::testChain() if (i == executorNode) { fwdToExec = fwd; } - CPPUNIT_ASSERT_EQUAL(uint16_t(i), _servers[i]->getIndex()); + ASSERT_EQ(i, _servers[i]->getIndex()); // No matter the node order, command is always sent to node 0 -> 1 -> 2 etc _topLinks[i]->sendDown(fwd); _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Forwarded merge should not be sent down. Should not be necessary // to lock throttler here, since it should be sleeping like a champion - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[i]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[i]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[i]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[i]->getNumCommands()); + ASSERT_EQ(1, _topLinks[i]->getNumReplies()); + ASSERT_EQ(1, _throttlers[i]->getActiveMerges().size()); fwd = _topLinks[i]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(i + 1), fwd->getAddress()->getIndex()); - CPPUNIT_ASSERT_EQUAL(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); + ASSERT_EQ(i + 1, fwd->getAddress()->getIndex()); + ASSERT_EQ(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); { std::vector<uint16_t> chain; for (int j = 0; j <= i; ++j) { chain.push_back(j); } - CPPUNIT_ASSERT(checkChain(fwd, chain.begin(), chain.end())); + EXPECT_TRUE(checkChain(fwd, chain.begin(), chain.end())); } // Ensure priority, cluster state version and timeout is correctly forwarded - CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); - CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + EXPECT_EQ(7, static_cast<int>(fwd->getPriority())); + EXPECT_EQ(123, dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + EXPECT_EQ(54321, dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); } _topLinks[lastNodeIdx]->sendDown(fwd); @@ -374,26 +314,25 @@ MergeThrottlerTest::testChain() // If node 2 is the first in the node list, it should immediately execute // the merge. Otherwise, a cycle with the first node should be formed. if (executorNode != lastNodeIdx) { - //std::cout << "cycle " << lastNodeIdx << " -> " << executorNode << "\n"; _topLinks[lastNodeIdx]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Forwarded merge should not be sent down - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[lastNodeIdx]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[lastNodeIdx]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[lastNodeIdx]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[lastNodeIdx]->getNumCommands()); + ASSERT_EQ(1, _topLinks[lastNodeIdx]->getNumReplies()); + ASSERT_EQ(1, _throttlers[lastNodeIdx]->getActiveMerges().size()); fwd = _topLinks[lastNodeIdx]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(executorNode), fwd->getAddress()->getIndex()); - CPPUNIT_ASSERT_EQUAL(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); + ASSERT_EQ(executorNode, fwd->getAddress()->getIndex()); + ASSERT_EQ(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); { std::vector<uint16_t> chain; for (int j = 0; j < _storageNodeCount; ++j) { chain.push_back(j); } - CPPUNIT_ASSERT(checkChain(fwd, chain.begin(), chain.end())); + EXPECT_TRUE(checkChain(fwd, chain.begin(), chain.end())); } - CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); - CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + EXPECT_EQ(7, static_cast<int>(fwd->getPriority())); + EXPECT_EQ(123, dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + EXPECT_EQ(54321, dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); _topLinks[executorNode]->sendDown(fwd); } @@ -401,29 +340,28 @@ MergeThrottlerTest::testChain() _bottomLinks[executorNode]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Forwarded merge has now been sent down to persistence layer - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[executorNode]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[executorNode]->getNumReplies()); // No reply sent yet - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[executorNode]->getActiveMerges().size()); // no re-registering merge + ASSERT_EQ(1, _bottomLinks[executorNode]->getNumCommands()); + ASSERT_EQ(0, _topLinks[executorNode]->getNumReplies()); // No reply sent yet + ASSERT_EQ(1, _throttlers[executorNode]->getActiveMerges().size()); // no re-registering merge if (executorNode != lastNodeIdx) { // The MergeBucketCommand that is kept in the executor node should // be the one from the node it initially got it from, NOT the one // from the last node, since the chain has looped - CPPUNIT_ASSERT(_throttlers[executorNode]->getActiveMerges().find(bucket) - != _throttlers[executorNode]->getActiveMerges().end()); - CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), - _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); + ASSERT_TRUE(_throttlers[executorNode]->getActiveMerges().find(bucket) + != _throttlers[executorNode]->getActiveMerges().end()); + ASSERT_EQ(static_cast<StorageMessage*>(fwdToExec.get()), + _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); } // Send reply up from persistence layer to simulate a completed // merge operation. Chain should now unwind properly fwd = _bottomLinks[executorNode]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); - CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + EXPECT_EQ(7, static_cast<int>(fwd->getPriority())); + EXPECT_EQ(123, dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + EXPECT_EQ(54321, dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); reply->setResult(ReturnCode(ReturnCode::OK, "Great success! :D-|-<")); _bottomLinks[executorNode]->sendUp(reply); @@ -431,45 +369,41 @@ MergeThrottlerTest::testChain() if (executorNode != lastNodeIdx) { // Merge should not be removed yet from executor, since it's pending an unwind - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[executorNode]->getActiveMerges().size()); - CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), - _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); + ASSERT_EQ(1, _throttlers[executorNode]->getActiveMerges().size()); + ASSERT_EQ(static_cast<StorageMessage*>(fwdToExec.get()), + _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); } // MergeBucketReply waiting to be sent back to node 2. NOTE: we don't have any // transport context stuff set up here to perform the reply mapping, so we // have to emulate it - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[executorNode]->getNumReplies()); + ASSERT_EQ(1, _topLinks[executorNode]->getNumReplies()); - StorageMessage::SP unwind = _topLinks[executorNode]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL(uint16_t(executorNode), unwind->getAddress()->getIndex()); + auto unwind = _topLinks[executorNode]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + ASSERT_EQ(executorNode, unwind->getAddress()->getIndex()); // eg: 0 -> 2 -> 1 -> 0. Or: 2 -> 1 -> 0 if no cycle for (int i = (executorNode != lastNodeIdx ? _storageNodeCount - 1 : _storageNodeCount - 2); i >= 0; --i) { _topLinks[i]->sendDown(unwind); _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[i]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[i]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[i]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[i]->getNumCommands()); + ASSERT_EQ(1, _topLinks[i]->getNumReplies()); + ASSERT_EQ(0, _throttlers[i]->getActiveMerges().size()); unwind = _topLinks[i]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL(uint16_t(i), unwind->getAddress()->getIndex()); + ASSERT_EQ(uint16_t(i), unwind->getAddress()->getIndex()); } const MergeBucketReply& mbr = dynamic_cast<const MergeBucketReply&>(*unwind); - CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, mbr.getResult().getResult()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("Great success! :D-|-<"), mbr.getResult().getMessage()); - CPPUNIT_ASSERT_EQUAL(bucket, mbr.getBucket()); + EXPECT_EQ(ReturnCode::OK, mbr.getResult().getResult()); + EXPECT_EQ(vespalib::string("Great success! :D-|-<"), mbr.getResult().getMessage()); + EXPECT_EQ(bucket, mbr.getBucket()); } while (std::next_permutation(indices, indices + _storageNodeCount)); - - //std::cout << "\n" << *_topLinks[0] << "\n"; } -void -MergeThrottlerTest::testWithSourceOnlyNode() -{ +TEST_F(MergeThrottlerTest, with_source_only_node) { BucketId bid(14, 0x1337); StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); @@ -478,39 +412,37 @@ MergeThrottlerTest::testWithSourceOnlyNode() nodes.push_back(0); nodes.push_back(2); nodes.push_back(MergeBucketCommand::Node(1, true)); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, UINT_MAX, 123)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, UINT_MAX, 123); cmd->setAddress(address); _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(1), fwd->getAddress()->getIndex()); + ASSERT_EQ(1, fwd->getAddress()->getIndex()); _topLinks[1]->sendDown(fwd); _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); + ASSERT_EQ(2, fwd->getAddress()->getIndex()); _topLinks[2]->sendDown(fwd); _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); fwd = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), fwd->getAddress()->getIndex()); + ASSERT_EQ(0, fwd->getAddress()->getIndex()); _topLinks[0]->sendDown(fwd); _bottomLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); _bottomLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); reply->setResult(ReturnCode(ReturnCode::OK, "Great success! :D-|-<")); _bottomLinks[0]->sendUp(reply); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), fwd->getAddress()->getIndex()); + ASSERT_EQ(0, fwd->getAddress()->getIndex()); // Assume everything's fine from here on out } @@ -519,17 +451,15 @@ MergeThrottlerTest::testWithSourceOnlyNode() // index, so we must detect such situations and execute the merge // immediately rather than attempt to chain it. Test that this // is done correctly. -void -MergeThrottlerTest::test42DistributorBehavior() -{ +// TODO remove functionality and test +TEST_F(MergeThrottlerTest, legacy_42_distributor_behavior) { BucketId bid(32, 0xfeef00); std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); // Send to node 1, which is not the lowest index StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); @@ -539,40 +469,37 @@ MergeThrottlerTest::test42DistributorBehavior() _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Should now have been sent to persistence layer - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[1]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); // No reply sent yet - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[1]->getActiveMerges().size()); + ASSERT_EQ(1, _bottomLinks[1]->getNumCommands()); + ASSERT_EQ(0, _topLinks[1]->getNumReplies()); // No reply sent yet + ASSERT_EQ(1, _throttlers[1]->getActiveMerges().size()); // Send reply up from persistence layer to simulate a completed // merge operation. Merge should be removed from state. _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*cmd)); reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); _bottomLinks[1]->sendUp(reply); _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[1]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[1]->getNumCommands()); + ASSERT_EQ(1, _topLinks[1]->getNumReplies()); + ASSERT_EQ(0, _throttlers[1]->getActiveMerges().size()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[1]->getMetrics().local.ok.getValue()); + EXPECT_EQ(uint64_t(1), _throttlers[1]->getMetrics().local.ok.getValue()); } // Test that we don't take ownership of the merge command when we're // just passing it through to the persistence layer when receiving // a merge command that presumably comes form a 4.2 distributor -void -MergeThrottlerTest::test42DistributorBehaviorDoesNotTakeOwnership() -{ +// TODO remove functionality and test +TEST_F(MergeThrottlerTest, legacy_42_distributor_behavior_does_not_take_ownership) { BucketId bid(32, 0xfeef00); std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); // Send to node 1, which is not the lowest index StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); @@ -582,9 +509,9 @@ MergeThrottlerTest::test42DistributorBehaviorDoesNotTakeOwnership() _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Should now have been sent to persistence layer - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[1]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); // No reply sent yet - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[1]->getActiveMerges().size()); + ASSERT_EQ(1, _bottomLinks[1]->getNumCommands()); + ASSERT_EQ(0, _topLinks[1]->getNumReplies()); // No reply sent yet + ASSERT_EQ(1, _throttlers[1]->getActiveMerges().size()); _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); @@ -597,29 +524,26 @@ MergeThrottlerTest::test42DistributorBehaviorDoesNotTakeOwnership() // for the merge command, as it is not owned by the throttler _throttlers[1]->onFlush(true); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[1]->getNumCommands()); + ASSERT_EQ(0, _topLinks[1]->getNumReplies()); + ASSERT_EQ(0, _throttlers[1]->getActiveMerges().size()); // Send a belated reply from persistence up just to ensure the // throttler doesn't throw a fit if it receives an unknown merge - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*cmd)); reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); _bottomLinks[1]->sendUp(reply); _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[1]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[1]->getNumCommands()); + ASSERT_EQ(1, _topLinks[1]->getNumReplies()); + ASSERT_EQ(0, _throttlers[1]->getActiveMerges().size()); } // Test that we don't take ownership of the merge command when we're // just passing it through to the persistence layer when we're at the // the end of the chain and also the designated executor -void -MergeThrottlerTest::testEndOfChainExecutionDoesNotTakeOwnership() -{ +TEST_F(MergeThrottlerTest, end_of_chain_execution_does_not_take_ownership) { BucketId bid(32, 0xfeef00); std::vector<MergeBucketCommand::Node> nodes; @@ -629,8 +553,7 @@ MergeThrottlerTest::testEndOfChainExecutionDoesNotTakeOwnership() std::vector<uint16_t> chain; chain.push_back(0); chain.push_back(1); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234, 1, chain); // Send to last node, which is not the lowest index StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); @@ -640,9 +563,9 @@ MergeThrottlerTest::testEndOfChainExecutionDoesNotTakeOwnership() _bottomLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); // Should now have been sent to persistence layer - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[2]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[2]->getNumReplies()); // No reply sent yet - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[2]->getActiveMerges().size()); + ASSERT_EQ(1, _bottomLinks[2]->getNumCommands()); + ASSERT_EQ(0, _topLinks[2]->getNumReplies()); // No reply sent yet + ASSERT_EQ(1, _throttlers[2]->getActiveMerges().size()); _bottomLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); @@ -655,37 +578,33 @@ MergeThrottlerTest::testEndOfChainExecutionDoesNotTakeOwnership() // for the merge command, as it is not owned by the throttler _throttlers[2]->onFlush(true); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[2]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[2]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[2]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[2]->getNumCommands()); + ASSERT_EQ(0, _topLinks[2]->getNumReplies()); + ASSERT_EQ(0, _throttlers[2]->getActiveMerges().size()); // Send a belated reply from persistence up just to ensure the // throttler doesn't throw a fit if it receives an unknown merge - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*cmd)); reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); _bottomLinks[2]->sendUp(reply); _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[2]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[2]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[2]->getActiveMerges().size()); + ASSERT_EQ(0, _bottomLinks[2]->getNumCommands()); + ASSERT_EQ(1, _topLinks[2]->getNumReplies()); + ASSERT_EQ(0, _throttlers[2]->getActiveMerges().size()); } // Test that nodes resending a merge command won't lead to duplicate // state registration/forwarding or erasing the already present state // information. -void -MergeThrottlerTest::testResendHandling() -{ +TEST_F(MergeThrottlerTest, resend_handling) { BucketId bid(32, 0xbadbed); std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); @@ -702,9 +621,8 @@ MergeThrottlerTest::testResendHandling() // Reply should be BUSY StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::BUSY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); _topLinks[1]->sendDown(fwd); _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -717,9 +635,8 @@ MergeThrottlerTest::testResendHandling() // Reply should be BUSY reply = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::BUSY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); fwd = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); @@ -729,24 +646,21 @@ MergeThrottlerTest::testResendHandling() _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::BUSY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); } -void -MergeThrottlerTest::testPriorityQueuing() -{ +TEST_F(MergeThrottlerTest, priority_queuing) { // Fill up all active merges std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - CPPUNIT_ASSERT(maxPending >= 4u); + ASSERT_GE(maxPending, 4u); for (std::size_t i = 0; i < maxPending; ++i) { - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); cmd->setPriority(100); _topLinks[0]->sendDown(cmd); } @@ -771,15 +685,14 @@ MergeThrottlerTest::testPriorityQueuing() for (std::size_t i = 0; i < maxPending - 4; ++i) { _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); } - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(4), _topLinks[0]->getNumReplies()); + ASSERT_EQ(0, _topLinks[0]->getNumCommands()); + ASSERT_EQ(4, _topLinks[0]->getNumReplies()); // Now when we start replying to merges, queued merges should be // processed in priority order for (int i = 0; i < 4; ++i) { StorageMessage::SP replyTo = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*replyTo))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*replyTo)); reply->setResult(ReturnCode(ReturnCode::OK, "whee")); _topLinks[0]->sendDown(reply); } @@ -789,26 +702,24 @@ MergeThrottlerTest::testPriorityQueuing() for (int i = 0; i < 4; ++i) { StorageMessage::SP cmd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint8_t(sortedPris[i]), cmd->getPriority()); + EXPECT_EQ(uint8_t(sortedPris[i]), cmd->getPriority()); } } // Test that we can detect and reject merges that due to resending // and potential priority queue sneaking etc may end up with duplicates // in the queue for a merge that is already known. -void -MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() -{ +TEST_F(MergeThrottlerTest, command_in_queue_duplicate_of_known_merge) { // Fill up all active merges and 1 queued one - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + 1; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(2 + i); nodes.push_back(5 + i); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); cmd->setPriority(100 - i); _topLinks[0]->sendDown(cmd); } @@ -823,8 +734,8 @@ MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() nodes.push_back(0); nodes.push_back(12); nodes.push_back(123); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf000feee)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf000feee)), nodes, 1234); _topLinks[0]->sendDown(cmd); } { @@ -832,22 +743,21 @@ MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() nodes.push_back(0); nodes.push_back(124); // Different node set doesn't matter nodes.push_back(14); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf000feee)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf000feee)), nodes, 1234); _topLinks[0]->sendDown(cmd); } waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); - StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + auto fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); // Remove and success-reply for 2 merges. This will give enough room // for the 2 first queued merges to be processed, the last one having a // duplicate in the queue. for (int i = 0; i < 2; ++i) { StorageMessage::SP fwd2 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd2))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd2)); reply->setResult(ReturnCode(ReturnCode::OK, "")); _topLinks[0]->sendDown(reply); } @@ -859,8 +769,7 @@ MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() _topLinks[0]->getRepliesOnce(); // Send a success-reply for fwd, allowing the duplicate from the queue // to have its moment to shine only to then be struck down mercilessly - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); reply->setResult(ReturnCode(ReturnCode::OK, "")); _topLinks[0]->sendDown(reply); @@ -868,62 +777,55 @@ MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() waitUntilMergeQueueIs(*_throttlers[0], 0, _messageWaitTime); // First reply is the successful merge reply - StorageMessage::SP reply2 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply2).getResult().getResult(), - ReturnCode::OK); + auto reply2 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply2).getResult().getResult(), + ReturnCode::OK); // Second reply should be the BUSY-rejected duplicate - StorageMessage::SP reply1 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply1).getResult().getResult(), - ReturnCode::BUSY); - CPPUNIT_ASSERT(static_cast<MergeBucketReply&>(*reply1).getResult() - .getMessage().find("out of date;") != std::string::npos); + auto reply1 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply1).getResult().getResult(), + ReturnCode::BUSY); + EXPECT_TRUE(static_cast<MergeBucketReply&>(*reply1).getResult() + .getMessage().find("out of date;") != std::string::npos); } // Test that sending a merge command to a node not in the set of // to-be-merged nodes is handled gracefully. // This is not a scenario that should ever actually happen, but for // the sake of robustness, include it anyway. -void -MergeThrottlerTest::testInvalidReceiverNode() -{ +TEST_F(MergeThrottlerTest, invalid_receiver_node) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(1); nodes.push_back(5); nodes.push_back(9); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baaaa)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baaaa)), nodes, 1234); // Send to node with index 0 _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::REJECTED); - CPPUNIT_ASSERT(static_cast<MergeBucketReply&>(*reply).getResult() - .getMessage().find("which is not in its forwarding chain") != std::string::npos); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::REJECTED); + EXPECT_TRUE(static_cast<MergeBucketReply&>(*reply).getResult() + .getMessage().find("which is not in its forwarding chain") != std::string::npos); } // Test that the throttling policy kicks in after a certain number of // merges are forwarded and that the rest are queued in a prioritized // order. -void -MergeThrottlerTest::testForwardQueuedMerge() -{ +TEST_F(MergeThrottlerTest, forward_queued_merge) { // Fill up all active merges and then 3 queued ones - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(2 + i); nodes.push_back(5 + i); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); cmd->setPriority(100 - i); _topLinks[0]->sendDown(cmd); } @@ -933,70 +835,61 @@ MergeThrottlerTest::testForwardQueuedMerge() waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); // Merge queue state should not be touched by worker thread now - StorageMessage::SP nextMerge = _throttlers[0]->getMergeQueue().begin()->_msg; + auto nextMerge = _throttlers[0]->getMergeQueue().begin()->_msg; - StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + auto fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); // Remove all the rest of the active merges while (!_topLinks[0]->getReplies().empty()) { _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); } - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); reply->setResult(ReturnCode(ReturnCode::OK, "Celebrate good times come on")); _topLinks[0]->sendDown(reply); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); // Success rewind reply // Remove reply bound for distributor - StorageMessage::SP distReply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), - ReturnCode::OK); + auto distReply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), + ReturnCode::OK); waitUntilMergeQueueIs(*_throttlers[0], 2, _messageWaitTime); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[0]->getNumReplies()); + ASSERT_EQ(0, _topLinks[0]->getNumCommands()); + ASSERT_EQ(1, _topLinks[0]->getNumReplies()); // First queued merge should now have been registered and forwarded fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL( - static_cast<const MergeBucketCommand&>(*fwd).getBucketId(), - static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + ASSERT_EQ(static_cast<const MergeBucketCommand&>(*fwd).getBucketId(), + static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); - CPPUNIT_ASSERT( - static_cast<const MergeBucketCommand&>(*fwd).getNodes() - == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); + ASSERT_TRUE(static_cast<const MergeBucketCommand&>(*fwd).getNodes() + == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); // Ensure forwarded merge has a higher priority than the next queued one - CPPUNIT_ASSERT(fwd->getPriority() < _throttlers[0]->getMergeQueue().begin()->_msg->getPriority()); - - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().chaining.ok.getValue()); + EXPECT_LT(fwd->getPriority(), _throttlers[0]->getMergeQueue().begin()->_msg->getPriority()); - /*framework::HttpUrlPath path("?xml"); - _forwarders[0]->reportStatus(std::cerr, path);*/ + EXPECT_EQ(uint64_t(1), _throttlers[0]->getMetrics().chaining.ok.getValue()); } -void -MergeThrottlerTest::testExecuteQueuedMerge() -{ +TEST_F(MergeThrottlerTest, execute_queued_merge) { MergeThrottler& throttler(*_throttlers[1]); DummyStorageLink& topLink(*_topLinks[1]); DummyStorageLink& bottomLink(*_bottomLinks[1]); // Fill up all active merges and then 3 queued ones - std::size_t maxPending = throttler.getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = throttler.getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(1); nodes.push_back(5 + i); nodes.push_back(7 + i); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1); cmd->setPriority(250 - i + 5); topLink.sendDown(cmd); } @@ -1013,8 +906,8 @@ MergeThrottlerTest::testExecuteQueuedMerge() nodes.push_back(0); std::vector<uint16_t> chain; chain.push_back(0); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0x1337)), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0x1337)), nodes, 1234, 1, chain); cmd->setPriority(0); topLink.sendDown(cmd); } @@ -1022,19 +915,12 @@ MergeThrottlerTest::testExecuteQueuedMerge() waitUntilMergeQueueIs(throttler, 4, _messageWaitTime); // Merge queue state should not be touched by worker thread now - StorageMessage::SP nextMerge(throttler.getMergeQueue().begin()->_msg); - /*StorageMessage::SP nextMerge; - { - vespalib::LockGuard lock(_throttlers[0]->getStateLock()); - // Dirty: have to check internal state - nextMerge = _throttlers[0]->getMergeQueue().begin()->_msg; - }*/ + auto nextMerge = throttler.getMergeQueue().begin()->_msg; - CPPUNIT_ASSERT_EQUAL( - BucketId(32, 0x1337), - dynamic_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + ASSERT_EQ(BucketId(32, 0x1337), + dynamic_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); - StorageMessage::SP fwd(topLink.getAndRemoveMessage(MessageType::MERGEBUCKET)); + auto fwd = topLink.getAndRemoveMessage(MessageType::MERGEBUCKET); // Remove all the rest of the active merges while (!topLink.getReplies().empty()) { @@ -1042,50 +928,44 @@ MergeThrottlerTest::testExecuteQueuedMerge() } // Free up a merge slot - std::shared_ptr<MergeBucketReply> reply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto reply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); reply->setResult(ReturnCode(ReturnCode::OK, "Celebrate good times come on")); topLink.sendDown(reply); topLink.waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); // Remove chain reply - StorageMessage::SP distReply(topLink.getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY)); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), - ReturnCode::OK); + auto distReply = topLink.getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + ASSERT_EQ(static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), + ReturnCode::OK); waitUntilMergeQueueIs(throttler, 3, _messageWaitTime); bottomLink.waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), topLink.getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), topLink.getNumReplies()); - CPPUNIT_ASSERT_EQUAL(std::size_t(1), bottomLink.getNumCommands()); + ASSERT_EQ(0, topLink.getNumCommands()); + ASSERT_EQ(0, topLink.getNumReplies()); + ASSERT_EQ(1, bottomLink.getNumCommands()); // First queued merge should now have been registered and sent down - StorageMessage::SP cmd(bottomLink.getAndRemoveMessage(MessageType::MERGEBUCKET)); + auto cmd = bottomLink.getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL( - static_cast<const MergeBucketCommand&>(*cmd).getBucketId(), - static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + ASSERT_EQ(static_cast<const MergeBucketCommand&>(*cmd).getBucketId(), + static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); - CPPUNIT_ASSERT( - static_cast<const MergeBucketCommand&>(*cmd).getNodes() - == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); + ASSERT_TRUE(static_cast<const MergeBucketCommand&>(*cmd).getNodes() + == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); } -void -MergeThrottlerTest::testFlush() -{ +TEST_F(MergeThrottlerTest, flush) { // Fill up all active merges and then 3 queued ones std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1); _topLinks[0]->sendDown(cmd); } @@ -1095,7 +975,7 @@ MergeThrottlerTest::testFlush() // Remove all forwarded commands uint32_t removed = _topLinks[0]->getRepliesOnce().size(); - CPPUNIT_ASSERT(removed >= 5); + ASSERT_GE(removed, 5); // Flush the storage link, triggering an abort of all commands // no matter what their current state is. @@ -1105,9 +985,8 @@ MergeThrottlerTest::testFlush() while (!_topLinks[0]->getReplies().empty()) { StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - ReturnCode::ABORTED, - static_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + ASSERT_EQ(ReturnCode::ABORTED, + static_cast<const MergeBucketReply&>(*reply).getResult().getResult()); } // NOTE: merges that have been immediately executed (i.e. not cycled) // on the node should _not_ be replied to, since they're not owned @@ -1119,9 +998,7 @@ MergeThrottlerTest::testFlush() // it knows nothing about when it comes back up. If this is not handled // properly, it will attempt to forward this node again with a bogus // index. This should be implicitly handled by checking for a full node -void -MergeThrottlerTest::testUnseenMergeWithNodeInChain() -{ +TEST_F(MergeThrottlerTest, unseen_merge_with_node_in_chain) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(5); @@ -1130,8 +1007,8 @@ MergeThrottlerTest::testUnseenMergeWithNodeInChain() chain.push_back(0); chain.push_back(5); chain.push_back(9); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xdeadbeef)), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xdeadbeef)), nodes, 1234, 1, chain); StorageMessageAddress address("storage", lib::NodeType::STORAGE, 9); @@ -1141,21 +1018,19 @@ MergeThrottlerTest::testUnseenMergeWithNodeInChain() // First, test that we get rejected when processing merge immediately // Should get a rejection in return _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - ReturnCode::REJECTED, - dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + ASSERT_EQ(ReturnCode::REJECTED, + dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); // Second, test that we get rejected before queueing up. This is to // avoid a hypothetical deadlock scenario. // Fill up all active merges { - std::size_t maxPending( - _throttlers[0]->getThrottlePolicy().getMaxPendingCount()); - for (std::size_t i = 0; i < maxPending; ++i) { - std::shared_ptr<MergeBucketCommand> fillCmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234)); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + for (size_t i = 0; i < maxPending; ++i) { + auto fillCmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234); _topLinks[0]->sendDown(fillCmd); } } @@ -1164,26 +1039,23 @@ MergeThrottlerTest::testUnseenMergeWithNodeInChain() _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - ReturnCode::REJECTED, - dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + ASSERT_EQ(ReturnCode::REJECTED, + dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); } -void -MergeThrottlerTest::testMergeWithNewerClusterStateFlushesOutdatedQueued() -{ +TEST_F(MergeThrottlerTest, merge_with_newer_cluster_state_flushes_outdated_queued){ // Fill up all active merges and then 3 queued ones with the same // system state - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); std::vector<api::StorageMessage::Id> ids; for (std::size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1); ids.push_back(cmd->getMsgId()); _topLinks[0]->sendDown(cmd); } @@ -1198,8 +1070,8 @@ MergeThrottlerTest::testMergeWithNewerClusterStateFlushesOutdatedQueued() nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0x12345678)), nodes, 1234, 2)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0x12345678)), nodes, 1234, 2); ids.push_back(cmd->getMsgId()); _topLinks[0]->sendDown(cmd); } @@ -1211,30 +1083,27 @@ MergeThrottlerTest::testMergeWithNewerClusterStateFlushesOutdatedQueued() for (int i = 0; i < 3; ++i) { StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::WRONG_DISTRIBUTION); - CPPUNIT_ASSERT_EQUAL(1u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(ids[maxPending + i], reply->getMsgId()); + ASSERT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); + ASSERT_EQ(1u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); + ASSERT_EQ(ids[maxPending + i], reply->getMsgId()); } - CPPUNIT_ASSERT_EQUAL(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); + EXPECT_EQ(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); } -void -MergeThrottlerTest::testUpdatedClusterStateFlushesOutdatedQueued() -{ +TEST_F(MergeThrottlerTest, updated_cluster_state_flushes_outdated_queued) { // State is version 1. Send down several merges with state version 2. - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); std::vector<api::StorageMessage::Id> ids; for (std::size_t i = 0; i < maxPending + 3; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 2)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 2); ids.push_back(cmd->getMsgId()); _topLinks[0]->sendDown(cmd); } @@ -1245,8 +1114,8 @@ MergeThrottlerTest::testUpdatedClusterStateFlushesOutdatedQueued() // Send down new system state (also set it explicitly) _servers[0]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:3")); - std::shared_ptr<api::SetSystemStateCommand> stateCmd( - new api::SetSystemStateCommand(lib::ClusterState("distributor:100 storage:100 version:3"))); + auto stateCmd = std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState("distributor:100 storage:100 version:3")); _topLinks[0]->sendDown(stateCmd); // Queue should now be flushed with all being replied to with WRONG_DISTRIBUTION @@ -1255,29 +1124,27 @@ MergeThrottlerTest::testUpdatedClusterStateFlushesOutdatedQueued() for (int i = 0; i < 3; ++i) { StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::WRONG_DISTRIBUTION); - CPPUNIT_ASSERT_EQUAL(2u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(ids[maxPending + i], reply->getMsgId()); + ASSERT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); + ASSERT_EQ(2u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); + ASSERT_EQ(ids[maxPending + i], reply->getMsgId()); } - CPPUNIT_ASSERT_EQUAL(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); + EXPECT_EQ(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); } -void -MergeThrottlerTest::test42MergesDoNotTriggerFlush() -{ +// TODO remove functionality and test +TEST_F(MergeThrottlerTest, legacy_42_merges_do_not_trigger_flush) { // Fill up all active merges and then 1 queued one - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - CPPUNIT_ASSERT(maxPending < 100); + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + 1; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00baa00 + i)), nodes, 1234, 1); _topLinks[0]->sendDown(cmd); } @@ -1285,7 +1152,7 @@ MergeThrottlerTest::test42MergesDoNotTriggerFlush() _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); waitUntilMergeQueueIs(*_throttlers[0], 1, _messageWaitTime); - StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + auto fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); // Remove all the rest of the active merges while (!_topLinks[0]->getReplies().empty()) { @@ -1299,24 +1166,22 @@ MergeThrottlerTest::test42MergesDoNotTriggerFlush() nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xbaaadbed)), nodes, 1234, 0)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xbaaadbed)), nodes, 1234, 0); _topLinks[0]->sendDown(cmd); } waitUntilMergeQueueIs(*_throttlers[0], 2, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumReplies()); + ASSERT_EQ(0, _topLinks[0]->getNumCommands()); + ASSERT_EQ(0, _topLinks[0]->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().local.failures.wrongdistribution.getValue()); + EXPECT_EQ(uint64_t(0), _throttlers[0]->getMetrics().local.failures.wrongdistribution.getValue()); } // Test that a merge that arrive with a state version that is less than // that of the node is rejected immediately -void -MergeThrottlerTest::testOutdatedClusterStateMergesAreRejectedOnArrival() -{ +TEST_F(MergeThrottlerTest, outdated_cluster_state_merges_are_rejected_on_arrival) { _servers[0]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:10")); // Send down a merge with a cluster state version of 9, which should @@ -1326,28 +1191,25 @@ MergeThrottlerTest::testOutdatedClusterStateMergesAreRejectedOnArrival() nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 9)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 9); _topLinks[0]->sendDown(cmd); } _topLinks[0]->waitForMessages(1, _messageWaitTime); - StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - static_cast<MergeBucketReply&>(*reply).getResult().getResult(), - ReturnCode::WRONG_DISTRIBUTION); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + EXPECT_EQ(static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); + EXPECT_EQ(uint64_t(1), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); } // Test erroneous case where node receives merge where the merge does // not exist in the state, but it exists in the chain without the chain // being full. This is something that shouldn't happen, but must still // not crash the node -void -MergeThrottlerTest::testUnknownMergeWithSelfInChain() -{ +TEST_F(MergeThrottlerTest, unknown_merge_with_self_in_chain) { BucketId bid(32, 0xbadbed); std::vector<MergeBucketCommand::Node> nodes; @@ -1356,8 +1218,7 @@ MergeThrottlerTest::testUnknownMergeWithSelfInChain() nodes.push_back(2); std::vector<uint16_t> chain; chain.push_back(0); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234, 1, chain); StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); @@ -1365,26 +1226,23 @@ MergeThrottlerTest::testUnknownMergeWithSelfInChain() _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - ReturnCode::REJECTED, - static_cast<MergeBucketReply&>(*reply).getResult().getResult()); + EXPECT_EQ(ReturnCode::REJECTED, + static_cast<MergeBucketReply&>(*reply).getResult().getResult()); } -void -MergeThrottlerTest::testBusyReturnedOnFullQueue() -{ - std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); - std::size_t maxQueue = _throttlers[0]->getMaxQueueSize(); - CPPUNIT_ASSERT(maxPending < 100); +TEST_F(MergeThrottlerTest, busy_returned_on_full_queue) { + size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + size_t maxQueue = _throttlers[0]->getMaxQueueSize(); + ASSERT_LT(maxPending, 100); for (std::size_t i = 0; i < maxPending + maxQueue; ++i) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf00000 + i)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf00000 + i)), nodes, 1234, 1); _topLinks[0]->sendDown(cmd); } @@ -1400,32 +1258,24 @@ MergeThrottlerTest::testBusyReturnedOnFullQueue() nodes.push_back(0); nodes.push_back(1); nodes.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xf000baaa)), nodes, 1234, 1)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xf000baaa)), nodes, 1234, 1); _topLinks[0]->sendDown(cmd); } _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); - StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL( - BucketId(32, 0xf000baaa), - static_cast<MergeBucketReply&>(*reply).getBucketId()); + EXPECT_EQ(BucketId(32, 0xf000baaa), + static_cast<MergeBucketReply&>(*reply).getBucketId()); - CPPUNIT_ASSERT_EQUAL( - ReturnCode::BUSY, - static_cast<MergeBucketReply&>(*reply).getResult().getResult()); + EXPECT_EQ(ReturnCode::BUSY, + static_cast<MergeBucketReply&>(*reply).getResult().getResult()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), - _throttlers[0]->getMetrics().chaining - .failures.busy.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), - _throttlers[0]->getMetrics().local - .failures.busy.getValue()); + EXPECT_EQ(0, _throttlers[0]->getMetrics().chaining.failures.busy.getValue()); + EXPECT_EQ(1, _throttlers[0]->getMetrics().local.failures.busy.getValue()); } -void -MergeThrottlerTest::testBrokenCycle() -{ +TEST_F(MergeThrottlerTest, broken_cycle) { std::vector<MergeBucketCommand::Node> nodes; nodes.push_back(1); nodes.push_back(0); @@ -1433,14 +1283,14 @@ MergeThrottlerTest::testBrokenCycle() { std::vector<uint16_t> chain; chain.push_back(0); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain); _topLinks[1]->sendDown(cmd); } _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - StorageMessage::SP fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); + auto fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + ASSERT_EQ(2, fwd->getAddress()->getIndex()); // Send cycled merge which will be executed { @@ -1448,23 +1298,21 @@ MergeThrottlerTest::testBrokenCycle() chain.push_back(0); chain.push_back(1); chain.push_back(2); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain); _topLinks[1]->sendDown(cmd); } _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - StorageMessage::SP cycled = _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + auto cycled = _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); // Now, node 2 goes down, auto sending back a failed merge - std::shared_ptr<MergeBucketReply> nodeDownReply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + auto nodeDownReply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*fwd)); nodeDownReply->setResult(ReturnCode(ReturnCode::NOT_CONNECTED, "Node went sightseeing")); _topLinks[1]->sendDown(nodeDownReply); // Merge reply also arrives from persistence - std::shared_ptr<MergeBucketReply> persistenceReply( - new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cycled))); + auto persistenceReply = std::make_shared<MergeBucketReply>(dynamic_cast<const MergeBucketCommand&>(*cycled)); persistenceReply->setResult(ReturnCode(ReturnCode::ABORTED, "Oh dear")); _bottomLinks[1]->sendUp(persistenceReply); @@ -1474,8 +1322,8 @@ MergeThrottlerTest::testBrokenCycle() // Unwind reply shares the result of the persistence reply for (int i = 0; i < 2; ++i) { StorageMessage::SP reply = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(ReturnCode::ABORTED, "Oh dear"), - static_cast<MergeBucketReply&>(*reply).getResult()); + ASSERT_EQ(api::ReturnCode(ReturnCode::ABORTED, "Oh dear"), + static_cast<MergeBucketReply&>(*reply).getResult()); } // Make sure it has been removed from the internal state so we can @@ -1483,14 +1331,14 @@ MergeThrottlerTest::testBrokenCycle() { std::vector<uint16_t> chain; chain.push_back(0); - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain)); + auto cmd = std::make_shared<MergeBucketCommand>( + makeDocumentBucket(BucketId(32, 0xfeef00)), nodes, 1234, 1, chain); _topLinks[1]->sendDown(cmd); } _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, 5); fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); - CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); + ASSERT_EQ(2, fwd->getAddress()->getIndex()); } void @@ -1501,39 +1349,33 @@ MergeThrottlerTest::sendAndExpectReply( { _topLinks[0]->sendDown(msg); _topLinks[0]->waitForMessage(expectedReplyType, _messageWaitTime); - StorageMessage::SP reply(_topLinks[0]->getAndRemoveMessage( - expectedReplyType)); + auto reply = _topLinks[0]->getAndRemoveMessage(expectedReplyType); auto& storageReply = dynamic_cast<api::StorageReply&>(*reply); - CPPUNIT_ASSERT_EQUAL(expectedResultCode, - storageReply.getResult().getResult()); + ASSERT_EQ(expectedResultCode, storageReply.getResult().getResult()); } -void -MergeThrottlerTest::testGetBucketDiffCommandNotInActiveSetIsRejected() -{ +TEST_F(MergeThrottlerTest, get_bucket_diff_command_not_in_active_set_is_rejected) { document::BucketId bucket(16, 1234); std::vector<api::GetBucketDiffCommand::Node> nodes; - std::shared_ptr<api::GetBucketDiffCommand> getDiffCmd( - new api::GetBucketDiffCommand(makeDocumentBucket(bucket), nodes, api::Timestamp(1234))); + auto getDiffCmd = std::make_shared<api::GetBucketDiffCommand>( + makeDocumentBucket(bucket), nodes, api::Timestamp(1234)); - sendAndExpectReply(getDiffCmd, - api::MessageType::GETBUCKETDIFF_REPLY, - api::ReturnCode::ABORTED); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[0]->getNumCommands()); + ASSERT_NO_FATAL_FAILURE(sendAndExpectReply(getDiffCmd, + api::MessageType::GETBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED)); + ASSERT_EQ(0, _bottomLinks[0]->getNumCommands()); } -void -MergeThrottlerTest::testApplyBucketDiffCommandNotInActiveSetIsRejected() -{ +TEST_F(MergeThrottlerTest, apply_bucket_diff_command_not_in_active_set_is_rejected) { document::BucketId bucket(16, 1234); std::vector<api::GetBucketDiffCommand::Node> nodes; - std::shared_ptr<api::ApplyBucketDiffCommand> applyDiffCmd( - new api::ApplyBucketDiffCommand(makeDocumentBucket(bucket), nodes, api::Timestamp(1234))); + auto applyDiffCmd = std::make_shared<api::ApplyBucketDiffCommand>( + makeDocumentBucket(bucket), nodes, api::Timestamp(1234)); - sendAndExpectReply(applyDiffCmd, - api::MessageType::APPLYBUCKETDIFF_REPLY, - api::ReturnCode::ABORTED); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[0]->getNumCommands()); + ASSERT_NO_FATAL_FAILURE(sendAndExpectReply(applyDiffCmd, + api::MessageType::APPLYBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED)); + ASSERT_EQ(0, _bottomLinks[0]->getNumCommands()); } api::MergeBucketCommand::SP @@ -1544,51 +1386,47 @@ MergeThrottlerTest::sendMerge(const MergeBuilder& builder) return cmd; } -void -MergeThrottlerTest::testNewClusterStateAbortsAllOutdatedActiveMerges() -{ +TEST_F(MergeThrottlerTest, new_cluster_state_aborts_all_outdated_active_merges) { document::BucketId bucket(16, 6789); _throttlers[0]->getThrottlePolicy().setMaxPendingCount(1); // Merge will be forwarded (i.e. active). sendMerge(MergeBuilder(bucket).clusterStateVersion(10)); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - StorageMessage::SP fwd(_topLinks[0]->getAndRemoveMessage( - MessageType::MERGEBUCKET)); + auto fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); - _topLinks[0]->sendDown(makeSystemStateCmd( - "version:11 distributor:100 storage:100")); + _topLinks[0]->sendDown(makeSystemStateCmd("version:11 distributor:100 storage:100")); // Cannot send reply until we're unwinding - CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumReplies()); + ASSERT_EQ(0, _topLinks[0]->getNumReplies()); // Trying to diff the bucket should now fail { - std::shared_ptr<api::GetBucketDiffCommand> getDiffCmd( - new api::GetBucketDiffCommand(makeDocumentBucket(bucket), {}, api::Timestamp(123))); + auto getDiffCmd = std::make_shared<api::GetBucketDiffCommand>( + makeDocumentBucket(bucket), std::vector<api::GetBucketDiffCommand::Node>(), api::Timestamp(123)); - sendAndExpectReply(getDiffCmd, - api::MessageType::GETBUCKETDIFF_REPLY, - api::ReturnCode::ABORTED); + ASSERT_NO_FATAL_FAILURE(sendAndExpectReply(getDiffCmd, + api::MessageType::GETBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED)); } } -void MergeThrottlerTest::backpressure_busy_bounces_merges_for_configured_duration() { +TEST_F(MergeThrottlerTest, backpressure_busy_bounces_merges_for_configured_duration) { _servers[0]->getClock().setAbsoluteTimeInSeconds(1000); - CPPUNIT_ASSERT(!_throttlers[0]->backpressure_mode_active()); + EXPECT_FALSE(_throttlers[0]->backpressure_mode_active()); _throttlers[0]->apply_timed_backpressure(); - CPPUNIT_ASSERT(_throttlers[0]->backpressure_mode_active()); + EXPECT_TRUE(_throttlers[0]->backpressure_mode_active()); document::BucketId bucket(16, 6789); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().local.failures.busy.getValue()); + EXPECT_EQ(0, _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); + EXPECT_EQ(uint64_t(0), _throttlers[0]->getMetrics().local.failures.busy.getValue()); - sendAndExpectReply(MergeBuilder(bucket).create(), - api::MessageType::MERGEBUCKET_REPLY, - api::ReturnCode::BUSY); + ASSERT_NO_FATAL_FAILURE(sendAndExpectReply(MergeBuilder(bucket).create(), + api::MessageType::MERGEBUCKET_REPLY, + api::ReturnCode::BUSY)); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().local.failures.busy.getValue()); + EXPECT_EQ(1, _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); + EXPECT_EQ(1, _throttlers[0]->getMetrics().local.failures.busy.getValue()); _servers[0]->getClock().addSecondsToTime(15); // Test-config has duration set to 15 seconds // Backpressure has now been lifted. New merges should be forwarded @@ -1596,11 +1434,11 @@ void MergeThrottlerTest::backpressure_busy_bounces_merges_for_configured_duratio sendMerge(MergeBuilder(bucket)); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - CPPUNIT_ASSERT(!_throttlers[0]->backpressure_mode_active()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); + EXPECT_FALSE(_throttlers[0]->backpressure_mode_active()); + EXPECT_EQ(1, _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); } -void MergeThrottlerTest::source_only_merges_are_not_affected_by_backpressure() { +TEST_F(MergeThrottlerTest, source_only_merges_are_not_affected_by_backpressure) { _servers[2]->getClock().setAbsoluteTimeInSeconds(1000); _throttlers[2]->apply_timed_backpressure(); document::BucketId bucket(16, 6789); @@ -1608,12 +1446,12 @@ void MergeThrottlerTest::source_only_merges_are_not_affected_by_backpressure() { _topLinks[2]->sendDown(MergeBuilder(bucket).chain(0, 1).source_only(2).create()); _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); + EXPECT_EQ(0, _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); } void MergeThrottlerTest::fill_throttler_queue_with_n_commands(uint16_t throttler_index, size_t queued_count) { - std::size_t max_pending = _throttlers[throttler_index]->getThrottlePolicy().getMaxPendingCount(); - for (std::size_t i = 0; i < max_pending + queued_count; ++i) { + size_t max_pending = _throttlers[throttler_index]->getThrottlePolicy().getMaxPendingCount(); + for (size_t i = 0; i < max_pending + queued_count; ++i) { _topLinks[throttler_index]->sendDown(MergeBuilder(document::BucketId(16, i)).create()); } @@ -1622,7 +1460,7 @@ void MergeThrottlerTest::fill_throttler_queue_with_n_commands(uint16_t throttler waitUntilMergeQueueIs(*_throttlers[throttler_index], queued_count, _messageWaitTime); } -void MergeThrottlerTest::backpressure_evicts_all_queued_merges() { +TEST_F(MergeThrottlerTest, backpressure_evicts_all_queued_merges) { _servers[0]->getClock().setAbsoluteTimeInSeconds(1000); fill_throttler_queue_with_n_commands(0, 1); @@ -1631,7 +1469,7 @@ void MergeThrottlerTest::backpressure_evicts_all_queued_merges() { _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); - CPPUNIT_ASSERT_EQUAL(ReturnCode::BUSY, dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + EXPECT_EQ(ReturnCode::BUSY, dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); } // TODO test message queue aborting (use rendezvous functionality--make guard) diff --git a/storage/src/tests/storageserver/priorityconvertertest.cpp b/storage/src/tests/storageserver/priorityconvertertest.cpp index 54f451ebdb2..af0b35e0869 100644 --- a/storage/src/tests/storageserver/priorityconvertertest.cpp +++ b/storage/src/tests/storageserver/priorityconvertertest.cpp @@ -3,101 +3,71 @@ #include <vespa/documentapi/documentapi.h> #include <vespa/storage/storageserver/priorityconverter.h> #include <tests/common/testhelper.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace ::testing; namespace storage { -struct PriorityConverterTest : public CppUnit::TestFixture -{ +struct PriorityConverterTest : Test { std::unique_ptr<PriorityConverter> _converter; - void setUp() override { + void SetUp() override { vdstestlib::DirConfig config(getStandardConfig(true)); - _converter.reset(new PriorityConverter(config.getConfigId())); + _converter = std::make_unique<PriorityConverter>(config.getConfigId()); }; - - void testNormalUsage(); - void testLowestPriorityIsReturnedForUnknownCode(); - - CPPUNIT_TEST_SUITE(PriorityConverterTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testLowestPriorityIsReturnedForUnknownCode); - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(PriorityConverterTest); - -void PriorityConverterTest::testNormalUsage() -{ - for (int p=0; p<16; ++p) { - CPPUNIT_ASSERT_EQUAL( - (uint8_t)(50+p*10), +TEST_F(PriorityConverterTest, normal_usage) { + for (int p = 0; p < 16; ++p) { + EXPECT_EQ( + (50 + p * 10), _converter->toStoragePriority( static_cast<documentapi::Priority::Value>(p))); } - for (int i=0; i<256; ++i) { + for (int i = 0; i < 256; ++i) { uint8_t p = i; if (p <= 50) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGHEST, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_HIGHEST, _converter->toDocumentPriority(p)); } else if (p <= 60) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_VERY_HIGH, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_VERY_HIGH, _converter->toDocumentPriority(p)); } else if (p <= 70) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_1, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_HIGH_1, _converter->toDocumentPriority(p)); } else if (p <= 80) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_2, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_HIGH_2, _converter->toDocumentPriority(p)); } else if (p <= 90) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_3, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_HIGH_3, _converter->toDocumentPriority(p)); } else if (p <= 100) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_1, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_1, _converter->toDocumentPriority(p)); } else if (p <= 110) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_2, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_2, _converter->toDocumentPriority(p)); } else if (p <= 120) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_3, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_3, _converter->toDocumentPriority(p)); } else if (p <= 130) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_4, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_4, _converter->toDocumentPriority(p)); } else if (p <= 140) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_5, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_5, _converter->toDocumentPriority(p)); } else if (p <= 150) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_6, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_NORMAL_6, _converter->toDocumentPriority(p)); } else if (p <= 160) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_1, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_LOW_1, _converter->toDocumentPriority(p)); } else if (p <= 170) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_2, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_LOW_2, _converter->toDocumentPriority(p)); } else if (p <= 180) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_3, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_LOW_3, _converter->toDocumentPriority(p)); } else if (p <= 190) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_VERY_LOW, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_VERY_LOW, _converter->toDocumentPriority(p)); } else if (p <= 200) { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOWEST, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_LOWEST, _converter->toDocumentPriority(p)); } else { - CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOWEST, - _converter->toDocumentPriority(p)); + EXPECT_EQ(documentapi::Priority::PRI_LOWEST, _converter->toDocumentPriority(p)); } } } - -void -PriorityConverterTest::testLowestPriorityIsReturnedForUnknownCode() -{ - CPPUNIT_ASSERT_EQUAL(255, - static_cast<int>(_converter->toStoragePriority( - static_cast<documentapi::Priority::Value>(123)))); +TEST_F(PriorityConverterTest, lowest_priority_is_returned_for_unknown_code) { + EXPECT_EQ(255, static_cast<int>(_converter->toStoragePriority( + static_cast<documentapi::Priority::Value>(123)))); } } diff --git a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp index b726a24b6b6..dc5324c00e3 100644 --- a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp +++ b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp @@ -3,30 +3,22 @@ #include <vespa/storage/storageserver/service_layer_error_listener.h> #include <vespa/storage/storageserver/mergethrottler.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vdstestlib/cppunit/dirconfig.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> +#include <vespa/vespalib/gtest/gtest.h> -namespace storage { +using namespace ::testing; -class ServiceLayerErrorListenerTest : public CppUnit::TestFixture { -public: - CPPUNIT_TEST_SUITE(ServiceLayerErrorListenerTest); - CPPUNIT_TEST(shutdown_invoked_on_fatal_error); - CPPUNIT_TEST(merge_throttle_backpressure_invoked_on_resource_exhaustion_error); - CPPUNIT_TEST_SUITE_END(); +namespace storage { - void shutdown_invoked_on_fatal_error(); - void merge_throttle_backpressure_invoked_on_resource_exhaustion_error(); +struct ServiceLayerErrorListenerTest : Test { }; -CPPUNIT_TEST_SUITE_REGISTRATION(ServiceLayerErrorListenerTest); - namespace { class TestShutdownListener - : public framework::defaultimplementation::ShutdownListener + : public framework::defaultimplementation::ShutdownListener { public: TestShutdownListener() : _reason() {} @@ -52,31 +44,31 @@ struct Fixture { ~Fixture(); }; -Fixture::~Fixture() {} +Fixture::~Fixture() = default; } -void ServiceLayerErrorListenerTest::shutdown_invoked_on_fatal_error() { +TEST_F(ServiceLayerErrorListenerTest, shutdown_invoked_on_fatal_error) { Fixture f; f.app.getComponentRegister().registerShutdownListener(f.shutdown_listener); - CPPUNIT_ASSERT(!f.shutdown_listener.shutdown_requested()); + EXPECT_FALSE(f.shutdown_listener.shutdown_requested()); f.error_listener.on_fatal_error("eject! eject!"); - CPPUNIT_ASSERT(f.shutdown_listener.shutdown_requested()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("eject! eject!"), f.shutdown_listener.reason()); + EXPECT_TRUE(f.shutdown_listener.shutdown_requested()); + EXPECT_EQ("eject! eject!", f.shutdown_listener.reason()); // Should only be invoked once f.error_listener.on_fatal_error("here be dragons"); - CPPUNIT_ASSERT_EQUAL(vespalib::string("eject! eject!"), f.shutdown_listener.reason()); + EXPECT_EQ("eject! eject!", f.shutdown_listener.reason()); } -void ServiceLayerErrorListenerTest::merge_throttle_backpressure_invoked_on_resource_exhaustion_error() { +TEST_F(ServiceLayerErrorListenerTest, merge_throttle_backpressure_invoked_on_resource_exhaustion_error) { Fixture f; - CPPUNIT_ASSERT(!f.merge_throttler.backpressure_mode_active()); + EXPECT_FALSE(f.merge_throttler.backpressure_mode_active()); f.error_listener.on_resource_exhaustion_error("buy more RAM!"); - CPPUNIT_ASSERT(f.merge_throttler.backpressure_mode_active()); + EXPECT_TRUE(f.merge_throttler.backpressure_mode_active()); } } diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index cdf990fa28f..c2074c53dd7 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cppunit/extensions/HelperMacros.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> @@ -11,15 +10,17 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> using storage::lib::NodeState; using storage::lib::NodeType; using storage::lib::State; using storage::lib::ClusterState; +using namespace ::testing; namespace storage { -struct StateManagerTest : public CppUnit::TestFixture { +struct StateManagerTest : Test { std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _upper; std::unique_ptr<metrics::MetricManager> _metricManager; @@ -28,26 +29,8 @@ struct StateManagerTest : public CppUnit::TestFixture { StateManagerTest(); - void setUp() override; - void tearDown() override; - - void testSystemState(); - void testReportedNodeState(); - void current_cluster_state_version_is_included_in_host_info_json(); - void can_explicitly_send_get_node_state_reply(); - void explicit_node_state_replying_without_pending_request_immediately_replies_on_next_request(); - void immediate_node_state_replying_is_tracked_per_controller(); - void activation_command_is_bounced_with_current_cluster_state_version(); - - CPPUNIT_TEST_SUITE(StateManagerTest); - CPPUNIT_TEST(testSystemState); - CPPUNIT_TEST(testReportedNodeState); - CPPUNIT_TEST(current_cluster_state_version_is_included_in_host_info_json); - CPPUNIT_TEST(can_explicitly_send_get_node_state_reply); - CPPUNIT_TEST(explicit_node_state_replying_without_pending_request_immediately_replies_on_next_request); - CPPUNIT_TEST(immediate_node_state_replying_is_tracked_per_controller); - CPPUNIT_TEST(activation_command_is_bounced_with_current_cluster_state_version); - CPPUNIT_TEST_SUITE_END(); + void SetUp() override; + void TearDown() override; void force_current_cluster_state_version(uint32_t version); void mark_reported_node_state_up(); @@ -55,52 +38,49 @@ struct StateManagerTest : public CppUnit::TestFixture { void assert_ok_get_node_state_reply_sent_and_clear(); void clear_sent_replies(); void mark_reply_observed_from_n_controllers(uint16_t n); -}; -CPPUNIT_TEST_SUITE_REGISTRATION(StateManagerTest); + std::string get_node_info() const { + return _manager->getNodeInfo(); + } +}; StateManagerTest::StateManagerTest() : _node(), _upper(), - _manager(0), - _lower(0) + _manager(nullptr), + _lower(nullptr) { } void -StateManagerTest::setUp() { - try{ - vdstestlib::DirConfig config(getStandardConfig(true)); - _node.reset(new TestServiceLayerApp(DiskCount(1), NodeIndex(2))); - // Clock will increase 1 sec per call. - _node->getClock().setAbsoluteTimeInSeconds(1); - _metricManager.reset(new metrics::MetricManager); - _upper.reset(new DummyStorageLink()); - _manager = new StateManager(_node->getComponentRegister(), - *_metricManager, - std::unique_ptr<HostInfo>(new HostInfo)); - _lower = new DummyStorageLink(); - _upper->push_back(StorageLink::UP(_manager)); - _upper->push_back(StorageLink::UP(_lower)); - _upper->open(); - } catch (std::exception& e) { - std::cerr << "Failed to static initialize objects: " << e.what() - << "\n"; - } +StateManagerTest::SetUp() { + vdstestlib::DirConfig config(getStandardConfig(true)); + _node = std::make_unique<TestServiceLayerApp>(DiskCount(1), NodeIndex(2)); + // Clock will increase 1 sec per call. + _node->getClock().setAbsoluteTimeInSeconds(1); + _metricManager = std::make_unique<metrics::MetricManager>(); + _upper = std::make_unique<DummyStorageLink>(); + _manager = new StateManager(_node->getComponentRegister(), + *_metricManager, + std::make_unique<HostInfo>()); + _lower = new DummyStorageLink(); + _upper->push_back(StorageLink::UP(_manager)); + _upper->push_back(StorageLink::UP(_lower)); + _upper->open(); } void -StateManagerTest::tearDown() { - CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumCommands()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumCommands()); - _manager = 0; - _lower = 0; +StateManagerTest::TearDown() { + assert(_lower->getNumReplies() == 0); + assert(_lower->getNumCommands() == 0); + assert(_upper->getNumReplies() == 0); + assert(_upper->getNumCommands() == 0); + _manager = nullptr; + _lower = nullptr; _upper->close(); _upper->flush(); - _upper.reset(0); - _node.reset(0); + _upper.reset(); + _node.reset(); _metricManager.reset(); } @@ -112,107 +92,96 @@ void StateManagerTest::force_current_cluster_state_version(uint32_t version) { #define GET_ONLY_OK_REPLY(varname) \ { \ - CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); \ - CPPUNIT_ASSERT(_upper->getReply(0)->getType().isReply()); \ + ASSERT_EQ(size_t(1), _upper->getNumReplies()); \ + ASSERT_TRUE(_upper->getReply(0)->getType().isReply()); \ varname = std::dynamic_pointer_cast<api::StorageReply>( \ _upper->getReply(0)); \ - CPPUNIT_ASSERT(varname != 0); \ + ASSERT_TRUE(varname.get() != nullptr); \ _upper->reset(); \ - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), \ - varname->getResult()); \ + ASSERT_EQ(api::ReturnCode(api::ReturnCode::OK), \ + varname->getResult()); \ } -void -StateManagerTest::testSystemState() -{ +TEST_F(StateManagerTest, cluster_state) { std::shared_ptr<api::StorageReply> reply; - // Verify initial state on startup - ClusterState::CSP currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); - CPPUNIT_ASSERT_EQUAL(std::string("cluster:d"), - currentState->toString(false)); + // Verify initial state on startup + auto currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); + EXPECT_EQ("cluster:d", currentState->toString(false)); - NodeState::CSP currentNodeState = _manager->getCurrentNodeState(); - CPPUNIT_ASSERT_EQUAL(std::string("s:d"), currentNodeState->toString(false)); + auto currentNodeState = _manager->getCurrentNodeState(); + EXPECT_EQ("s:d", currentNodeState->toString(false)); ClusterState sendState("storage:4 .2.s:m"); - std::shared_ptr<api::SetSystemStateCommand> cmd( - new api::SetSystemStateCommand(sendState)); + auto cmd = std::make_shared<api::SetSystemStateCommand>(sendState); _upper->sendDown(cmd); GET_ONLY_OK_REPLY(reply); currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); - CPPUNIT_ASSERT_EQUAL(sendState, *currentState); + EXPECT_EQ(sendState, *currentState); currentNodeState = _manager->getCurrentNodeState(); - CPPUNIT_ASSERT_EQUAL(std::string("s:m"), currentNodeState->toString(false)); + EXPECT_EQ("s:m", currentNodeState->toString(false)); } namespace { - struct MyStateListener : public StateListener { - const NodeStateUpdater& updater; - lib::NodeState current; - std::ostringstream ost; - - MyStateListener(const NodeStateUpdater& upd) - : updater(upd), current(*updater.getReportedNodeState()) {} - ~MyStateListener() { } - - void handleNewState() override { - ost << current << " -> "; - current = *updater.getReportedNodeState(); - ost << current << "\n"; - } - }; +struct MyStateListener : public StateListener { + const NodeStateUpdater& updater; + lib::NodeState current; + std::ostringstream ost; + + MyStateListener(const NodeStateUpdater& upd) + : updater(upd), current(*updater.getReportedNodeState()) {} + ~MyStateListener() override = default; + + void handleNewState() override { + ost << current << " -> "; + current = *updater.getReportedNodeState(); + ost << current << "\n"; + } +}; } -void -StateManagerTest::testReportedNodeState() -{ +TEST_F(StateManagerTest, reported_node_state) { std::shared_ptr<api::StorageReply> reply; - // Add a state listener to check that we get events. + // Add a state listener to check that we get events. MyStateListener stateListener(*_manager); _manager->addStateListener(stateListener); - // Test that initial state is initializing - NodeState::CSP nodeState = _manager->getReportedNodeState(); - CPPUNIT_ASSERT_EQUAL(std::string("s:i b:58 i:0 t:1"), nodeState->toString(false)); - // Test that it works to update the state + // Test that initial state is initializing + auto nodeState = _manager->getReportedNodeState(); + EXPECT_EQ("s:i b:58 i:0 t:1", nodeState->toString(false)); + // Test that it works to update the state { - NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); + auto lock = _manager->grabStateChangeLock(); NodeState ns(*_manager->getReportedNodeState()); ns.setState(State::UP); _manager->setReportedNodeState(ns); } - // And that we get the change both through state interface + // And that we get the change both through state interface nodeState = _manager->getReportedNodeState(); - CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), - nodeState->toString(false)); - // And get node state command (no expected state) - std::shared_ptr<api::GetNodeStateCommand> cmd( - new api::GetNodeStateCommand(lib::NodeState::UP())); + EXPECT_EQ("s:u b:58 t:1", nodeState->toString(false)); + // And get node state command (no expected state) + auto cmd = std::make_shared<api::GetNodeStateCommand>(lib::NodeState::UP()); _upper->sendDown(cmd); GET_ONLY_OK_REPLY(reply); - CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, - reply->getType()); - nodeState.reset(new NodeState( - dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); - CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), - nodeState->toString(false)); - // We should also get it with wrong expected state - cmd.reset(new api::GetNodeStateCommand(lib::NodeState::UP(new NodeState(NodeType::STORAGE, State::INITIALIZING)))); + ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); + nodeState = std::make_shared<NodeState>( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); + EXPECT_EQ("s:u b:58 t:1", nodeState->toString(false)); + // We should also get it with wrong expected state + cmd = std::make_shared<api::GetNodeStateCommand>( + std::make_unique<NodeState>(NodeType::STORAGE, State::INITIALIZING)); _upper->sendDown(cmd); GET_ONLY_OK_REPLY(reply); - CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, - reply->getType()); - nodeState.reset(new NodeState( - dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); - CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), - nodeState->toString(false)); - // With correct wanted state we should not get response right away - cmd.reset(new api::GetNodeStateCommand( - lib::NodeState::UP(new NodeState("s:u b:58 t:1", &NodeType::STORAGE)))); + ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); + nodeState = std::make_unique<NodeState>( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); + EXPECT_EQ("s:u b:58 t:1", nodeState->toString(false)); + // With correct wanted state we should not get response right away + cmd = std::make_shared<api::GetNodeStateCommand>( + std::make_unique<lib::NodeState>("s:u b:58 t:1", &NodeType::STORAGE)); _upper->sendDown(cmd); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); - // But when we update state, we get the reply + ASSERT_EQ(size_t(0), _upper->getNumReplies()); + // But when we update state, we get the reply { NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); NodeState ns(*_manager->getReportedNodeState()); @@ -222,16 +191,14 @@ StateManagerTest::testReportedNodeState() } GET_ONLY_OK_REPLY(reply); - CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, - reply->getType()); - nodeState.reset(new NodeState( - dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); - CPPUNIT_ASSERT_EQUAL(std::string("s:s b:58 t:1 m:Stopping\\x20node"), - nodeState->toString(false)); - - // Removing state listener, it stops getting updates + ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); + nodeState = std::make_unique<NodeState>( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); + EXPECT_EQ("s:s b:58 t:1 m:Stopping\\x20node", nodeState->toString(false)); + + // Removing state listener, it stops getting updates _manager->removeStateListener(stateListener); - // Do another update which listener should not get.. + // Do another update which listener should not get.. { NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); NodeState ns(*_manager->getReportedNodeState()); @@ -241,35 +208,34 @@ StateManagerTest::testReportedNodeState() std::string expectedEvents = "s:i b:58 i:0 t:1 -> s:u b:58 t:1\n" "s:u b:58 t:1 -> s:s b:58 t:1 m:Stopping\\x20node\n"; - CPPUNIT_ASSERT_EQUAL(expectedEvents, stateListener.ost.str()); + EXPECT_EQ(expectedEvents, stateListener.ost.str()); } -void StateManagerTest::current_cluster_state_version_is_included_in_host_info_json() { +TEST_F(StateManagerTest, current_cluster_state_version_is_included_in_host_info_json) { force_current_cluster_state_version(123); - std::string nodeInfoString(_manager->getNodeInfo()); + std::string nodeInfoString = get_node_info(); vespalib::Memory goldenMemory(nodeInfoString); vespalib::Slime nodeInfo; vespalib::slime::JsonFormat::decode(nodeInfoString, nodeInfo); - vespalib::slime::Symbol lookupSymbol = - nodeInfo.lookup("cluster-state-version"); + vespalib::slime::Symbol lookupSymbol = nodeInfo.lookup("cluster-state-version"); if (lookupSymbol.undefined()) { - CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + FAIL() << "No cluster-state-version was found in the node info"; } auto& cursor = nodeInfo.get(); auto& clusterStateVersionCursor = cursor["cluster-state-version"]; if (!clusterStateVersionCursor.valid()) { - CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + FAIL() << "No cluster-state-version was found in the node info"; } if (clusterStateVersionCursor.type().getId() != vespalib::slime::LONG::ID) { - CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + FAIL() << "No cluster-state-version was found in the node info"; } int version = clusterStateVersionCursor.asLong(); - CPPUNIT_ASSERT_EQUAL(123, version); + EXPECT_EQ(123, version); } void StateManagerTest::mark_reported_node_state_up() { @@ -286,10 +252,10 @@ void StateManagerTest::send_down_get_node_state_request(uint16_t controller_inde } void StateManagerTest::assert_ok_get_node_state_reply_sent_and_clear() { - CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); + ASSERT_EQ(1, _upper->getNumReplies()); std::shared_ptr<api::StorageReply> reply; GET_ONLY_OK_REPLY(reply); // Implicitly clears messages from _upper - CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, reply->getType()); + ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); } void StateManagerTest::clear_sent_replies() { @@ -299,11 +265,11 @@ void StateManagerTest::clear_sent_replies() { void StateManagerTest::mark_reply_observed_from_n_controllers(uint16_t n) { for (uint16_t i = 0; i < n; ++i) { send_down_get_node_state_request(i); - assert_ok_get_node_state_reply_sent_and_clear(); + ASSERT_NO_FATAL_FAILURE(assert_ok_get_node_state_reply_sent_and_clear()); } } -void StateManagerTest::can_explicitly_send_get_node_state_reply() { +TEST_F(StateManagerTest, can_explicitly_send_get_node_state_reply) { mark_reported_node_state_up(); // Must "pre-trigger" that a controller has already received a GetNodeState // reply, or an immediate reply will be sent by default when the first request @@ -311,13 +277,13 @@ void StateManagerTest::can_explicitly_send_get_node_state_reply() { mark_reply_observed_from_n_controllers(1); send_down_get_node_state_request(0); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + ASSERT_EQ(0, _upper->getNumReplies()); _manager->immediately_send_get_node_state_replies(); - assert_ok_get_node_state_reply_sent_and_clear(); + ASSERT_NO_FATAL_FAILURE(assert_ok_get_node_state_reply_sent_and_clear()); } -void StateManagerTest::explicit_node_state_replying_without_pending_request_immediately_replies_on_next_request() { +TEST_F(StateManagerTest, explicit_node_state_replying_without_pending_request_immediately_replies_on_next_request) { mark_reported_node_state_up(); mark_reply_observed_from_n_controllers(1); @@ -325,13 +291,13 @@ void StateManagerTest::explicit_node_state_replying_without_pending_request_imme _manager->immediately_send_get_node_state_replies(); send_down_get_node_state_request(0); - assert_ok_get_node_state_reply_sent_and_clear(); + ASSERT_NO_FATAL_FAILURE(assert_ok_get_node_state_reply_sent_and_clear()); // Sending a new request should now _not_ immediately receive a reply send_down_get_node_state_request(0); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + ASSERT_EQ(0, _upper->getNumReplies()); } -void StateManagerTest::immediate_node_state_replying_is_tracked_per_controller() { +TEST_F(StateManagerTest, immediate_node_state_replying_is_tracked_per_controller) { mark_reported_node_state_up(); mark_reply_observed_from_n_controllers(3); @@ -340,17 +306,17 @@ void StateManagerTest::immediate_node_state_replying_is_tracked_per_controller() send_down_get_node_state_request(0); send_down_get_node_state_request(1); send_down_get_node_state_request(2); - CPPUNIT_ASSERT_EQUAL(size_t(3), _upper->getNumReplies()); + ASSERT_EQ(3, _upper->getNumReplies()); clear_sent_replies(); // Sending a new request should now _not_ immediately receive a reply send_down_get_node_state_request(0); send_down_get_node_state_request(1); send_down_get_node_state_request(2); - CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + ASSERT_EQ(0, _upper->getNumReplies()); } -void StateManagerTest::activation_command_is_bounced_with_current_cluster_state_version() { +TEST_F(StateManagerTest, activation_command_is_bounced_with_current_cluster_state_version) { force_current_cluster_state_version(12345); auto cmd = std::make_shared<api::ActivateClusterStateVersionCommand>(12340); @@ -358,13 +324,13 @@ void StateManagerTest::activation_command_is_bounced_with_current_cluster_state_ cmd->setSourceIndex(0); _upper->sendDown(cmd); - CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); + ASSERT_EQ(1, _upper->getNumReplies()); std::shared_ptr<api::StorageReply> reply; GET_ONLY_OK_REPLY(reply); // Implicitly clears messages from _upper - CPPUNIT_ASSERT_EQUAL(api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_REPLY, reply->getType()); + ASSERT_EQ(api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_REPLY, reply->getType()); auto& activate_reply = dynamic_cast<api::ActivateClusterStateVersionReply&>(*reply); - CPPUNIT_ASSERT_EQUAL(uint32_t(12340), activate_reply.activateVersion()); - CPPUNIT_ASSERT_EQUAL(uint32_t(12345), activate_reply.actualVersion()); + EXPECT_EQ(12340, activate_reply.activateVersion()); + EXPECT_EQ(12345, activate_reply.actualVersion()); } } // storage diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index d0cdf41823b..c84f9311c52 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <cppunit/extensions/HelperMacros.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/storageserver/applicationgenerationfetcher.h> @@ -11,10 +10,13 @@ #include <tests/common/dummystoragelink.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP(".test.statereporter"); +using namespace ::testing; + namespace storage { class DummyApplicationGenerationFether : public ApplicationGenerationFetcher { @@ -23,7 +25,7 @@ public: std::string getComponentName() const override { return "component"; } }; -struct StateReporterTest : public CppUnit::TestFixture { +struct StateReporterTest : Test { FastOS_ThreadPool _threadPool; framework::defaultimplementation::FakeClock* _clock; std::unique_ptr<TestServiceLayerApp> _node; @@ -37,87 +39,72 @@ struct StateReporterTest : public CppUnit::TestFixture { StateReporterTest(); - void setUp() override; - void tearDown() override; + void SetUp() override; + void TearDown() override; void runLoad(uint32_t count = 1); +}; - void testReportConfigGeneration(); - void testReportHealth(); - void testReportMetrics(); +namespace { - CPPUNIT_TEST_SUITE(StateReporterTest); - CPPUNIT_TEST(testReportConfigGeneration); - CPPUNIT_TEST(testReportHealth); - CPPUNIT_TEST(testReportMetrics); - CPPUNIT_TEST_SUITE_END(); +struct MetricClock : public metrics::MetricManager::Timer +{ + framework::Clock& _clock; + explicit MetricClock(framework::Clock& c) : _clock(c) {} + time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } + time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } }; -CPPUNIT_TEST_SUITE_REGISTRATION(StateReporterTest); - -namespace { - struct MetricClock : public metrics::MetricManager::Timer - { - framework::Clock& _clock; - MetricClock(framework::Clock& c) : _clock(c) {} - time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } - time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } - }; } StateReporterTest::StateReporterTest() : _threadPool(256*1024), - _clock(0), + _clock(nullptr), _top(), _stateReporter() { } -void StateReporterTest::setUp() { - _config.reset(new vdstestlib::DirConfig(getStandardConfig(true, "statereportertest"))); +void StateReporterTest::SetUp() { + _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, "statereportertest")); assert(system(("rm -rf " + getRootFolder(*_config)).c_str()) == 0); - try { - _node.reset(new TestServiceLayerApp(DiskCount(4), NodeIndex(0), - _config->getConfigId())); - _node->setupDummyPersistence(); - _clock = &_node->getClock(); - _clock->setAbsoluteTimeInSeconds(1000000); - _top.reset(new DummyStorageLink); - } catch (config::InvalidConfigException& e) { - fprintf(stderr, "%s\n", e.what()); - } - _metricManager.reset(new metrics::MetricManager( - std::unique_ptr<metrics::MetricManager::Timer>( - new MetricClock(*_clock)))); + + _node = std::make_unique<TestServiceLayerApp>(DiskCount(4), NodeIndex(0), _config->getConfigId()); + _node->setupDummyPersistence(); + _clock = &_node->getClock(); + _clock->setAbsoluteTimeInSeconds(1000000); + _top = std::make_unique<DummyStorageLink>(); + + _metricManager = std::make_unique<metrics::MetricManager>(std::make_unique<MetricClock>(*_clock)); _topSet.reset(new metrics::MetricSet("vds", {}, "")); { metrics::MetricLockGuard guard(_metricManager->getMetricLock()); _metricManager->registerMetric(guard, *_topSet); } - _stateReporter.reset(new StateReporter( + _stateReporter = std::make_unique<StateReporter>( _node->getComponentRegister(), *_metricManager, _generationFetcher, - "status")); + "status"); uint16_t diskCount = _node->getPartitions().size(); documentapi::LoadTypeSet::SP loadTypes(_node->getLoadTypes()); - _filestorMetrics.reset(new FileStorMetrics(_node->getLoadTypes()->getMetricLoadTypes())); + _filestorMetrics = std::make_shared<FileStorMetrics>(_node->getLoadTypes()->getMetricLoadTypes()); _filestorMetrics->initDiskMetrics(diskCount, loadTypes->getMetricLoadTypes(), 1, 1); _topSet->registerMetric(*_filestorMetrics); _metricManager->init(_config->getConfigId(), _node->getThreadPool()); } -void StateReporterTest::tearDown() { +void StateReporterTest::TearDown() { _metricManager->stop(); - _stateReporter.reset(0); - _topSet.reset(0); - _metricManager.reset(0); - _top.reset(0); - _node.reset(0); - _config.reset(0); + _stateReporter.reset(); + _topSet.reset(); + _metricManager.reset(); + _top.reset(); + _node.reset(); + _config.reset(); _filestorMetrics.reset(); } @@ -129,17 +116,15 @@ vespalib::Slime slime; \ vespalib::SimpleBuffer buffer; \ JsonFormat::encode(slime, buffer, false); \ if (parsed == 0) { \ - std::ostringstream error; \ - error << "Failed to parse JSON: '\n" \ - << jsonData << "'\n:" << buffer.get().make_string() << "\n"; \ - CPPUNIT_ASSERT_EQUAL_MSG(error.str(), jsonData.size(), parsed); \ + ASSERT_EQ(jsonData.size(), parsed) << "Failed to parse JSON: '\n" \ + << jsonData << "':" << buffer.get().make_string(); \ } \ } #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - CPPUNIT_ASSERT_EQUAL( \ + ASSERT_EQ( \ generation, \ slime.get()["config"][component]["generation"].asDouble()); \ } @@ -147,10 +132,10 @@ vespalib::Slime slime; \ #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - CPPUNIT_ASSERT_EQUAL( \ + ASSERT_EQ( \ vespalib::string(code), \ slime.get()["status"]["code"].asString().make_string()); \ - CPPUNIT_ASSERT_EQUAL( \ + ASSERT_EQ( \ vespalib::string(message), \ slime.get()["status"]["message"].asString().make_string()); \ } @@ -161,7 +146,6 @@ vespalib::Slime slime; \ double getCount = -1; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ - /*std::cerr << "\nmetric count=" << metricCount << "\n";*/ \ for (size_t j=0; j<metricCount; j++) { \ const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ .asString().make_string(); \ @@ -177,22 +161,21 @@ vespalib::Slime slime; \ .asDouble(); \ } \ } \ - CPPUNIT_ASSERT_EQUAL(expGetCount, getCount); \ - CPPUNIT_ASSERT_EQUAL(expPutCount, putCount); \ - CPPUNIT_ASSERT(metricCount > 100); \ + ASSERT_EQ(expGetCount, getCount); \ + ASSERT_EQ(expPutCount, putCount); \ + ASSERT_GT(metricCount, 100); \ } -void StateReporterTest::testReportConfigGeneration() { +TEST_F(StateReporterTest, report_config_generation) { std::ostringstream ost; framework::HttpUrlPath path("/state/v1/config"); _stateReporter->reportStatus(ost, path); std::string jsonData = ost.str(); - //std::cerr << "\nConfig: " << jsonData << "\n"; ASSERT_GENERATION(jsonData, "component", 1.0); } -void StateReporterTest::testReportHealth() { +TEST_F(StateReporterTest, report_health) { const int stateCount = 7; const lib::NodeState nodeStates[stateCount] = { lib::NodeState(lib::NodeType::STORAGE, lib::State::UNKNOWN), @@ -228,23 +211,22 @@ void StateReporterTest::testReportHealth() { std::ostringstream ost; _stateReporter->reportStatus(ost, path); std::string jsonData = ost.str(); - //std::cerr << "\nHealth " << i << ":" << jsonData << "\n"; ASSERT_NODE_STATUS(jsonData, codes[i], messages[i]); } } -void StateReporterTest::testReportMetrics() { +TEST_F(StateReporterTest, report_metrics) { FileStorDiskMetrics& disk0(*_filestorMetrics->disks[0]); FileStorThreadMetrics& thread0(*disk0.threads[0]); - LOG(info, "Adding to get metric"); + LOG(debug, "Adding to get metric"); using documentapi::LoadType; thread0.get[LoadType::DEFAULT].count.inc(1); - LOG(info, "Waiting for 5 minute snapshot to be taken"); + LOG(debug, "Waiting for 5 minute snapshot to be taken"); // Wait until active metrics have been added to 5 min snapshot and reset - for (uint32_t i=0; i<6; ++i) { + for (uint32_t i = 0; i < 6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); while ( @@ -254,7 +236,7 @@ void StateReporterTest::testReportMetrics() { FastOS_Thread::Sleep(1); } } - LOG(info, "5 minute snapshot should have been taken. Adding put count"); + LOG(debug, "5 minute snapshot should have been taken. Adding put count"); thread0.put[LoadType::DEFAULT].count.inc(1); @@ -264,12 +246,11 @@ void StateReporterTest::testReportMetrics() { "/state/v1/metrics?consumer=status" }; - for (int i=0; i<pathCount; i++) { + for (int i = 0; i < pathCount; i++) { framework::HttpUrlPath path(paths[i]); std::ostringstream ost; _stateReporter->reportStatus(ost, path); std::string jsonData = ost.str(); - //std::cerr << "\nMetrics:" << jsonData << "\n"; ASSERT_METRIC_GET_PUT(jsonData, 1.0, 0.0); } } diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt index 438086fddd3..c1b19960cea 100644 --- a/storage/src/tests/visiting/CMakeLists.txt +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -1,19 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Remove test library when all tests have been migrated to gtest. -vespa_add_library(storage_testvisiting TEST +vespa_add_executable(storage_visiting_gtest_runner_app TEST SOURCES commandqueuetest.cpp memory_bounded_trace_test.cpp visitormanagertest.cpp visitortest.cpp - DEPENDS - storage - storage_teststorageserver -) - -vespa_add_executable(storage_visiting_gtest_runner_app TEST - SOURCES gtest_runner.cpp DEPENDS storage diff --git a/storage/src/tests/visiting/commandqueuetest.cpp b/storage/src/tests/visiting/commandqueuetest.cpp index e335df353f6..c152e4c5191 100644 --- a/storage/src/tests/visiting/commandqueuetest.cpp +++ b/storage/src/tests/visiting/commandqueuetest.cpp @@ -3,62 +3,44 @@ #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/storage/visiting/commandqueue.h> #include <vespa/storageapi/message/visitor.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/document/test/make_bucket_space.h> - +#include <vespa/vespalib/gtest/gtest.h> using vespalib::string; using document::test::makeBucketSpace; +using namespace ::testing; namespace storage { -struct CommandQueueTest : public CppUnit::TestFixture -{ - void testFIFO(); - void testFIFOWithPriorities(); - void testReleaseOldest(); - void testReleaseLowestPriority(); - void testDeleteIterator(); - - CPPUNIT_TEST_SUITE(CommandQueueTest); - CPPUNIT_TEST(testFIFO); - CPPUNIT_TEST(testFIFOWithPriorities); - CPPUNIT_TEST(testReleaseOldest); - CPPUNIT_TEST(testReleaseLowestPriority); - CPPUNIT_TEST(testDeleteIterator); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(CommandQueueTest); - namespace { - std::shared_ptr<api::CreateVisitorCommand> getCommand( - vespalib::stringref name, int timeout, - uint8_t priority = 0) - { - vespalib::asciistream ost; - ost << name << " t=" << timeout << " p=" << static_cast<unsigned int>(priority); - // Piggyback name in document selection - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "", "", ost.str())); - cmd->setQueueTimeout(timeout); - cmd->setPriority(priority); - return cmd; - } - const vespalib::string & - getCommandString(const std::shared_ptr<api::CreateVisitorCommand>& cmd) - { - return cmd->getDocumentSelection(); - } +std::shared_ptr<api::CreateVisitorCommand> getCommand( + vespalib::stringref name, int timeout, + uint8_t priority = 0) +{ + vespalib::asciistream ost; + ost << name << " t=" << timeout << " p=" << static_cast<unsigned int>(priority); + // Piggyback name in document selection + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand(makeBucketSpace(), "", "", ost.str())); + cmd->setQueueTimeout(timeout); + cmd->setPriority(priority); + return cmd; +} +const vespalib::string & +getCommandString(const std::shared_ptr<api::CreateVisitorCommand>& cmd) +{ + return cmd->getDocumentSelection(); } -void CommandQueueTest::testFIFO() { +} + +TEST(CommandQueueTest, fifo) { framework::defaultimplementation::FakeClock clock; CommandQueue<api::CreateVisitorCommand> queue(clock); - CPPUNIT_ASSERT(queue.empty()); + ASSERT_TRUE(queue.empty()); // Use all default priorities, meaning what comes out should be in the same order // as what went in queue.add(getCommand("first", 1)); @@ -69,64 +51,66 @@ void CommandQueueTest::testFIFO() { queue.add(getCommand("sixth", 14)); queue.add(getCommand("seventh", 7)); - CPPUNIT_ASSERT(!queue.empty()); - std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + ASSERT_FALSE(queue.empty()); + std::vector<std::shared_ptr<api::CreateVisitorCommand>> commands; for (;;) { std::shared_ptr<api::CreateVisitorCommand> cmd( queue.releaseNextCommand().first); if (cmd.get() == 0) break; commands.push_back(cmd); } - CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); - CPPUNIT_ASSERT_EQUAL(string("first t=1 p=0"), getCommandString(commands[0])); - CPPUNIT_ASSERT_EQUAL(string("second t=10 p=0"), getCommandString(commands[1])); - CPPUNIT_ASSERT_EQUAL(string("third t=5 p=0"), getCommandString(commands[2])); - CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=0"), getCommandString(commands[3])); - CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=0"), getCommandString(commands[4])); - CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=0"), getCommandString(commands[5])); - CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[6])); + ASSERT_EQ(7, commands.size()); + EXPECT_EQ("first t=1 p=0", getCommandString(commands[0])); + EXPECT_EQ("second t=10 p=0", getCommandString(commands[1])); + EXPECT_EQ("third t=5 p=0", getCommandString(commands[2])); + EXPECT_EQ("fourth t=0 p=0", getCommandString(commands[3])); + EXPECT_EQ("fifth t=3 p=0", getCommandString(commands[4])); + EXPECT_EQ("sixth t=14 p=0", getCommandString(commands[5])); + EXPECT_EQ("seventh t=7 p=0", getCommandString(commands[6])); } -void CommandQueueTest::testFIFOWithPriorities() { +TEST(CommandQueueTest, fifo_with_priorities) { framework::defaultimplementation::FakeClock clock; CommandQueue<api::CreateVisitorCommand> queue(clock); - CPPUNIT_ASSERT(queue.empty()); + ASSERT_TRUE(queue.empty()); queue.add(getCommand("first", 1, 10)); - CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(queue.peekLowestPriorityCommand())); + EXPECT_EQ("first t=1 p=10", getCommandString(queue.peekLowestPriorityCommand())); queue.add(getCommand("second", 10, 22)); queue.add(getCommand("third", 5, 9)); - CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(queue.peekLowestPriorityCommand())); + EXPECT_EQ("second t=10 p=22", getCommandString(queue.peekLowestPriorityCommand())); queue.add(getCommand("fourth", 0, 22)); queue.add(getCommand("fifth", 3, 22)); - CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(queue.peekLowestPriorityCommand())); + EXPECT_EQ("fifth t=3 p=22", getCommandString(queue.peekLowestPriorityCommand())); queue.add(getCommand("sixth", 14, 50)); queue.add(getCommand("seventh", 7, 0)); - CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(queue.peekLowestPriorityCommand())); + EXPECT_EQ("sixth t=14 p=50", getCommandString(queue.peekLowestPriorityCommand())); - CPPUNIT_ASSERT(!queue.empty()); - std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + ASSERT_FALSE(queue.empty()); + std::vector<std::shared_ptr<api::CreateVisitorCommand>> commands; for (;;) { std::shared_ptr<api::CreateVisitorCommand> cmdPeek(queue.peekNextCommand()); std::shared_ptr<api::CreateVisitorCommand> cmd(queue.releaseNextCommand().first); - if (cmd.get() == 0 || cmdPeek != cmd) break; + if (cmd.get() == 0 || cmdPeek != cmd) { + break; + } commands.push_back(cmd); } - CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); - CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[0])); - CPPUNIT_ASSERT_EQUAL(string("third t=5 p=9"), getCommandString(commands[1])); - CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(commands[2])); - CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(commands[3])); - CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=22"), getCommandString(commands[4])); - CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(commands[5])); - CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(commands[6])); + ASSERT_EQ(7, commands.size()); + EXPECT_EQ("seventh t=7 p=0", getCommandString(commands[0])); + EXPECT_EQ("third t=5 p=9", getCommandString(commands[1])); + EXPECT_EQ("first t=1 p=10", getCommandString(commands[2])); + EXPECT_EQ("second t=10 p=22", getCommandString(commands[3])); + EXPECT_EQ("fourth t=0 p=22", getCommandString(commands[4])); + EXPECT_EQ("fifth t=3 p=22", getCommandString(commands[5])); + EXPECT_EQ("sixth t=14 p=50", getCommandString(commands[6])); } -void CommandQueueTest::testReleaseOldest() { +TEST(CommandQueueTest, release_oldest) { framework::defaultimplementation::FakeClock clock(framework::defaultimplementation::FakeClock::FAKE_ABSOLUTE); CommandQueue<api::CreateVisitorCommand> queue(clock); - CPPUNIT_ASSERT(queue.empty()); + ASSERT_TRUE(queue.empty()); queue.add(getCommand("first", 10)); queue.add(getCommand("second", 100)); queue.add(getCommand("third", 1000)); @@ -134,32 +118,31 @@ void CommandQueueTest::testReleaseOldest() { queue.add(getCommand("fifth", 3000)); queue.add(getCommand("sixth", 400)); queue.add(getCommand("seventh", 700)); - CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + ASSERT_EQ(7u, queue.size()); - typedef CommandQueue<api::CreateVisitorCommand>::CommandEntry CommandEntry; + using CommandEntry = CommandQueue<api::CreateVisitorCommand>::CommandEntry; std::list<CommandEntry> timedOut(queue.releaseTimedOut()); - CPPUNIT_ASSERT(timedOut.empty()); + ASSERT_TRUE(timedOut.empty()); clock.addMilliSecondsToTime(400 * 1000); timedOut = queue.releaseTimedOut(); - CPPUNIT_ASSERT_EQUAL(size_t(4), timedOut.size()); + ASSERT_EQ(4, timedOut.size()); std::ostringstream ost; for (std::list<CommandEntry>::const_iterator it = timedOut.begin(); it != timedOut.end(); ++it) { ost << getCommandString(it->_command) << "\n"; } - CPPUNIT_ASSERT_EQUAL(std::string( - "fourth t=5 p=0\n" - "first t=10 p=0\n" - "second t=100 p=0\n" - "sixth t=400 p=0\n"), ost.str()); - CPPUNIT_ASSERT_EQUAL(3u, queue.size()); + EXPECT_EQ("fourth t=5 p=0\n" + "first t=10 p=0\n" + "second t=100 p=0\n" + "sixth t=400 p=0\n", ost.str()); + EXPECT_EQ(3u, queue.size()); } -void CommandQueueTest::testReleaseLowestPriority() { +TEST(CommandQueueTest, release_lowest_priority) { framework::defaultimplementation::FakeClock clock; CommandQueue<api::CreateVisitorCommand> queue(clock); - CPPUNIT_ASSERT(queue.empty()); + ASSERT_TRUE(queue.empty()); queue.add(getCommand("first", 1, 10)); queue.add(getCommand("second", 10, 22)); @@ -168,30 +151,32 @@ void CommandQueueTest::testReleaseLowestPriority() { queue.add(getCommand("fifth", 3, 22)); queue.add(getCommand("sixth", 14, 50)); queue.add(getCommand("seventh", 7, 0)); - CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + ASSERT_EQ(7u, queue.size()); - std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + std::vector<std::shared_ptr<api::CreateVisitorCommand>> commands; for (;;) { std::shared_ptr<api::CreateVisitorCommand> cmdPeek(queue.peekLowestPriorityCommand()); std::pair<std::shared_ptr<api::CreateVisitorCommand>, uint64_t> cmd( queue.releaseLowestPriorityCommand()); - if (cmd.first.get() == 0 || cmdPeek != cmd.first) break; + if (cmd.first.get() == 0 || cmdPeek != cmd.first) { + break; + } commands.push_back(cmd.first); } - CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); - CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(commands[0])); - CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(commands[1])); - CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=22"), getCommandString(commands[2])); - CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(commands[3])); - CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(commands[4])); - CPPUNIT_ASSERT_EQUAL(string("third t=5 p=9"), getCommandString(commands[5])); - CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[6])); + ASSERT_EQ(7, commands.size()); + EXPECT_EQ("sixth t=14 p=50", getCommandString(commands[0])); + EXPECT_EQ("fifth t=3 p=22", getCommandString(commands[1])); + EXPECT_EQ("fourth t=0 p=22", getCommandString(commands[2])); + EXPECT_EQ("second t=10 p=22", getCommandString(commands[3])); + EXPECT_EQ("first t=1 p=10", getCommandString(commands[4])); + EXPECT_EQ("third t=5 p=9", getCommandString(commands[5])); + EXPECT_EQ("seventh t=7 p=0", getCommandString(commands[6])); } -void CommandQueueTest::testDeleteIterator() { +TEST(CommandQueueTest, delete_iterator) { framework::defaultimplementation::FakeClock clock; CommandQueue<api::CreateVisitorCommand> queue(clock); - CPPUNIT_ASSERT(queue.empty()); + ASSERT_TRUE(queue.empty()); queue.add(getCommand("first", 10)); queue.add(getCommand("second", 100)); queue.add(getCommand("third", 1000)); @@ -199,28 +184,30 @@ void CommandQueueTest::testDeleteIterator() { queue.add(getCommand("fifth", 3000)); queue.add(getCommand("sixth", 400)); queue.add(getCommand("seventh", 700)); - CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + ASSERT_EQ(7u, queue.size()); CommandQueue<api::CreateVisitorCommand>::iterator it = queue.begin(); ++it; ++it; queue.erase(it); - CPPUNIT_ASSERT_EQUAL(6u, queue.size()); + ASSERT_EQ(6u, queue.size()); - std::vector<std::shared_ptr<api::CreateVisitorCommand> > cmds; + std::vector<std::shared_ptr<api::CreateVisitorCommand>> cmds; for (;;) { std::shared_ptr<api::CreateVisitorCommand> cmd( std::dynamic_pointer_cast<api::CreateVisitorCommand>( queue.releaseNextCommand().first)); - if (cmd.get() == 0) break; + if (cmd.get() == 0) { + break; + } cmds.push_back(cmd); } - CPPUNIT_ASSERT_EQUAL(size_t(6), cmds.size()); - CPPUNIT_ASSERT_EQUAL(string("first t=10 p=0"), getCommandString(cmds[0])); - CPPUNIT_ASSERT_EQUAL(string("second t=100 p=0"), getCommandString(cmds[1])); - CPPUNIT_ASSERT_EQUAL(string("fourth t=5 p=0"), getCommandString(cmds[2])); - CPPUNIT_ASSERT_EQUAL(string("fifth t=3000 p=0"), getCommandString(cmds[3])); - CPPUNIT_ASSERT_EQUAL(string("sixth t=400 p=0"), getCommandString(cmds[4])); - CPPUNIT_ASSERT_EQUAL(string("seventh t=700 p=0"), getCommandString(cmds[5])); + ASSERT_EQ(6, cmds.size()); + EXPECT_EQ("first t=10 p=0", getCommandString(cmds[0])); + EXPECT_EQ("second t=100 p=0", getCommandString(cmds[1])); + EXPECT_EQ("fourth t=5 p=0", getCommandString(cmds[2])); + EXPECT_EQ("fifth t=3000 p=0", getCommandString(cmds[3])); + EXPECT_EQ("sixth t=400 p=0", getCommandString(cmds[4])); + EXPECT_EQ("seventh t=700 p=0", getCommandString(cmds[5])); } } diff --git a/storage/src/tests/visiting/memory_bounded_trace_test.cpp b/storage/src/tests/visiting/memory_bounded_trace_test.cpp index 5d7dda16ced..0cfd3dad4c3 100644 --- a/storage/src/tests/visiting/memory_bounded_trace_test.cpp +++ b/storage/src/tests/visiting/memory_bounded_trace_test.cpp @@ -1,73 +1,43 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/visiting/memory_bounded_trace.h> +#include <vespa/vespalib/gtest/gtest.h> -namespace storage { - -class MemoryBoundedTraceTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(MemoryBoundedTraceTest); - CPPUNIT_TEST(noMemoryReportedUsedWhenEmpty); - CPPUNIT_TEST(memoryUsedIsStringLengthForLeafNode); - CPPUNIT_TEST(memoryUsedIsAccumulatedRecursivelyForNonLeafNodes); - CPPUNIT_TEST(traceNodesCanBeMovedAndImplicitlyCleared); - CPPUNIT_TEST(movedTraceTreeIsMarkedAsStrict); - CPPUNIT_TEST(canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound); - CPPUNIT_TEST(movedTreeIncludesStatsNodeWhenNodesOmitted); - CPPUNIT_TEST_SUITE_END(); - -public: - void noMemoryReportedUsedWhenEmpty(); - void memoryUsedIsStringLengthForLeafNode(); - void memoryUsedIsAccumulatedRecursivelyForNonLeafNodes(); - void traceNodesCanBeMovedAndImplicitlyCleared(); - void movedTraceTreeIsMarkedAsStrict(); - void canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound(); - void movedTreeIncludesStatsNodeWhenNodesOmitted(); -}; +using namespace ::testing; -CPPUNIT_TEST_SUITE_REGISTRATION(MemoryBoundedTraceTest); +namespace storage { -void -MemoryBoundedTraceTest::noMemoryReportedUsedWhenEmpty() -{ +TEST(MemoryBoundedTraceTest, no_memory_reported_used_when_empty) { MemoryBoundedTrace trace(100); - CPPUNIT_ASSERT_EQUAL(size_t(0), trace.getApproxMemoryUsed()); + EXPECT_EQ(0, trace.getApproxMemoryUsed()); } -void -MemoryBoundedTraceTest::memoryUsedIsStringLengthForLeafNode() -{ +TEST(MemoryBoundedTraceTest, memory_used_is_string_length_for_leaf_node) { MemoryBoundedTrace trace(100); - CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); - CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); + EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", 0))); + EXPECT_EQ(11, trace.getApproxMemoryUsed()); } -void -MemoryBoundedTraceTest::memoryUsedIsAccumulatedRecursivelyForNonLeafNodes() -{ +TEST(MemoryBoundedTraceTest, memory_used_is_accumulated_recursively_for_non_leaf_nodes) { MemoryBoundedTrace trace(100); mbus::TraceNode innerNode; innerNode.addChild("hello world"); innerNode.addChild("goodbye moon"); - CPPUNIT_ASSERT(trace.add(innerNode)); - CPPUNIT_ASSERT_EQUAL(size_t(23), trace.getApproxMemoryUsed()); + EXPECT_TRUE(trace.add(innerNode)); + EXPECT_EQ(23, trace.getApproxMemoryUsed()); } -void -MemoryBoundedTraceTest::traceNodesCanBeMovedAndImplicitlyCleared() -{ +TEST(MemoryBoundedTraceTest, trace_nodes_can_be_moved_and_implicitly_cleared) { MemoryBoundedTrace trace(100); - CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", 0))); mbus::TraceNode target; trace.moveTraceTo(target); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); - CPPUNIT_ASSERT_EQUAL(size_t(0), trace.getApproxMemoryUsed()); + EXPECT_EQ(1, target.getNumChildren()); + EXPECT_EQ(0, trace.getApproxMemoryUsed()); mbus::TraceNode emptinessCheck; trace.moveTraceTo(emptinessCheck); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), emptinessCheck.getNumChildren()); + EXPECT_EQ(0, emptinessCheck.getNumChildren()); } /** @@ -77,54 +47,46 @@ MemoryBoundedTraceTest::traceNodesCanBeMovedAndImplicitlyCleared() * best of my knowledge, since the internal backing data structure is an * ordered vector anyhow. */ -void -MemoryBoundedTraceTest::movedTraceTreeIsMarkedAsStrict() -{ +TEST(MemoryBoundedTraceTest, moved_trace_tree_is_marked_as_strict) { MemoryBoundedTrace trace(100); - CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", 0))); mbus::TraceNode target; trace.moveTraceTo(target); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); - CPPUNIT_ASSERT(target.getChild(0).isStrict()); + EXPECT_EQ(1, target.getNumChildren()); + EXPECT_TRUE(target.getChild(0).isStrict()); } -void -MemoryBoundedTraceTest::canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound() -{ +TEST(MemoryBoundedTraceTest, can_not_add_more_nodes_when_memory_used_exceeds_upper_bound) { // Note: we allow one complete node tree to exceed the bounds, but as soon // as the bound is exceeded no further nodes can be added. MemoryBoundedTrace trace(10); - CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); - CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); + EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", 0))); + EXPECT_EQ(11, trace.getApproxMemoryUsed()); - CPPUNIT_ASSERT(!trace.add(mbus::TraceNode("the quick red fox runs across " - "the freeway", 0))); - CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); + EXPECT_FALSE(trace.add(mbus::TraceNode("the quick red fox runs across " + "the freeway", 0))); + EXPECT_EQ(11, trace.getApproxMemoryUsed()); mbus::TraceNode target; trace.moveTraceTo(target); // Twice nested node (root -> added trace tree -> leaf with txt). - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); - CPPUNIT_ASSERT(target.getChild(0).getNumChildren() >= 1); - CPPUNIT_ASSERT_EQUAL(vespalib::string("hello world"), - target.getChild(0).getChild(0).getNote()); + EXPECT_EQ(1, target.getNumChildren()); + EXPECT_GE(target.getChild(0).getNumChildren(), 1); + EXPECT_EQ("hello world", target.getChild(0).getChild(0).getNote()); } -void -MemoryBoundedTraceTest::movedTreeIncludesStatsNodeWhenNodesOmitted() -{ +TEST(MemoryBoundedTraceTest, moved_tree_includes_stats_node_when_nodes_omitted) { MemoryBoundedTrace trace(5); - CPPUNIT_ASSERT(trace.add(mbus::TraceNode("abcdef", 0))); - CPPUNIT_ASSERT(!trace.add(mbus::TraceNode("ghijkjlmn", 0))); + EXPECT_TRUE(trace.add(mbus::TraceNode("abcdef", 0))); + EXPECT_FALSE(trace.add(mbus::TraceNode("ghijkjlmn", 0))); mbus::TraceNode target; trace.moveTraceTo(target); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getChild(0).getNumChildren()); + EXPECT_EQ(1, target.getNumChildren()); + EXPECT_EQ(2, target.getChild(0).getNumChildren()); vespalib::string expected("Trace too large; omitted 1 subsequent trace " "trees containing a total of 9 bytes"); - CPPUNIT_ASSERT_EQUAL(expected, target.getChild(0).getChild(1).getNote()); + EXPECT_EQ(expected, target.getChild(0).getChild(1).getNote()); } } // storage - diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index b9d4f24072d..1f2782de456 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -19,6 +19,8 @@ #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <gmock/gmock.h> #include <optional> #include <thread> #include <chrono> @@ -27,37 +29,17 @@ using document::test::makeDocumentBucket; using document::test::makeBucketSpace; using documentapi::Priority; using namespace std::chrono_literals; +using namespace ::testing; namespace storage { namespace { - typedef std::vector<api::StorageMessage::SP> msg_ptr_vector; -} -class VisitorManagerTest : public CppUnit::TestFixture -{ -private: - CPPUNIT_TEST_SUITE(VisitorManagerTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testResending); - CPPUNIT_TEST(testVisitEmptyBucket); - CPPUNIT_TEST(testMultiBucketVisit); - CPPUNIT_TEST(testNoBuckets); - CPPUNIT_TEST(testVisitPutsAndRemoves); - CPPUNIT_TEST(testVisitWithTimeframeAndSelection); - CPPUNIT_TEST(testVisitWithTimeframeAndBogusSelection); - CPPUNIT_TEST(testVisitorCallbacks); - CPPUNIT_TEST(testVisitorCleanup); - CPPUNIT_TEST(testAbortOnFailedVisitorInfo); - CPPUNIT_TEST(testAbortOnFieldPathError); - CPPUNIT_TEST(testVisitorQueueTimeout); - CPPUNIT_TEST(testVisitorProcessingTimeout); - CPPUNIT_TEST(testPrioritizedVisitorQueing); - CPPUNIT_TEST(testPrioritizedMaxConcurrentVisitors); - CPPUNIT_TEST(testVisitorQueingZeroQueueSize); - CPPUNIT_TEST(testHitCounter); - CPPUNIT_TEST(testStatusPage); - CPPUNIT_TEST_SUITE_END(); +using msg_ptr_vector = std::vector<api::StorageMessage::SP>; + +} +struct VisitorManagerTest : Test { +protected: static uint32_t docCount; std::vector<document::Document::SP > _documents; std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; @@ -65,18 +47,19 @@ private: std::unique_ptr<DummyStorageLink> _top; VisitorManager* _manager; -public: VisitorManagerTest() : _node() {} + ~VisitorManagerTest(); - // Not using setUp since can't throw exception out of it. + // Not using setUp since can't throw exception out of it. void initializeTest(); void addSomeRemoves(bool removeAll = false); - void tearDown() override; + void TearDown() override; TestVisitorMessageSession& getSession(uint32_t n); - uint64_t verifyCreateVisitorReply( + void verifyCreateVisitorReply( api::ReturnCode::Result expectedResult, int checkStatsDocsVisited = -1, - int checkStatsBytesVisited = -1); + int checkStatsBytesVisited = -1, + uint64_t* message_id_out = nullptr); void getMessagesAndReply( int expectedCount, TestVisitorMessageSession& session, @@ -86,31 +69,11 @@ public: std::optional<Priority::Value> priority = documentapi::Priority::PRI_NORMAL_4); uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); void finishAndWaitForVisitorSessionCompletion(uint32_t sessionIndex); - - void testNormalUsage(); - void testResending(); - void testVisitEmptyBucket(); - void testMultiBucketVisit(); - void testNoBuckets(); - void testVisitPutsAndRemoves(); - void testVisitWithTimeframeAndSelection(); - void testVisitWithTimeframeAndBogusSelection(); - void testVisitorCallbacks(); - void testVisitorCleanup(); - void testAbortOnFailedVisitorInfo(); - void testAbortOnFieldPathError(); - void testVisitorQueueTimeout(); - void testVisitorProcessingTimeout(); - void testPrioritizedVisitorQueing(); - void testPrioritizedMaxConcurrentVisitors(); - void testVisitorQueingZeroQueueSize(); - void testHitCounter(); - void testStatusPage(); }; -uint32_t VisitorManagerTest::docCount = 10; +VisitorManagerTest::~VisitorManagerTest() = default; -CPPUNIT_TEST_SUITE_REGISTRATION(VisitorManagerTest); +uint32_t VisitorManagerTest::docCount = 10; void VisitorManagerTest::initializeTest() @@ -118,28 +81,25 @@ VisitorManagerTest::initializeTest() vdstestlib::DirConfig config(getStandardConfig(true)); config.getConfig("stor-visitor").set("visitorthreads", "1"); - try { - _messageSessionFactory.reset( - new TestVisitorMessageSessionFactory(config.getConfigId())); - _node.reset( - new TestServiceLayerApp(config.getConfigId())); - _node->setupDummyPersistence(); - _node->getStateUpdater().setClusterState( - lib::ClusterState::CSP( - new lib::ClusterState("storage:1 distributor:1"))); - _top.reset(new DummyStorageLink()); - _top->push_back(std::unique_ptr<StorageLink>(_manager - = new VisitorManager( - config.getConfigId(), _node->getComponentRegister(), - *_messageSessionFactory))); - _top->push_back(std::unique_ptr<StorageLink>(new FileStorManager( - config.getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); - _manager->setTimeBetweenTicks(10); - _top->open(); - } catch (config::InvalidConfigException& e) { - fprintf(stderr, "%s\n", e.what()); - } - // Adding some documents so database isn't empty + _messageSessionFactory.reset( + new TestVisitorMessageSessionFactory(config.getConfigId())); + _node.reset( + new TestServiceLayerApp(config.getConfigId())); + _node->setupDummyPersistence(); + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP( + new lib::ClusterState("storage:1 distributor:1"))); + _top.reset(new DummyStorageLink()); + _top->push_back(std::unique_ptr<StorageLink>(_manager + = new VisitorManager( + config.getConfigId(), _node->getComponentRegister(), + *_messageSessionFactory))); + _top->push_back(std::unique_ptr<StorageLink>(new FileStorManager( + config.getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + _manager->setTimeBetweenTicks(10); + _top->open(); + + // Adding some documents so database isn't empty api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::string content( "To be, or not to be: that is the question:\n" @@ -212,13 +172,10 @@ VisitorManagerTest::initializeTest() _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); - std::shared_ptr<api::PutReply> reply( - std::dynamic_pointer_cast<api::PutReply>( - replies[0])); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), - reply->getResult()); + ASSERT_EQ(1, replies.size()); + auto reply = std::dynamic_pointer_cast<api::PutReply>(replies[0]); + ASSERT_TRUE(reply.get()); + ASSERT_EQ(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); } } @@ -236,15 +193,15 @@ VisitorManagerTest::addSomeRemoves(bool removeAll) _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + ASSERT_EQ(1, replies.size()); auto reply = std::dynamic_pointer_cast<api::RemoveReply>(replies[0]); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); + ASSERT_TRUE(reply.get()); + ASSERT_EQ(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); } } void -VisitorManagerTest::tearDown() +VisitorManagerTest::TearDown() { if (_top) { assert(_top->getNumReplies() == 0); @@ -277,7 +234,7 @@ VisitorManagerTest::getSession(uint32_t n) } std::this_thread::sleep_for(10ms); } - throw std::logic_error("unreachable"); + abort(); } void @@ -296,8 +253,7 @@ VisitorManagerTest::getMessagesAndReply( vespalib::MonitorGuard guard(session.getMonitor()); if (priority) { - CPPUNIT_ASSERT_EQUAL(*priority, - session.sentMessages[i]->getPriority()); + ASSERT_EQ(*priority, session.sentMessages[i]->getPriority()); } switch (session.sentMessages[i]->getType()) { @@ -327,34 +283,37 @@ VisitorManagerTest::getMessagesAndReply( } } -uint64_t +void VisitorManagerTest::verifyCreateVisitorReply( api::ReturnCode::Result expectedResult, int checkStatsDocsVisited, - int checkStatsBytesVisited) + int checkStatsBytesVisited, + uint64_t* message_id_out) { _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(1, (int)replies.size()); + ASSERT_EQ(1, replies.size()); std::shared_ptr<api::StorageMessage> msg(replies[0]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(expectedResult, reply->getResult().getResult()); + ASSERT_TRUE(reply.get()); + ASSERT_EQ(expectedResult, reply->getResult().getResult()); if (checkStatsDocsVisited >= 0) { - CPPUNIT_ASSERT_EQUAL(checkStatsDocsVisited, - int(reply->getVisitorStatistics().getDocumentsVisited())); + ASSERT_EQ(checkStatsDocsVisited, + reply->getVisitorStatistics().getDocumentsVisited()); } if (checkStatsBytesVisited >= 0) { - CPPUNIT_ASSERT_EQUAL(checkStatsBytesVisited, - int(reply->getVisitorStatistics().getBytesVisited())); + ASSERT_EQ(checkStatsBytesVisited, + reply->getVisitorStatistics().getBytesVisited()); } - return reply->getMsgId(); + if (message_id_out) { + *message_id_out = reply->getMsgId(); + } } uint32_t @@ -364,7 +323,6 @@ VisitorManagerTest::getMatchingDocuments(std::vector<document::Document::SP >& d for (uint32_t j=0; j<_documents.size(); ++j) { if (docs[i]->getId() == _documents[j]->getId() && *docs[i] == *_documents[j]) - { equalCount++; } @@ -374,9 +332,7 @@ VisitorManagerTest::getMatchingDocuments(std::vector<document::Document::SP >& d return equalCount; } -void -VisitorManagerTest::testHitCounter() -{ +TEST_F(VisitorManagerTest, hit_counter) { document::OrderingSpecification spec(document::OrderingSpecification::ASCENDING, 42, 7, 2); Visitor::HitCounter hitCounter(&spec); @@ -386,10 +342,10 @@ VisitorManagerTest::testHitCounter() hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:10:foo"), 450); hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:21:foo"), 450); - CPPUNIT_ASSERT_EQUAL(3, (int)hitCounter.getFirstPassHits()); - CPPUNIT_ASSERT_EQUAL(1350, (int)hitCounter.getFirstPassBytes()); - CPPUNIT_ASSERT_EQUAL(2, (int)hitCounter.getSecondPassHits()); - CPPUNIT_ASSERT_EQUAL(900, (int)hitCounter.getSecondPassBytes()); + EXPECT_EQ(3, hitCounter.getFirstPassHits()); + EXPECT_EQ(1350, hitCounter.getFirstPassBytes()); + EXPECT_EQ(2, hitCounter.getSecondPassHits()); + EXPECT_EQ(900, hitCounter.getSecondPassBytes()); } namespace { @@ -405,10 +361,8 @@ int getTotalSerializedSize(const std::vector<document::Document::SP>& docs) } -void -VisitorManagerTest::testNormalUsage() -{ - initializeTest(); +TEST_F(VisitorManagerTest, normal_usage) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); @@ -422,18 +376,17 @@ VisitorManagerTest::testNormalUsage() getMessagesAndReply(1, getSession(0), docs, docIds); // All data has been replied to, expecting to get a create visitor reply - verifyCreateVisitorReply(api::ReturnCode::OK, - int(docs.size()), - getTotalSerializedSize(docs)); + ASSERT_NO_FATAL_FAILURE( + verifyCreateVisitorReply(api::ReturnCode::OK, + int(docs.size()), + getTotalSerializedSize(docs))); - CPPUNIT_ASSERT_EQUAL(1u, getMatchingDocuments(docs)); - CPPUNIT_ASSERT(!_manager->hasPendingMessageState()); + EXPECT_EQ(1u, getMatchingDocuments(docs)); + EXPECT_FALSE(_manager->hasPendingMessageState()); } -void -VisitorManagerTest::testResending() -{ - initializeTest(); +TEST_F(VisitorManagerTest, resending) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); @@ -453,8 +406,8 @@ VisitorManagerTest::testResending() mbus::Reply::UP reply = msg->createReply(); - CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_VISITORINFO, - session.sentMessages[1]->getType()); + ASSERT_EQ(documentapi::DocumentProtocol::MESSAGE_VISITORINFO, + session.sentMessages[1]->getType()); reply->swapState(*session.sentMessages[1]); reply->setMessage(mbus::Message::UP(session.sentMessages[1].release())); session.reply(std::move(reply)); @@ -475,13 +428,11 @@ VisitorManagerTest::testResending() } // All data has been replied to, expecting to get a create visitor reply - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); } -void -VisitorManagerTest::testVisitEmptyBucket() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visit_empty_bucket) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); addSomeRemoves(true); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); @@ -491,13 +442,11 @@ VisitorManagerTest::testVisitEmptyBucket() _top->sendDown(cmd); // All data has been replied to, expecting to get a create visitor reply - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); } -void -VisitorManagerTest::testMultiBucketVisit() -{ - initializeTest(); +TEST_F(VisitorManagerTest, multi_bucket_visit) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); for (uint32_t i=0; i<10; ++i) { @@ -506,43 +455,40 @@ VisitorManagerTest::testMultiBucketVisit() cmd->setAddress(address); cmd->setDataDestination("fooclient.0"); _top->sendDown(cmd); - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; // Should receive one multioperation message for each bucket getMessagesAndReply(10, getSession(0), docs, docIds); // All data has been replied to, expecting to get a create visitor reply - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); - CPPUNIT_ASSERT_EQUAL(docCount, getMatchingDocuments(docs)); + EXPECT_EQ(docCount, getMatchingDocuments(docs)); } -void -VisitorManagerTest::testNoBuckets() -{ - initializeTest(); +TEST_F(VisitorManagerTest, no_buckets) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->setAddress(address); _top->sendDown(cmd); - // Should get one reply; a CreateVisitorReply with error since no - // buckets where specified in the CreateVisitorCommand + // Should get one reply; a CreateVisitorReply with error since no + // buckets where specified in the CreateVisitorCommand _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + ASSERT_EQ(1, replies.size()); auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(replies[0]); - // Verify that cast went ok => it was a CreateVisitorReply message - CPPUNIT_ASSERT(reply.get()); + // Verify that cast went ok => it was a CreateVisitorReply message + ASSERT_TRUE(reply.get()); api::ReturnCode ret(api::ReturnCode::ILLEGAL_PARAMETERS, "No buckets specified"); - CPPUNIT_ASSERT_EQUAL(ret, reply->getResult()); + EXPECT_EQ(ret, reply->getResult()); } -void VisitorManagerTest::testVisitPutsAndRemoves() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visit_puts_and_removes) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); addSomeRemoves(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); @@ -552,25 +498,22 @@ void VisitorManagerTest::testVisitPutsAndRemoves() cmd->addBucketToBeVisited(document::BucketId(16, i)); } _top->sendDown(cmd); - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; getMessagesAndReply(10, getSession(0), docs, docIds); - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); - CPPUNIT_ASSERT_EQUAL( - docCount - (docCount + 3) / 4, - getMatchingDocuments(docs)); + EXPECT_EQ(docCount - (docCount + 3) / 4, + getMatchingDocuments(docs)); - CPPUNIT_ASSERT_EQUAL( - (size_t) (docCount + 3) / 4, - docIds.size()); + EXPECT_EQ((docCount + 3) / 4, + docIds.size()); } -void VisitorManagerTest::testVisitWithTimeframeAndSelection() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visit_with_timeframe_and_selection) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval < 2"); cmd->setFromTime(3); @@ -580,14 +523,14 @@ void VisitorManagerTest::testVisitWithTimeframeAndSelection() } cmd->setAddress(address); _top->sendDown(cmd); - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; getMessagesAndReply(2, getSession(0), docs, docIds); - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); - CPPUNIT_ASSERT_EQUAL((size_t) 2, docs.size()); + ASSERT_EQ(2, docs.size()); std::set<std::string> expected; expected.insert("userdoc:test:4:http://www.ntnu.no/4.html"); expected.insert("userdoc:test:5:http://www.ntnu.no/5.html"); @@ -595,12 +538,11 @@ void VisitorManagerTest::testVisitWithTimeframeAndSelection() for (uint32_t i=0; i<docs.size(); ++i) { actual.insert(docs[i]->getId().toString()); } - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_THAT(expected, ContainerEq(actual)); } -void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visit_with_timeframe_and_bogus_selection) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", "DocType(testdoctype1---///---) XXX BAD Field(headerval) < 2"); @@ -614,17 +556,33 @@ void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + ASSERT_EQ(1, replies.size()); auto* reply = dynamic_cast<api::StorageReply*>(replies.front().get()); - CPPUNIT_ASSERT(reply); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ILLEGAL_PARAMETERS, reply->getResult().getResult()); + ASSERT_TRUE(reply); + EXPECT_EQ(api::ReturnCode::ILLEGAL_PARAMETERS, reply->getResult().getResult()); } -void -VisitorManagerTest::testVisitorCallbacks() -{ - initializeTest(); +#define ASSERT_SUBSTRING_COUNT(source, expectedCount, substring) \ + { \ + uint32_t count = 0; \ + std::ostringstream value; /* Let value be non-strings */ \ + value << source; \ + std::string s(value.str()); \ + std::string::size_type pos = s.find(substring); \ + while (pos != std::string::npos) { \ + ++count; \ + pos = s.find(substring, pos+1); \ + } \ + if (count != (uint32_t) expectedCount) { \ + FAIL() << "Value of '" << s << "' contained " << count \ + << " instances of substring '" << substring << "', not " \ + << expectedCount << " as expected."; \ + } \ + } + +TEST_F(VisitorManagerTest, visitor_callbacks) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); std::ostringstream replydata; api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "TestVisitor", "testvis", ""); @@ -642,10 +600,10 @@ VisitorManagerTest::testVisitorCallbacks() { vespalib::MonitorGuard guard(session.getMonitor()); - CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_MAPVISITOR, session.sentMessages[i]->getType()); + ASSERT_EQ(documentapi::DocumentProtocol::MESSAGE_MAPVISITOR, session.sentMessages[i]->getType()); auto* mapvisitormsg = dynamic_cast<documentapi::MapVisitorMessage*>(session.sentMessages[i].get()); - CPPUNIT_ASSERT(mapvisitormsg != nullptr); + ASSERT_TRUE(mapvisitormsg != nullptr); replydata << mapvisitormsg->getData().get("msg"); @@ -657,18 +615,16 @@ VisitorManagerTest::testVisitorCallbacks() } // All data has been replied to, expecting to get a create visitor reply - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); - CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "Starting visitor"); - CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "Handling block of 1 documents"); - CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "completedBucket"); - CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "completedVisiting"); + ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "Starting visitor"); + ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "Handling block of 1 documents"); + ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "completedBucket"); + ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "completedVisiting"); } -void -VisitorManagerTest::testVisitorCleanup() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visitor_cleanup) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); // Start a bunch of invalid visitors @@ -699,16 +655,16 @@ VisitorManagerTest::testVisitorCleanup() const int expected_total = 16; _top->waitForMessages(expected_total, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(size_t(expected_total), replies.size()); + ASSERT_EQ(expected_total, replies.size()); int failures = 0; int busy = 0; for (uint32_t i=0; i< expected_total; ++i) { std::shared_ptr<api::StorageMessage> msg(replies[i]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); if (i < 10) { if (api::ReturnCode::ILLEGAL_PARAMETERS == reply->getResult().getResult()) { @@ -723,20 +679,20 @@ VisitorManagerTest::testVisitorCleanup() } } - CPPUNIT_ASSERT_EQUAL(10, failures); - CPPUNIT_ASSERT_EQUAL(expected_total - 10, busy); + ASSERT_EQ(10, failures); + ASSERT_EQ(expected_total - 10, busy); } // 4 pending // Finish a visitor - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; getMessagesAndReply(1, getSession(0), docs, docIds); // Should get a reply for the visitor. - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); // 3 pending @@ -744,7 +700,7 @@ VisitorManagerTest::testVisitorCleanup() getMessagesAndReply(1, getSession(1), docs, docIds, api::ReturnCode::INTERNAL_FAILURE); // Should get a reply for the visitor. - verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE)); // Wait until there are 2 pending. Visitor threads might not have completed // cleanup of existing visitors yet. @@ -766,15 +722,15 @@ VisitorManagerTest::testVisitorCleanup() // Should now get 8 busy. _top->waitForMessages(8, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(size_t(8), replies.size()); + ASSERT_EQ(8, replies.size()); for (uint32_t i=0; i< replies.size(); ++i) { std::shared_ptr<api::StorageMessage> msg(replies[i]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); - CPPUNIT_ASSERT(reply.get()); + ASSERT_TRUE(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::BUSY, reply->getResult().getResult()); + ASSERT_EQ(api::ReturnCode::BUSY, reply->getResult().getResult()); } for (uint32_t i = 0; i < 4; ++i) { @@ -783,10 +739,8 @@ VisitorManagerTest::testVisitorCleanup() } } -void -VisitorManagerTest::testAbortOnFailedVisitorInfo() -{ - initializeTest(); +TEST_F(VisitorManagerTest, abort_on_failed_visitor_info) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); { @@ -797,7 +751,7 @@ VisitorManagerTest::testAbortOnFailedVisitorInfo() _top->sendDown(cmd); } - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; TestVisitorMessageSession& session = getSession(0); @@ -810,18 +764,16 @@ VisitorManagerTest::testAbortOnFailedVisitorInfo() mbus::Reply::UP reply = cmd->createReply(); - CPPUNIT_ASSERT_EQUAL(uint32_t(documentapi::DocumentProtocol::MESSAGE_VISITORINFO), session.sentMessages[1]->getType()); + ASSERT_EQ(documentapi::DocumentProtocol::MESSAGE_VISITORINFO, session.sentMessages[1]->getType()); reply->swapState(*session.sentMessages[1]); reply->setMessage(mbus::Message::UP(session.sentMessages[1].release())); reply->addError(mbus::Error(api::ReturnCode::NOT_CONNECTED, "Me no ready")); session.reply(std::move(reply)); } - verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED)); } -void -VisitorManagerTest::testAbortOnFieldPathError() -{ +TEST_F(VisitorManagerTest, abort_on_field_path_error) { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); @@ -833,13 +785,11 @@ VisitorManagerTest::testAbortOnFieldPathError() cmd->setQueueTimeout(0); _top->sendDown(cmd); - verifyCreateVisitorReply(api::ReturnCode::ILLEGAL_PARAMETERS); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::ILLEGAL_PARAMETERS)); } -void -VisitorManagerTest::testVisitorQueueTimeout() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visitor_queue_timeout) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); _manager->enforceQueueUsage(); @@ -861,16 +811,14 @@ VisitorManagerTest::testVisitorQueueTimeout() const msg_ptr_vector replies = _top->getRepliesOnce(); std::shared_ptr<api::StorageMessage> msg(replies[0]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::BUSY, "Visitor timed out in visitor queue"), - reply->getResult()); + ASSERT_EQ(api::ReturnCode(api::ReturnCode::BUSY, "Visitor timed out in visitor queue"), + reply->getResult()); } -void -VisitorManagerTest::testVisitorProcessingTimeout() -{ - initializeTest(); +TEST_F(VisitorManagerTest, visitor_processing_timeout) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); @@ -886,32 +834,32 @@ VisitorManagerTest::testVisitorProcessingTimeout() _node->getClock().addSecondsToTime(1000); - verifyCreateVisitorReply(api::ReturnCode::ABORTED); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::ABORTED)); } namespace { - uint32_t nextVisitor = 0; - api::StorageMessage::Id - sendCreateVisitor(uint32_t timeout, DummyStorageLink& top, uint8_t priority = 127) { - std::ostringstream ost; - ost << "testvis" << ++nextVisitor; - api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); - cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(address); - cmd->setQueueTimeout(timeout); - cmd->setPriority(priority); - top.sendDown(cmd); - return cmd->getMsgId(); - } +uint32_t nextVisitor = 0; + +api::StorageMessage::Id +sendCreateVisitor(uint32_t timeout, DummyStorageLink& top, uint8_t priority = 127) { + std::ostringstream ost; + ost << "testvis" << ++nextVisitor; + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(timeout); + cmd->setPriority(priority); + top.sendDown(cmd); + return cmd->getMsgId(); } -void -VisitorManagerTest::testPrioritizedVisitorQueing() -{ +} + +TEST_F(VisitorManagerTest, prioritized_visitor_queing) { framework::HttpUrlPath path("?verbose=true&allvisitors=true"); - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); _manager->setMaxConcurrentVisitors(4); _manager->setMaxVisitorQueueSize(4); @@ -933,27 +881,31 @@ VisitorManagerTest::testPrioritizedVisitorQueing() // Send a lower pri visitor that will be busy-returned immediately ids[8] = sendCreateVisitor(1000, *_top, 130); - CPPUNIT_ASSERT_EQUAL(ids[8], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + uint64_t message_id = 0; + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY, -1, -1, &message_id)); + ASSERT_EQ(ids[8], message_id); // Send a higher pri visitor that will take the place of pri 100 visitor ids[9] = sendCreateVisitor(1000, *_top, 60); - CPPUNIT_ASSERT_EQUAL(ids[4], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY, -1, -1, &message_id)); + ASSERT_EQ(ids[4], message_id); // Finish the first visitor - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; getMessagesAndReply(1, getSession(0), docs, docIds, api::ReturnCode::OK, Priority::PRI_HIGHEST); - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK, -1, -1, &message_id)); // We should now start the highest priority visitor. getMessagesAndReply(1, getSession(4), docs, docIds, api::ReturnCode::OK, Priority::PRI_VERY_HIGH); - CPPUNIT_ASSERT_EQUAL(ids[9], verifyCreateVisitorReply(api::ReturnCode::OK)); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK, -1, -1, &message_id)); + ASSERT_EQ(ids[9], message_id); // 3 pending, 3 in queue. Clean them up std::vector<uint32_t> pending_sessions = {1, 2, 3, 5, 6, 7}; for (auto session : pending_sessions) { - finishAndWaitForVisitorSessionCompletion(session); + ASSERT_NO_FATAL_FAILURE(finishAndWaitForVisitorSessionCompletion(session)); } } @@ -961,13 +913,12 @@ void VisitorManagerTest::finishAndWaitForVisitorSessionCompletion(uint32_t sessi std::vector<document::Document::SP > docs; std::vector<document::DocumentId> docIds; getMessagesAndReply(1, getSession(sessionIndex), docs, docIds, api::ReturnCode::OK, std::optional<Priority::Value>()); - verifyCreateVisitorReply(api::ReturnCode::OK); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); } -void -VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { +TEST_F(VisitorManagerTest, prioritized_max_concurrent_visitors) { framework::HttpUrlPath path("?verbose=true&allvisitors=true"); - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); api::StorageMessage::Id ids[17] = { 0 }; @@ -997,12 +948,15 @@ VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { // Should punch pri203 msg out of the queue -> busy ids[11] = sendCreateVisitor(1000, *_top, 197); - CPPUNIT_ASSERT_EQUAL(ids[4], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + uint64_t message_id = 0; + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY, -1, -1, &message_id)); + ASSERT_EQ(ids[4], message_id); // No concurrency slots left for this message -> busy ids[12] = sendCreateVisitor(1000, *_top, 204); - CPPUNIT_ASSERT_EQUAL(ids[12], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY, -1, -1, &message_id)); + ASSERT_EQ(ids[12], message_id); // Gets a concurrent slot ids[13] = sendCreateVisitor(1000, *_top, 80); @@ -1010,7 +964,8 @@ VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { // Kicks pri 202 out of the queue -> busy ids[14] = sendCreateVisitor(1000, *_top, 79); - CPPUNIT_ASSERT_EQUAL(ids[5], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY, -1, -1, &message_id)); + ASSERT_EQ(ids[5], message_id); // Gets a concurrent slot ids[15] = sendCreateVisitor(1000, *_top, 63); @@ -1018,7 +973,7 @@ VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { // Very Important Visitor(tm) gets a concurrent slot ids[16] = sendCreateVisitor(1000, *_top, 0); - std::vector<document::Document::SP > docs; + std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; std::set<uint64_t> finishedVisitors; @@ -1034,19 +989,19 @@ VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { } else if (i == 6) { priority = documentapi::Priority::PRI_HIGH_1; // ids 15 } - getMessagesAndReply(1, getSession(i), docs, docIds, api::ReturnCode::OK, - priority); - finishedVisitors.insert(verifyCreateVisitorReply(api::ReturnCode::OK)); + getMessagesAndReply(1, getSession(i), docs, docIds, api::ReturnCode::OK, priority); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK, -1, -1, &message_id)); + finishedVisitors.insert(message_id); } for (int i = 0; i < 4; i++) { - CPPUNIT_ASSERT(finishedVisitors.find(ids[i]) != finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[i]), finishedVisitors.end()); } - CPPUNIT_ASSERT(finishedVisitors.find(ids[10]) != finishedVisitors.end()); - CPPUNIT_ASSERT(finishedVisitors.find(ids[13]) != finishedVisitors.end()); - CPPUNIT_ASSERT(finishedVisitors.find(ids[15]) != finishedVisitors.end()); - CPPUNIT_ASSERT(finishedVisitors.find(ids[16]) != finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[10]), finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[13]), finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[15]), finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[16]), finishedVisitors.end()); finishedVisitors.clear(); @@ -1058,22 +1013,22 @@ VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { } getMessagesAndReply(1, getSession(i), docs, docIds, api::ReturnCode::OK, priority); - uint64_t msgId = verifyCreateVisitorReply(api::ReturnCode::OK); + uint64_t msgId = 0; + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK, -1, -1, &msgId)); finishedVisitors.insert(msgId); } for (int i = 6; i < 10; i++) { - CPPUNIT_ASSERT(finishedVisitors.find(ids[i]) != finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[i]), finishedVisitors.end()); } - CPPUNIT_ASSERT(finishedVisitors.find(ids[11]) != finishedVisitors.end()); - CPPUNIT_ASSERT(finishedVisitors.find(ids[14]) != finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[11]), finishedVisitors.end()); + ASSERT_NE(finishedVisitors.find(ids[14]), finishedVisitors.end()); } -void -VisitorManagerTest::testVisitorQueingZeroQueueSize() { +TEST_F(VisitorManagerTest, visitor_queing_zero_queue_size) { framework::HttpUrlPath path("?verbose=true&allvisitors=true"); - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); _manager->setMaxConcurrentVisitors(4); _manager->setMaxVisitorQueueSize(0); @@ -1085,17 +1040,16 @@ VisitorManagerTest::testVisitorQueingZeroQueueSize() { // Queue size is zero, all visitors will be busy-returned for (uint32_t i = 0; i < 5; ++i) { sendCreateVisitor(1000, *_top, 100 - i); - verifyCreateVisitorReply(api::ReturnCode::BUSY); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUSY)); } for (uint32_t session = 0; session < 4; ++session) { finishAndWaitForVisitorSessionCompletion(session); } } -void -VisitorManagerTest::testStatusPage() { +TEST_F(VisitorManagerTest, status_page) { framework::HttpUrlPath path("?verbose=true&allvisitors=true"); - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); _manager->setMaxConcurrentVisitors(1, 1); _manager->setMaxVisitorQueueSize(6); @@ -1112,13 +1066,13 @@ VisitorManagerTest::testStatusPage() { static_cast<framework::HtmlStatusReporter&>(*_manager).reportHtmlStatus(ss, path); std::string str(ss.str()); - CPPUNIT_ASSERT(str.find("Currently running visitors") != std::string::npos); + EXPECT_THAT(str, HasSubstr("Currently running visitors")); // Should be propagated to visitor thread - CPPUNIT_ASSERT(str.find("Running 1 visitors") != std::string::npos); // 1 active - CPPUNIT_ASSERT(str.find("waiting visitors 1") != std::string::npos); // 1 queued - CPPUNIT_ASSERT(str.find("Visitor thread 0") != std::string::npos); - CPPUNIT_ASSERT(str.find("Disconnected visitor timeout") != std::string::npos); // verbose per thread - CPPUNIT_ASSERT(str.find("Message #1 <b>putdocumentmessage</b>") != std::string::npos); // 1 active + EXPECT_THAT(str, HasSubstr("Running 1 visitors")); // 1 active + EXPECT_THAT(str, HasSubstr("waiting visitors 1")); // 1 queued + EXPECT_THAT(str, HasSubstr("Visitor thread 0")); + EXPECT_THAT(str, HasSubstr("Disconnected visitor timeout")); // verbose per thread + EXPECT_THAT(str, HasSubstr("Message #1 <b>putdocumentmessage</b>")); // 1 active for (uint32_t session = 0; session < 2 ; ++session){ finishAndWaitForVisitorSessionCompletion(session); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 24b04c8d33a..3d37bbe434b 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -18,11 +18,13 @@ #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/gtest/gtest.h> #include <thread> #include <sys/stat.h> using namespace std::chrono_literals; using document::test::makeBucketSpace; +using namespace ::testing; namespace storage { @@ -30,8 +32,7 @@ namespace { using msg_ptr_vector = std::vector<api::StorageMessage::SP>; -struct TestParams -{ +struct TestParams { TestParams& maxVisitorMemoryUsage(uint32_t bytes) { _maxVisitorMemoryUsage = bytes; return *this; @@ -52,26 +53,7 @@ struct TestParams } -class VisitorTest : public CppUnit::TestFixture -{ -private: - CPPUNIT_TEST_SUITE(VisitorTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testFailedCreateIterator); - CPPUNIT_TEST(testFailedGetIter); - CPPUNIT_TEST(testDocumentAPIClientError); - CPPUNIT_TEST(testNoDocumentAPIResendingForFailedVisitor); - CPPUNIT_TEST(testIteratorCreatedForFailedVisitor); - CPPUNIT_TEST(testFailedDocumentAPISend); - CPPUNIT_TEST(testNoVisitorNotificationForTransientFailures); - CPPUNIT_TEST(testNotificationSentIfTransientErrorRetriedManyTimes); - CPPUNIT_TEST(testNoMbusTracingIfTraceLevelIsZero); - CPPUNIT_TEST(testReplyContainsTraceIfTraceLevelAboveZero); - CPPUNIT_TEST(testNoMoreIteratorsSentWhileMemoryUsedAboveLimit); - CPPUNIT_TEST(testDumpVisitorInvokesStrongReadConsistencyIteration); - CPPUNIT_TEST(testTestVisitorInvokesWeakReadConsistencyIteration); - CPPUNIT_TEST_SUITE_END(); - +struct VisitorTest : Test { static uint32_t docCount; std::vector<document::Document::SP > _documents; std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; @@ -80,26 +62,8 @@ private: DummyStorageLink* _bottom; VisitorManager* _manager; -public: VisitorTest() : _node() {} - void testNormalUsage(); - void testFailedCreateIterator(); - void testFailedGetIter(); - void testDocumentAPIClientError(); - void testNoDocumentAPIResendingForFailedVisitor(); - void testIteratorCreatedForFailedVisitor(); - void testFailedDocumentAPISend(); - void testNoVisitorNotificationForTransientFailures(); - void testNotificationSentIfTransientErrorRetriedManyTimes(); - void testNoMbusTracingIfTraceLevelIsZero(); - void testReplyContainsTraceIfTraceLevelAboveZero(); - void testNoMoreIteratorsSentWhileMemoryUsedAboveLimit(); - void testDumpVisitorInvokesStrongReadConsistencyIteration(); - void testTestVisitorInvokesWeakReadConsistencyIteration(); - // TODO: - void testVisitMultipleBuckets() {} - // Not using setUp since can't throw exception out of it. void initializeTest(const TestParams& params = TestParams()); @@ -118,13 +82,14 @@ public: std::shared_ptr<api::CreateVisitorCommand> makeCreateVisitor( const VisitorOptions& options = VisitorOptions()); - void tearDown() override; + void TearDown() override; bool waitUntilNoActiveVisitors(); TestVisitorMessageSession& getSession(uint32_t n); - uint64_t verifyCreateVisitorReply( + void verifyCreateVisitorReply( api::ReturnCode::Result expectedResult, int checkStatsDocsVisited = -1, - int checkStatsBytesVisited = -1); + int checkStatsBytesVisited = -1, + uint64_t* message_id_out = nullptr); void getMessagesAndReply( int expectedCount, TestVisitorMessageSession& session, @@ -134,18 +99,17 @@ public: api::ReturnCode::Result returnCode = api::ReturnCode::OK); uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); -private: +protected: void doTestVisitorInstanceHasConsistencyLevel( vespalib::stringref visitorType, spi::ReadConsistency expectedConsistency); template <typename T> - std::vector<std::shared_ptr<T> > - fetchMultipleCommands(DummyStorageLink& link, size_t count); + void fetchMultipleCommands(DummyStorageLink& link, size_t count, + std::vector<std::shared_ptr<T>>& commands_out); template <typename T> - std::shared_ptr<T> - fetchSingleCommand(DummyStorageLink& link); + void fetchSingleCommand(DummyStorageLink& link, std::shared_ptr<T>& msg_out); void sendGetIterReply(GetIterCommand& cmd, const api::ReturnCode& result = @@ -153,8 +117,9 @@ private: uint32_t maxDocuments = 0, bool overrideCompleted = false); void sendCreateIteratorReply(uint64_t iteratorId = 1234); - std::shared_ptr<api::CreateVisitorReply> doCompleteVisitingSession( - const std::shared_ptr<api::CreateVisitorCommand>& cmd); + void doCompleteVisitingSession( + const std::shared_ptr<api::CreateVisitorCommand>& cmd, + std::shared_ptr<api::CreateVisitorReply>& reply_out); void sendInitialCreateVisitorAndGetIterRound(); @@ -171,8 +136,6 @@ private: uint32_t VisitorTest::docCount = 10; -CPPUNIT_TEST_SUITE_REGISTRATION(VisitorTest); - void VisitorTest::initializeTest(const TestParams& params) { @@ -192,26 +155,23 @@ VisitorTest::initializeTest(const TestParams& params) vespalib::mkdir(vespalib::make_string("%s/disks/d0", rootFolder.c_str()), true); vespalib::mkdir(vespalib::make_string("%s/disks/d1", rootFolder.c_str()), true); - try { - _messageSessionFactory.reset( - new TestVisitorMessageSessionFactory(config.getConfigId())); - if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { - _messageSessionFactory->_autoReplyError = params._autoReplyError; - _messageSessionFactory->_createAutoReplyVisitorSessions = true; - } - _node.reset(new TestServiceLayerApp(config.getConfigId())); - _top.reset(new DummyStorageLink()); - _top->push_back(std::unique_ptr<StorageLink>(_manager - = new VisitorManager( - config.getConfigId(), - _node->getComponentRegister(), *_messageSessionFactory))); - _bottom = new DummyStorageLink(); - _top->push_back(std::unique_ptr<StorageLink>(_bottom)); - _manager->setTimeBetweenTicks(10); - _top->open(); - } catch (config::InvalidConfigException& e) { - fprintf(stderr, "%s\n", e.what()); + _messageSessionFactory.reset( + new TestVisitorMessageSessionFactory(config.getConfigId())); + if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { + _messageSessionFactory->_autoReplyError = params._autoReplyError; + _messageSessionFactory->_createAutoReplyVisitorSessions = true; } + _node.reset(new TestServiceLayerApp(config.getConfigId())); + _top.reset(new DummyStorageLink()); + _top->push_back(std::unique_ptr<StorageLink>(_manager + = new VisitorManager( + config.getConfigId(), + _node->getComponentRegister(), *_messageSessionFactory))); + _bottom = new DummyStorageLink(); + _top->push_back(std::unique_ptr<StorageLink>(_bottom)); + _manager->setTimeBetweenTicks(10); + _top->open(); + std::string content( "To be, or not to be: that is the question:\n" "Whether 'tis nobler in the mind to suffer\n" @@ -263,16 +223,16 @@ VisitorTest::initializeTest(const TestParams& params) } void -VisitorTest::tearDown() +VisitorTest::TearDown() { - if (_top.get() != 0) { + if (_top) { _top->close(); _top->flush(); - _top.reset(0); + _top.reset(); } - _node.reset(0); - _messageSessionFactory.reset(0); - _manager = 0; + _node.reset(); + _messageSessionFactory.reset(); + _manager = nullptr; } bool @@ -310,7 +270,7 @@ VisitorTest::getSession(uint32_t n) } std::this_thread::sleep_for(10ms); } - throw std::logic_error("unreachable"); + abort(); } void @@ -327,10 +287,10 @@ VisitorTest::getMessagesAndReply( mbus::Reply::UP reply; { vespalib::MonitorGuard guard(session.getMonitor()); - CPPUNIT_ASSERT(!session.sentMessages.empty()); + ASSERT_FALSE(session.sentMessages.empty()); std::unique_ptr<documentapi::DocumentMessage> msg(std::move(session.sentMessages.front())); session.sentMessages.pop_front(); - CPPUNIT_ASSERT(msg->getPriority() < 16); + ASSERT_LT(msg->getPriority(), 16); switch (msg->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: @@ -361,35 +321,37 @@ VisitorTest::getMessagesAndReply( } } -uint64_t +void VisitorTest::verifyCreateVisitorReply( api::ReturnCode::Result expectedResult, int checkStatsDocsVisited, - int checkStatsBytesVisited) + int checkStatsBytesVisited, + uint64_t* message_id_out) { _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(1, (int)replies.size()); + ASSERT_EQ(1, replies.size()); std::shared_ptr<api::StorageMessage> msg(replies[0]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(expectedResult, reply->getResult().getResult()); + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); + ASSERT_TRUE(reply.get()); + ASSERT_EQ(expectedResult, reply->getResult().getResult()); if (checkStatsDocsVisited >= 0) { - CPPUNIT_ASSERT_EQUAL(checkStatsDocsVisited, - int(reply->getVisitorStatistics().getDocumentsVisited())); + ASSERT_EQ(checkStatsDocsVisited, + reply->getVisitorStatistics().getDocumentsVisited()); } if (checkStatsBytesVisited >= 0) { - CPPUNIT_ASSERT_EQUAL(checkStatsBytesVisited, - int(reply->getVisitorStatistics().getBytesVisited())); + ASSERT_EQ(checkStatsBytesVisited, + reply->getVisitorStatistics().getBytesVisited()); } - return reply->getMsgId(); + if (message_id_out) { + *message_id_out = reply->getMsgId(); + } } uint32_t @@ -423,12 +385,11 @@ VisitorTest::sendGetIterReply(GetIterCommand& cmd, assert(maxDocuments < _documents.size()); size_t documentCount = maxDocuments != 0 ? maxDocuments : _documents.size(); for (size_t i = 0; i < documentCount; ++i) { - reply->getEntries().push_back( - spi::DocEntry::UP( - new spi::DocEntry( + reply->getEntries().emplace_back( + std::make_unique<spi::DocEntry>( spi::Timestamp(1000 + i), spi::NONE, - document::Document::UP(_documents[i]->clone())))); + document::Document::UP(_documents[i]->clone()))); } if (documentCount == _documents.size() || overrideCompleted) { reply->setCompleted(); @@ -437,12 +398,13 @@ VisitorTest::sendGetIterReply(GetIterCommand& cmd, } template <typename T> -std::vector<std::shared_ptr<T> > -VisitorTest::fetchMultipleCommands(DummyStorageLink& link, size_t count) +void +VisitorTest::fetchMultipleCommands(DummyStorageLink& link, size_t count, + std::vector<std::shared_ptr<T>>& commands_out) { link.waitForMessages(count, 60); std::vector<api::StorageMessage::SP> msgs(link.getCommandsOnce()); - std::vector<std::shared_ptr<T> > fetched; + std::vector<std::shared_ptr<T>> fetched; if (msgs.size() != count) { std::ostringstream oss; oss << "Expected " @@ -453,38 +415,38 @@ VisitorTest::fetchMultipleCommands(DummyStorageLink& link, size_t count) for (size_t i = 0; i < msgs.size(); ++i) { oss << i << ": " << *msgs[i] << "\n"; } - CPPUNIT_FAIL(oss.str()); + FAIL() << oss.str(); } for (size_t i = 0; i < count; ++i) { - std::shared_ptr<T> ret(std::dynamic_pointer_cast<T>(msgs[i])); + auto ret = std::dynamic_pointer_cast<T>(msgs[i]); if (!ret) { std::ostringstream oss; oss << "Expected message of type " << typeid(T).name() << ", but got " << msgs[0]->toString(); - CPPUNIT_FAIL(oss.str()); + FAIL() << oss.str(); } fetched.push_back(ret); } - return fetched; + commands_out = std::move(fetched); } template <typename T> -std::shared_ptr<T> -VisitorTest::fetchSingleCommand(DummyStorageLink& link) +void +VisitorTest::fetchSingleCommand(DummyStorageLink& link, std::shared_ptr<T>& msg_out) { - std::vector<std::shared_ptr<T> > ret( - fetchMultipleCommands<T>(link, 1)); - return ret[0]; + std::vector<std::shared_ptr<T>> ret; + ASSERT_NO_FATAL_FAILURE(fetchMultipleCommands<T>(link, 1, ret)); + msg_out = std::move(ret[0]); } std::shared_ptr<api::CreateVisitorCommand> VisitorTest::makeCreateVisitor(const VisitorOptions& options) { api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), options.visitorType, "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>( + makeBucketSpace(), options.visitorType, "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setMaximumPendingReplyCount(UINT32_MAX); @@ -496,35 +458,29 @@ VisitorTest::makeCreateVisitor(const VisitorOptions& options) void VisitorTest::sendCreateIteratorReply(uint64_t iteratorId) { - CreateIteratorCommand::SP createCmd( - fetchSingleCommand<CreateIteratorCommand>(*_bottom)); + CreateIteratorCommand::SP createCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<CreateIteratorCommand>(*_bottom, createCmd)); spi::IteratorId id(iteratorId); - api::StorageReply::SP reply( - new CreateIteratorReply(*createCmd, id)); + auto reply = std::make_shared<CreateIteratorReply>(*createCmd, id); _bottom->sendUp(reply); } -void -VisitorTest::testNormalUsage() -{ - initializeTest(); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); +TEST_F(VisitorTest, normal_usage) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); + auto cmd = makeCreateVisitor(); _top->sendDown(cmd); - CreateIteratorCommand::SP createCmd( - fetchSingleCommand<CreateIteratorCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(static_cast<int>(DefaultPriority), - static_cast<int>(createCmd->getPriority())); // Inherit pri + CreateIteratorCommand::SP createCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<CreateIteratorCommand>(*_bottom, createCmd)); + ASSERT_EQ(static_cast<int>(DefaultPriority), + static_cast<int>(createCmd->getPriority())); // Inherit pri spi::IteratorId id(1234); - api::StorageReply::SP reply( - new CreateIteratorReply(*createCmd, id)); + auto reply = std::make_shared<CreateIteratorReply>(*createCmd, id); _bottom->sendUp(reply); - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd); @@ -532,76 +488,64 @@ VisitorTest::testNormalUsage() std::vector<document::DocumentId> docIds; std::vector<std::string> infoMessages; getMessagesAndReply(_documents.size(), getSession(0), docs, docIds, infoMessages); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); - CPPUNIT_ASSERT_EQUAL(size_t(0), docIds.size()); + ASSERT_EQ(0, infoMessages.size()); + ASSERT_EQ(0, docIds.size()); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply(api::ReturnCode::OK); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); - CPPUNIT_ASSERT_EQUAL(INT64_C(0), getFailedVisitorDestinationReplyCount()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); + ASSERT_EQ(0, getFailedVisitorDestinationReplyCount()); } -void -VisitorTest::testFailedCreateIterator() -{ - initializeTest(); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); +TEST_F(VisitorTest, failed_create_iterator) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); + auto cmd = makeCreateVisitor(); cmd->addBucketToBeVisited(document::BucketId(16, 4)); _top->sendDown(cmd); - CreateIteratorCommand::SP createCmd( - fetchSingleCommand<CreateIteratorCommand>(*_bottom)); + CreateIteratorCommand::SP createCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<CreateIteratorCommand>(*_bottom, createCmd)); spi::IteratorId id(0); - api::StorageReply::SP reply( - new CreateIteratorReply(*createCmd, id)); + auto reply = std::make_shared<CreateIteratorReply>(*createCmd, id); reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE)); _bottom->sendUp(reply); - verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } -void -VisitorTest::testFailedGetIter() -{ - initializeTest(); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); +TEST_F(VisitorTest, failed_get_iter) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); + auto cmd = makeCreateVisitor(); _top->sendDown(cmd); sendCreateIteratorReply(); - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND)); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply(api::ReturnCode::BUCKET_NOT_FOUND, 0, 0); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::BUCKET_NOT_FOUND, 0, 0)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } -void -VisitorTest::testDocumentAPIClientError() -{ +TEST_F(VisitorTest, document_api_client_error) { initializeTest(); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); + auto cmd = makeCreateVisitor(); _top->sendDown(cmd); sendCreateIteratorReply(); { - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1); } @@ -612,40 +556,36 @@ VisitorTest::testDocumentAPIClientError() getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::INTERNAL_FAILURE); // INTERNAL_FAILURE is critical, so no visitor info sent - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); std::this_thread::sleep_for(100ms); { - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd); } - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } -void -VisitorTest::testNoDocumentAPIResendingForFailedVisitor() -{ - initializeTest(); +TEST_F(VisitorTest, no_document_api_resending_for_failed_visitor) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); std::shared_ptr<api::CreateVisitorCommand> cmd( makeCreateVisitor()); _top->sendDown(cmd); sendCreateIteratorReply(); { - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 2, true); } @@ -658,50 +598,44 @@ VisitorTest::testNoDocumentAPIResendingForFailedVisitor() // should cause the entire visitor to fail. getMessagesAndReply(3, getSession(0), docs, docIds, infoMessages, api::ReturnCode::NOT_CONNECTED); - CPPUNIT_ASSERT_EQUAL(size_t(1), infoMessages.size()); - CPPUNIT_ASSERT_EQUAL( - std::string("[From content node 0] NOT_CONNECTED: Generic error"), - infoMessages[0]); + ASSERT_EQ(1, infoMessages.size()); + EXPECT_EQ("[From content node 0] NOT_CONNECTED: Generic error", + infoMessages[0]); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); - CPPUNIT_ASSERT_EQUAL(INT64_C(3), getFailedVisitorDestinationReplyCount()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); + ASSERT_EQ(3, getFailedVisitorDestinationReplyCount()); } -void -VisitorTest::testIteratorCreatedForFailedVisitor() -{ +TEST_F(VisitorTest, iterator_created_for_failed_visitor) { initializeTest(TestParams().parallelBuckets(2)); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); + auto cmd = makeCreateVisitor(); cmd->addBucketToBeVisited(document::BucketId(16, 4)); _top->sendDown(cmd); - std::vector<CreateIteratorCommand::SP> createCmds( - fetchMultipleCommands<CreateIteratorCommand>(*_bottom, 2)); + std::vector<CreateIteratorCommand::SP> createCmds; + ASSERT_NO_FATAL_FAILURE(fetchMultipleCommands<CreateIteratorCommand>(*_bottom, 2, createCmds)); { spi::IteratorId id(0); - api::StorageReply::SP reply( - new CreateIteratorReply(*createCmds[0], id)); + auto reply = std::make_shared<CreateIteratorReply>(*createCmds[0], id); reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE)); _bottom->sendUp(reply); } { spi::IteratorId id(1234); - api::StorageReply::SP reply( - new CreateIteratorReply(*createCmds[1], id)); + auto reply = std::make_shared<CreateIteratorReply>(*createCmds[1], id); _bottom->sendUp(reply); } // Want to immediately receive destroyiterator for newly created // iterator, since we cannot use it anyway when the visitor has failed. - DestroyIteratorCommand::SP destroyCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyCmd)); - verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } /** @@ -710,62 +644,55 @@ VisitorTest::testIteratorCreatedForFailedVisitor() * and the visitor terminates cleanly without counting the failed message * as pending. */ -void -VisitorTest::testFailedDocumentAPISend() -{ - initializeTest(TestParams().autoReplyError( +TEST_F(VisitorTest, failed_document_api_send) { + ASSERT_NO_FATAL_FAILURE(initializeTest(TestParams().autoReplyError( mbus::Error(mbus::ErrorCode::HANDSHAKE_FAILED, - "abandon ship!"))); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); + "abandon ship!")))); + auto cmd = makeCreateVisitor(); cmd->addBucketToBeVisited(document::BucketId(16, 4)); _top->sendDown(cmd); sendCreateIteratorReply(); - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); + ASSERT_EQ(spi::IteratorId(1234), getIterCmd->getIteratorId()); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 2, true); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply( + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply( static_cast<api::ReturnCode::Result>( mbus::ErrorCode::HANDSHAKE_FAILED), 0, - 0); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + 0)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); // We currently don't count failures to send in this metric; send failures // indicate a message bus problem and already log a warning when they happen - CPPUNIT_ASSERT_EQUAL(INT64_C(0), getFailedVisitorDestinationReplyCount()); + ASSERT_EQ(0, getFailedVisitorDestinationReplyCount()); } void VisitorTest::sendInitialCreateVisitorAndGetIterRound() { - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); + auto cmd = makeCreateVisitor(); _top->sendDown(cmd); sendCreateIteratorReply(); { - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1, true); } } -void -VisitorTest::testNoVisitorNotificationForTransientFailures() -{ - initializeTest(); - sendInitialCreateVisitorAndGetIterRound(); +TEST_F(VisitorTest, no_visitor_notification_for_transient_failures) { + ASSERT_NO_FATAL_FAILURE(initializeTest()); + ASSERT_NO_FATAL_FAILURE(sendInitialCreateVisitorAndGetIterRound()); std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; @@ -776,41 +703,40 @@ VisitorTest::testNoVisitorNotificationForTransientFailures() // Should not get info message for BUCKET_DELETED, but resend of Put. getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::BUCKET_DELETED); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); // Should not get info message for BUCKET_NOT_FOUND, but resend of Put. getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::BUCKET_NOT_FOUND); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); // MessageBus error codes guaranteed to fit in return code result. // Should not get info message for SESSION_BUSY, but resend of Put. getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, static_cast<api::ReturnCode::Result>( mbus::ErrorCode::SESSION_BUSY)); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); // WRONG_DISTRIBUTION should not be reported, as it will happen all the // time when initiating remote migrations et al. getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::WRONG_DISTRIBUTION); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); // Complete message successfully to finish the visitor. getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::OK); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); - fetchSingleCommand<DestroyIteratorCommand>(*_bottom); + DestroyIteratorCommand::SP cmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, cmd)); - verifyCreateVisitorReply(api::ReturnCode::OK); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } -void -VisitorTest::testNotificationSentIfTransientErrorRetriedManyTimes() -{ +TEST_F(VisitorTest, notification_sent_if_transient_error_retried_many_times) { constexpr size_t retries( Visitor::TRANSIENT_ERROR_RETRIES_BEFORE_NOTIFY); - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); sendInitialCreateVisitorAndGetIterRound(); std::vector<document::Document::SP> docs; @@ -822,31 +748,33 @@ VisitorTest::testNotificationSentIfTransientErrorRetriedManyTimes() for (size_t attempt = 0; attempt < retries; ++attempt) { getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::WRONG_DISTRIBUTION); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + ASSERT_EQ(0, infoMessages.size()); } // Should now have a client notification along for the ride. // This has to be ACKed as OK or the visitor will fail. getMessagesAndReply(2, getSession(0), docs, docIds, infoMessages, api::ReturnCode::OK); - CPPUNIT_ASSERT_EQUAL(size_t(1), infoMessages.size()); + ASSERT_EQ(1, infoMessages.size()); // TODO(vekterli) ideally we'd want to test that this happens only once // per message, but this seems frustratingly complex to do currently. - fetchSingleCommand<DestroyIteratorCommand>(*_bottom); + DestroyIteratorCommand::SP cmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, cmd)); - verifyCreateVisitorReply(api::ReturnCode::OK); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } -std::shared_ptr<api::CreateVisitorReply> +void VisitorTest::doCompleteVisitingSession( - const std::shared_ptr<api::CreateVisitorCommand>& cmd) + const std::shared_ptr<api::CreateVisitorCommand>& cmd, + std::shared_ptr<api::CreateVisitorReply>& reply_out) { initializeTest(); _top->sendDown(cmd); sendCreateIteratorReply(); - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1, @@ -857,50 +785,44 @@ VisitorTest::doCompleteVisitingSession( std::vector<std::string> infoMessages; getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + ASSERT_EQ(1, replies.size()); std::shared_ptr<api::StorageMessage> msg(replies[0]); - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, - msg->getType()); - return std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); + ASSERT_EQ(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + reply_out = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); } -void -VisitorTest::testNoMbusTracingIfTraceLevelIsZero() -{ +TEST_F(VisitorTest, no_mbus_tracing_if_trace_level_is_zero) { std::shared_ptr<api::CreateVisitorCommand> cmd(makeCreateVisitor()); cmd->getTrace().setLevel(0); - auto reply = doCompleteVisitingSession(cmd); - CPPUNIT_ASSERT(reply->getTrace().getRoot().isEmpty()); + std::shared_ptr<api::CreateVisitorReply> reply; + ASSERT_NO_FATAL_FAILURE(doCompleteVisitingSession(cmd, reply)); + EXPECT_TRUE(reply->getTrace().getRoot().isEmpty()); } -void -VisitorTest::testReplyContainsTraceIfTraceLevelAboveZero() -{ +TEST_F(VisitorTest, reply_contains_trace_if_trace_level_above_zero) { std::shared_ptr<api::CreateVisitorCommand> cmd(makeCreateVisitor()); cmd->getTrace().setLevel(1); - auto reply = doCompleteVisitingSession(cmd); - CPPUNIT_ASSERT(!reply->getTrace().getRoot().isEmpty()); + std::shared_ptr<api::CreateVisitorReply> reply; + ASSERT_NO_FATAL_FAILURE(doCompleteVisitingSession(cmd, reply)); + EXPECT_FALSE(reply->getTrace().getRoot().isEmpty()); } -void -VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() -{ +TEST_F(VisitorTest, no_more_iterators_sent_while_memory_used_above_limit) { initializeTest(TestParams().maxVisitorMemoryUsage(1) .parallelBuckets(1)); - std::shared_ptr<api::CreateVisitorCommand> cmd( - makeCreateVisitor()); + auto cmd = makeCreateVisitor(); _top->sendDown(cmd); sendCreateIteratorReply(); - GetIterCommand::SP getIterCmd( - fetchSingleCommand<GetIterCommand>(*_bottom)); + GetIterCommand::SP getIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1); @@ -914,7 +836,7 @@ VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() // kind of explicit barrier with which we can synchronize the test and the // running visitor thread. std::this_thread::sleep_for(100ms); - CPPUNIT_ASSERT_EQUAL(size_t(0), _bottom->getNumCommands()); + ASSERT_EQ(0, _bottom->getNumCommands()); std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; @@ -922,7 +844,7 @@ VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); // 2nd round of GetIter now allowed. Send reply indicating completion. - getIterCmd = fetchSingleCommand<GetIterCommand>(*_bottom); + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<GetIterCommand>(*_bottom, getIterCmd)); sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1, @@ -930,11 +852,11 @@ VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); - DestroyIteratorCommand::SP destroyIterCmd( - fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + DestroyIteratorCommand::SP destroyIterCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<DestroyIteratorCommand>(*_bottom, destroyIterCmd)); - verifyCreateVisitorReply(api::ReturnCode::OK); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + ASSERT_NO_FATAL_FAILURE(verifyCreateVisitorReply(api::ReturnCode::OK)); + ASSERT_TRUE(waitUntilNoActiveVisitors()); } void @@ -942,19 +864,17 @@ VisitorTest::doTestVisitorInstanceHasConsistencyLevel( vespalib::stringref visitorType, spi::ReadConsistency expectedConsistency) { - initializeTest(); + ASSERT_NO_FATAL_FAILURE(initializeTest()); std::shared_ptr<api::CreateVisitorCommand> cmd( makeCreateVisitor(VisitorOptions().withVisitorType(visitorType))); _top->sendDown(cmd); - auto createCmd = fetchSingleCommand<CreateIteratorCommand>(*_bottom); - CPPUNIT_ASSERT_EQUAL(expectedConsistency, - createCmd->getReadConsistency()); + CreateIteratorCommand::SP createCmd; + ASSERT_NO_FATAL_FAILURE(fetchSingleCommand<CreateIteratorCommand>(*_bottom, createCmd)); + ASSERT_EQ(expectedConsistency, createCmd->getReadConsistency()); } -void -VisitorTest::testDumpVisitorInvokesStrongReadConsistencyIteration() -{ +TEST_F(VisitorTest, dump_visitor_invokes_strong_read_consistency_iteration) { doTestVisitorInstanceHasConsistencyLevel( "dumpvisitor", spi::ReadConsistency::STRONG); } @@ -965,9 +885,7 @@ VisitorTest::testDumpVisitorInvokesStrongReadConsistencyIteration() // any external client use cases. Our primary concern is to test that each // visitor subclass might report its own read consistency requirement and that // this is carried along to the CreateIteratorCommand. -void -VisitorTest::testTestVisitorInvokesWeakReadConsistencyIteration() -{ +TEST_F(VisitorTest, test_visitor_invokes_weak_read_consistency_iteration) { doTestVisitorInstanceHasConsistencyLevel( "testvisitor", spi::ReadConsistency::WEAK); } diff --git a/storage/src/vespa/storage/visiting/testvisitor.cpp b/storage/src/vespa/storage/visiting/testvisitor.cpp index a2e7116babd..9c1e04ecbbd 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.cpp +++ b/storage/src/vespa/storage/visiting/testvisitor.cpp @@ -21,7 +21,7 @@ TestVisitor::TestVisitor(StorageComponent& c, ost << "\n " << it->first << " = " << it->second.c_str(); } _params = ost.str(); - LOG(info, "Created TestVisitor: %s", _params.c_str()); + LOG(debug, "Created TestVisitor: %s", _params.c_str()); } void @@ -33,7 +33,7 @@ TestVisitor::startingVisitor(const std::vector<document::BucketId>& buckets) for (uint32_t i=0, n=buckets.size(); i<n; ++i) { ost << " " << buckets[i] << "\n"; } - LOG(info, "%s", ost.str().c_str()); + LOG(debug, "%s", ost.str().c_str()); report(ost.str()); } @@ -44,7 +44,7 @@ TestVisitor::handleDocuments(const document::BucketId& /*bucketId*/, { std::ostringstream ost; ost << "Handling block of " << entries.size() << " documents.\n"; - LOG(info, "%s", ost.str().c_str()); + LOG(debug, "%s", ost.str().c_str()); report(ost.str()); } @@ -52,19 +52,19 @@ void TestVisitor::completedBucket(const document::BucketId& bucket, HitCounter&) { std::ostringstream ost; ost << "completedBucket(" << bucket.getId() << ")\n"; - LOG(info, "%s", ost.str().c_str()); + LOG(debug, "%s", ost.str().c_str()); report(ost.str()); } void TestVisitor::completedVisiting(HitCounter&) { - LOG(info, "completedVisiting()"); + LOG(debug, "completedVisiting()"); report("completedVisiting()\n"); } void TestVisitor::abortedVisiting() { - LOG(info, "abortedVisiting()"); + LOG(debug, "abortedVisiting()"); report("abortedVisiting()\n"); } diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp index 3ff01cada85..9650834d0f1 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp @@ -6,12 +6,22 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/feature_resolver.h> #include <vespa/searchvisitor/hitcollector.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/vespalib/objects/nbostream.h> using namespace document; using namespace search::fef; using namespace vespalib; using namespace vdslib; using namespace vsm; +using vespalib::nbostream; +using vespalib::eval::Value; +using vespalib::eval::DoubleValue; +using vespalib::eval::TensorSpec; +using vespalib::tensor::DefaultTensorEngine; + namespace storage { @@ -226,26 +236,38 @@ HitCollectorTest::testEmpty() class MyRankProgram : public HitCollector::IRankProgram { private: + Value::UP _boxed_double; + Value::UP _tensor; NumberOrObject _fooValue; NumberOrObject _barValue; + NumberOrObject _bazValue; public: MyRankProgram() - : _fooValue(), - _barValue() + : _boxed_double(), + _tensor(), + _fooValue(), + _barValue(), + _bazValue() {} - virtual void run(uint32_t docid, const std::vector<search::fef::TermFieldMatchData> &) override { + ~MyRankProgram(); + virtual void run(uint32_t docid, const std::vector<search::fef::TermFieldMatchData> &) override { + _boxed_double = std::make_unique<DoubleValue>(docid + 30); + _tensor = DefaultTensorEngine::ref().from_spec(TensorSpec("tensor(x{})").add({{"x", "a"}}, docid + 20)); _fooValue.as_number = docid + 10; - _barValue.as_number = docid + 30; + _barValue.as_object = *_boxed_double; + _bazValue.as_object = *_tensor; } FeatureResolver get_resolver() { FeatureResolver resolver(2); resolver.add("foo", LazyValue(&_fooValue), false); - resolver.add("bar", LazyValue(&_barValue), false); + resolver.add("bar", LazyValue(&_barValue), true); + resolver.add("baz", LazyValue(&_bazValue), true); return resolver; } }; +MyRankProgram::~MyRankProgram() = default; void HitCollectorTest::testFeatureSet() @@ -262,28 +284,42 @@ HitCollectorTest::testFeatureSet() FeatureResolver resolver(rankProgram.get_resolver()); search::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver); - EXPECT_EQUAL(sf->getNames().size(), 2u); + EXPECT_EQUAL(sf->getNames().size(), 3u); EXPECT_EQUAL(sf->getNames()[0], "foo"); EXPECT_EQUAL(sf->getNames()[1], "bar"); - EXPECT_EQUAL(sf->numFeatures(), 2u); + EXPECT_EQUAL(sf->getNames()[2], "baz"); + EXPECT_EQUAL(sf->numFeatures(), 3u); EXPECT_EQUAL(sf->numDocs(), 3u); { - const search::feature_t * f = sf->getFeaturesByDocId(1); + const auto * f = sf->getFeaturesByDocId(1); ASSERT_TRUE(f != NULL); - EXPECT_EQUAL(f[0], 11); // 10 + docId - EXPECT_EQUAL(f[1], 31); // 30 + docId + EXPECT_EQUAL(f[0].as_double(), 11); // 10 + docId + EXPECT_EQUAL(f[1].as_double(), 31); // 30 + docId } { - const search::feature_t * f = sf->getFeaturesByDocId(3); + const auto * f = sf->getFeaturesByDocId(3); ASSERT_TRUE(f != NULL); - EXPECT_EQUAL(f[0], 13); - EXPECT_EQUAL(f[1], 33); + EXPECT_TRUE(f[0].is_double()); + EXPECT_TRUE(!f[0].is_data()); + EXPECT_EQUAL(f[0].as_double(), 13); + EXPECT_TRUE(f[1].is_double()); + EXPECT_TRUE(!f[1].is_data()); + EXPECT_EQUAL(f[1].as_double(), 33); + EXPECT_TRUE(!f[2].is_double()); + EXPECT_TRUE(f[2].is_data()); + { + auto &engine = DefaultTensorEngine::ref(); + nbostream buf(f[2].as_data().data, f[2].as_data().size); + auto actual = engine.to_spec(*engine.decode(buf)); + auto expect = TensorSpec("tensor(x{})").add({{"x", "a"}}, 23); + EXPECT_EQUAL(actual, expect); + } } { - const search::feature_t * f = sf->getFeaturesByDocId(4); + const auto * f = sf->getFeaturesByDocId(4); ASSERT_TRUE(f != NULL); - EXPECT_EQUAL(f[0], 14); - EXPECT_EQUAL(f[1], 34); + EXPECT_EQUAL(f[0].as_double(), 14); + EXPECT_EQUAL(f[1].as_double(), 34); } ASSERT_TRUE(sf->getFeaturesByDocId(0) == NULL); ASSERT_TRUE(sf->getFeaturesByDocId(2) == NULL); diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index a9b09cd7089..ce0cd967e06 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -4,6 +4,9 @@ #include <vespa/searchlib/fef/feature_resolver.h> #include <vespa/vespalib/util/stringfmt.h> #include <algorithm> +#include <vespa/eval/eval/tensor.h> +#include <vespa/eval/eval/tensor_engine.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.hitcollector"); @@ -156,10 +159,22 @@ HitCollector::getFeatureSet(IRankProgram &rankProgram, for (const Hit & hit : _hits) { rankProgram.run(hit.getDocId(), hit.getMatchData()); uint32_t docId = hit.getDocId(); - search::feature_t * f = retval->getFeaturesByIndex(retval->addDocId(docId)); + auto * f = retval->getFeaturesByIndex(retval->addDocId(docId)); for (uint32_t j = 0; j < names.size(); ++j) { - f[j] = resolver.resolve(j).as_number(docId); - LOG(debug, "getFeatureSet: lDocId(%u), '%s': %f", docId, names[j].c_str(), f[j]); + if (resolver.is_object(j)) { + auto obj = resolver.resolve(j).as_object(docId); + if (const auto *tensor = obj.get().as_tensor()) { + vespalib::nbostream buf; + tensor->engine().encode(*tensor, buf); + f[j].set_data(vespalib::Memory(buf.peek(), buf.size())); + } else { + f[j].set_double(obj.get().as_double()); + } + } else { + f[j].set_double(resolver.resolve(j).as_number(docId)); + } + LOG(debug, "getFeatureSet: lDocId(%u), '%s': %f %s", docId, names[j].c_str(), f[j].as_double(), + f[j].is_data() ? "[tensor]" : ""); } } return retval; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index a80fa9123ed..6fbf0134f45 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -202,7 +202,7 @@ RankProcessor::calculateFeatureSet() { LOG(debug, "Calculate feature set"); RankProgram &rankProgram = *(_summaryProgram.get() != nullptr ? _summaryProgram : _rankProgram); - search::fef::FeatureResolver resolver(rankProgram.get_seeds()); + search::fef::FeatureResolver resolver(rankProgram.get_seeds(false)); LOG(debug, "Feature handles: numNames(%ld)", resolver.num_features()); RankProgramWrapper wrapper(*_match_data); FeatureSet::SP sf = _hitCollector->getFeatureSet(wrapper, resolver); diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 3c48d22085e..de710e6e1b8 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -217,14 +217,14 @@ </profile> <profile> - <id>system-tests</id> + <id>functional-tests</id> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> - <groups>ai.vespa.hosted.cd.SystemTest</groups> + <groups>ai.vespa.hosted.cd.FunctionalTest</groups> <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> @@ -233,14 +233,14 @@ </profile> <profile> - <id>staging-tests</id> + <id>upgrade-tests</id> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> - <groups>ai.vespa.hosted.cd.StagingTest</groups> + <groups>ai.vespa.hosted.cd.UpgradeTest</groups> <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> @@ -371,8 +371,8 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <excludedGroups> - ai.vespa.hosted.cd.SystemTest, - ai.vespa.hosted.cd.StagingTest, + ai.vespa.hosted.cd.FunctionalTest, + ai.vespa.hosted.cd.UpgradeTest, ai.vespa.hosted.cd.ProductionTest </excludedGroups> </configuration> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java index efeb4214ebd..dbbb969efe2 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -3,6 +3,8 @@ package ai.vespa.hosted.cd; import ai.vespa.hosted.cd.metric.Metrics; import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; /** * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. @@ -16,6 +18,8 @@ public interface Endpoint { URI uri(); + <T> HttpResponse<T> send(HttpRequest.Builder request, HttpResponse.BodyHandler<T> handler); + Search search(Query query); Visit visit(Selection selection); diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java deleted file mode 100644 index ee2ee0add4c..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd; - -/** - * @deprecated Use {@link UpgradeTest}. - */ -@Deprecated -public class StagingTest { - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java deleted file mode 100644 index 6a8d1b4cbe4..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd; - -/** - * @deprecated use {@link FunctionalTest}. - */ -@Deprecated -public class SystemTest { - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java index 02c34501dd2..17703d8fbab 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -13,6 +13,8 @@ import ai.vespa.hosted.cd.TestEndpoint; import ai.vespa.hosted.cd.Visit; import ai.vespa.hosted.cd.metric.Metrics; +import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -58,15 +60,23 @@ public class HttpEndpoint implements TestEndpoint { } @Override + public <T> HttpResponse<T> send(HttpRequest.Builder request, HttpResponse.BodyHandler<T> handler) { + try { + return client.send(authenticator.authenticated(request).build(), handler); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override public Search search(Query query) { try { URI target = endpoint.resolve(searchApiPath).resolve("?" + query.rawQuery()); - HttpRequest request = authenticator.authenticated(HttpRequest.newBuilder() - .timeout(query.timeout().orElse(Duration.ofMillis(500)) - .plus(Duration.ofSeconds(1))) - .uri(target)) - .build(); - HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + HttpResponse<byte[]> response = send(HttpRequest.newBuilder(target) + .timeout(query.timeout().orElse(Duration.ofMillis(500)) + .plus(Duration.ofSeconds(1))), + HttpResponse.BodyHandlers.ofByteArray()); if (response.statusCode() / 100 != 2) // TODO consider allowing 504 if specified. throw new RuntimeException("Non-OK status code " + response.statusCode() + " at " + target + ", with response \n" + new String(response.body())); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java index 018acb17387..8170af15535 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestProfile.java @@ -6,8 +6,8 @@ package com.yahoo.vespa.hosted.testrunner; */ enum TestProfile { - SYSTEM_TEST("ai.vespa.hosted.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true), - STAGING_TEST("ai.vespa.hosted.cd.StagingTest, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true), + SYSTEM_TEST("ai.vespa.hosted.cd.FunctionalTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest", true), + STAGING_TEST("ai.vespa.hosted.cd.UpgradeTest, com.yahoo.vespa.tenant.systemtest.base.StagingTest", true), PRODUCTION_TEST("ai.vespa.hosted.cd.ProductionTest, com.yahoo.vespa.tenant.systemtest.base.ProductionTest", false); private final String group; diff --git a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests index 86c36afd636..d4b5bd54404 100644 --- a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests +++ b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests @@ -47,10 +47,10 @@ <dependenciesToScan> <dependency>com.yahoo.vespa.testrunner.test:main.jar</dependency> </dependenciesToScan> - <groups>ai.vespa.hosted.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</groups> + <groups>ai.vespa.hosted.cd.FunctionalTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</groups> <excludedGroups>com.yahoo.vespa.tenant.systemtest.base.impl.EmptyExcludeGroup.class</excludedGroups> <excludes> - <exclude>ai.vespa.hosted.cd.SystemTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</exclude> + <exclude>ai.vespa.hosted.cd.FunctionalTest, com.yahoo.vespa.tenant.systemtest.base.SystemTest</exclude> </excludes> <reportsDirectory>${env.TEST_DIR}</reportsDirectory> <redirectTestOutputToFile>false</redirectTestOutputToFile> diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index b2b895040bc..3b733105d2e 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -104,14 +104,22 @@ "com.yahoo.data.access.ObjectTraverser" ], "attributes": [ - "public", - "final" + "public" ], "methods": [ "public void <init>(java.lang.StringBuilder, boolean)", "public void encode(com.yahoo.data.access.Inspector)", + "protected void encodeEMPTY()", + "protected void encodeBOOL(boolean)", + "protected void encodeLONG(long)", + "protected void encodeDOUBLE(double)", + "protected void encodeSTRING(java.lang.String)", + "protected void encodeDATA(byte[])", + "protected void encodeARRAY(com.yahoo.data.access.Inspector)", + "protected void encodeOBJECT(com.yahoo.data.access.Inspector)", "public void entry(int, com.yahoo.data.access.Inspector)", - "public void field(java.lang.String, com.yahoo.data.access.Inspector)" + "public void field(java.lang.String, com.yahoo.data.access.Inspector)", + "public java.lang.StringBuilder target()" ], "fields": [] }, @@ -124,7 +132,8 @@ ], "methods": [ "public void <init>()", - "public static java.lang.StringBuilder render(com.yahoo.data.access.Inspectable, java.lang.StringBuilder, boolean)" + "public static java.lang.StringBuilder render(com.yahoo.data.access.Inspectable, java.lang.StringBuilder, boolean)", + "public static java.lang.StringBuilder render(com.yahoo.data.access.Inspectable, com.yahoo.data.access.simple.JsonRender$StringEncoder)" ], "fields": [] }, diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java index 253b0c60927..9f662c77c59 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java @@ -11,19 +11,25 @@ import com.yahoo.data.access.ObjectTraverser; * * @author arnej27959 */ -public final class JsonRender -{ +public final class JsonRender { + public static StringBuilder render(Inspectable value, StringBuilder target, - boolean compact) - { - StringEncoder enc = new StringEncoder(target, compact); - enc.encode(value.inspect()); - return target; + boolean compact) { + return render(value, new StringEncoder(target, compact)); + } + + /** + * Renders the given value to the target stringbuilder with a given encoder. + * This is useful to use an encoder where rendering of some value types is customized. + */ + public static StringBuilder render(Inspectable value, StringEncoder encoder) { + encoder.encode(value.inspect()); + return encoder.target(); } - public static final class StringEncoder implements ArrayTraverser, ObjectTraverser - { + public static class StringEncoder implements ArrayTraverser, ObjectTraverser { + private final StringBuilder out; private boolean head = true; private boolean compact; @@ -41,21 +47,21 @@ public final class JsonRender } } - private void encodeEMPTY() { + protected void encodeEMPTY() { out.append("null"); } - private void encodeBOOL(boolean value) { + protected void encodeBOOL(boolean value) { out.append(value ? "true" : "false"); } - private void encodeLONG(long value) { - out.append(String.valueOf(value)); + protected void encodeLONG(long value) { + out.append(value); } - private void encodeDOUBLE(double value) { + protected void encodeDOUBLE(double value) { if (Double.isFinite(value)) { - out.append(String.valueOf(value)); + out.append(value); } else { out.append("null"); } @@ -63,7 +69,7 @@ public final class JsonRender static final char[] hex = "0123456789ABCDEF".toCharArray(); - private void encodeSTRING(String value) { + protected void encodeSTRING(String value) { out.append('"'); for (char c : value.toCharArray()) { switch (c) { @@ -89,7 +95,7 @@ public final class JsonRender out.append('"'); } - private void encodeDATA(byte[] value) { + protected void encodeDATA(byte[] value) { out.append('"'); out.append("0x"); for (int pos = 0; pos < value.length; pos++) { @@ -99,14 +105,14 @@ public final class JsonRender out.append('"'); } - private void encodeARRAY(Inspector inspector) { + protected void encodeARRAY(Inspector inspector) { openScope("["); ArrayTraverser at = this; inspector.traverse(at); closeScope("]"); } - private void encodeOBJECT(Inspector inspector) { + protected void encodeOBJECT(Inspector inspector) { openScope("{"); ObjectTraverser ot = this; inspector.traverse(ot); @@ -164,5 +170,10 @@ public final class JsonRender out.append(' '); encodeValue(inspector); } + + /** Returns the target this is encoding values to */ + public StringBuilder target() { return out; } + } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 52635905d72..1a210a614cc 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -19,21 +19,33 @@ import java.util.Iterator; * A JSON map containing a 'cells' array. * See http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor */ -// TODO: We should probably move reading of this format from the document module to here public class JsonFormat { - /** Serializes the given tensor into JSON format */ + /** Serializes the given tensor value into JSON format */ public static byte[] encode(Tensor tensor) { Slime slime = new Slime(); Cursor root = slime.setObject(); - Cursor cellsArray = root.setArray("cells"); + encodeCells(tensor, root); + return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + } + + /** Serializes the given tensor type and value into JSON format */ + public static byte[] encodeWithType(Tensor tensor) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("type", tensor.type().toString()); + encodeCells(tensor, root); + return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + } + + private static void encodeCells(Tensor tensor, Cursor rootObject) { + Cursor cellsArray = rootObject.setArray("cells"); for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { Tensor.Cell cell = i.next(); Cursor cellObject = cellsArray.addObject(); encodeAddress(tensor.type(), cell.getKey(), cellObject.setObject("address")); cellObject.setDouble("value", cell.getValue()); } - return com.yahoo.slime.JsonFormat.toJsonBytes(slime); } private static void encodeAddress(TensorType type, TensorAddress address, Cursor addressObject) { diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index de9637c00f1..7c247cd01da 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -302,8 +302,7 @@ protected: * * @param pathSize New tree height (number of levels of internal nodes) */ - void - clearPath(uint32_t pathSize); + VESPA_DLL_LOCAL void clearPath(uint32_t pathSize); public: bool @@ -396,8 +395,7 @@ public: /** * Setup iterator to be empty and not be associated with any tree. */ - void - setupEmpty(); + VESPA_DLL_LOCAL void setupEmpty(); /** * Move iterator to beyond last element in the current tree. |