diff options
219 files changed, 4770 insertions, 2827 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index d2fe304a72c..33f2f909910 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -39,7 +39,7 @@ public class MasterElectionTest extends FleetControllerTest { @Rule public TestRule cleanupZookeeperLogsOnSuccess = new CleanupZookeeperLogsOnSuccess(); - private static int defaultZkSessionTimeoutInMillis() { return 10_000; } + private static int defaultZkSessionTimeoutInMillis() { return 30_000; } protected void setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions options) throws Exception { if (zooKeeperServer == null) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java index b0fd3a81732..5641233606e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java @@ -1,7 +1,5 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.vespa.applicationmodel.ClusterId; +package com.yahoo.config.model.api; import java.util.List; import java.util.Objects; @@ -15,15 +13,15 @@ import java.util.Objects; */ public class ContainerEndpoint { - private final ClusterId clusterId; + private final String clusterId; private final List<String> names; - public ContainerEndpoint(ClusterId clusterId, List<String> names) { + public ContainerEndpoint(String clusterId, List<String> names) { this.clusterId = Objects.requireNonNull(clusterId); this.names = List.copyOf(Objects.requireNonNull(names)); } - public ClusterId clusterId() { + public String clusterId() { return clusterId; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 031bc3467f5..136f30d437f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -50,6 +50,7 @@ public interface ModelContext { boolean hostedVespa(); Zone zone(); Set<Rotation> rotations(); + Set<ContainerEndpoint> endpoints(); boolean isBootstrap(); boolean isFirstTimeDeployment(); boolean useDedicatedNodeForLogserver(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 16fc6e7e2aa..1892c8920a7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -11,6 +11,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; @@ -68,6 +69,7 @@ public class DeployState implements ConfigDefinitionStore { private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<Rotation> rotations; + private final Set<ContainerEndpoint> endpoints; private final Zone zone; private final QueryProfiles queryProfiles; private final SemanticRules semanticRules; @@ -97,6 +99,7 @@ public class DeployState implements ConfigDefinitionStore { Optional<ConfigDefinitionRepo> configDefinitionRepo, java.util.Optional<Model> previousModel, Set<Rotation> rotations, + Set<ContainerEndpoint> endpoints, Collection<MlModelImporter> modelImporters, Zone zone, QueryProfiles queryProfiles, @@ -116,6 +119,7 @@ public class DeployState implements ConfigDefinitionStore { this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; this.rotations = rotations; + this.endpoints = Set.copyOf(endpoints); this.zone = zone; this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated @@ -235,6 +239,10 @@ public class DeployState implements ConfigDefinitionStore { return this.rotations; // todo: consider returning a copy or immutable view } + public Set<ContainerEndpoint> getEndpoints() { + return endpoints; + } + /** Returns the zone in which this is currently running */ public Zone zone() { return zone; } @@ -263,6 +271,7 @@ public class DeployState implements ConfigDefinitionStore { private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); private Optional<Model> previousModel = Optional.empty(); private Set<Rotation> rotations = new HashSet<>(); + private Set<ContainerEndpoint> endpoints = Set.of(); private Collection<MlModelImporter> modelImporters = Collections.emptyList(); private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); @@ -319,6 +328,11 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder endpoints(Set<ContainerEndpoint> endpoints) { + this.endpoints = endpoints; + return this; + } + public Builder modelImporters(Collection<MlModelImporter> modelImporters) { this.modelImporters = modelImporters; return this; @@ -360,6 +374,7 @@ public class DeployState implements ConfigDefinitionStore { configDefinitionRepo, previousModel, rotations, + endpoints, modelImporters, zone, queryProfiles, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 40d465d1ee6..d974db73547 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -3,6 +3,7 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; @@ -33,6 +34,7 @@ public class TestProperties implements ModelContext.Properties { private boolean hostedVespa = false; private Zone zone; private Set<Rotation> rotations; + private Set<ContainerEndpoint> endpoints = Collections.emptySet(); private boolean isBootstrap = false; private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; @@ -51,6 +53,8 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean hostedVespa() { return hostedVespa; } @Override public Zone zone() { return zone; } @Override public Set<Rotation> rotations() { return rotations; } + @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } + @Override public boolean isBootstrap() { return isBootstrap; } @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 9e0bbc395df..0dde5c99d4a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -56,7 +56,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { } if (! hostname.contains(".")) { deployLogger.log(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname"); + "since it is not a canonical hostname." + + "Disregard this warning when testing in a Docker container."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index af6400023cc..f69330eb196 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -139,6 +139,7 @@ public class VespaModelFactory implements ModelFactory { .vespaVersion(version()) .modelHostProvisioner(createHostProvisioner(modelContext)) .rotations(modelContext.properties().rotations()) + .endpoints(modelContext.properties().endpoints()) .modelImporters(modelImporters) .zone(zone) .now(clock.instant()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index fe40e6ffa84..29d1b557c49 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -10,14 +10,12 @@ import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.getDefaultMetricsConsumer; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; /** * Helper class to generate config for metrics consumers. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 47c6b2dbb52..5a41696c6f2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -47,7 +47,7 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.getDefaultMetricsConsumer; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer; import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; @@ -128,7 +128,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC @Override public void getConfig(ConsumersConfig.Builder builder) { - var amendedDefaultConsumer = addMetrics(getDefaultMetricsConsumer(), getAdditionalDefaultMetrics().getMetrics()); + var amendedDefaultConsumer = addMetrics(getVespaMetricsConsumer(), getAdditionalDefaultMetrics().getMetrics()); builder.consumer.addAll(generateConsumers(amendedDefaultConsumer, getUserMetricsConsumers())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricsConsumer.java index e9eca67a55a..81e9cfcd6a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricsConsumer.java @@ -10,22 +10,22 @@ import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricS import static java.util.Collections.emptyList; /** - * This class sets up the default 'Vespa' metrics consumer. + * This class sets up the 'Vespa' metrics consumer. * * @author trygve * @author gjoranv */ -public class DefaultMetricsConsumer { +public class VespaMetricsConsumer { public static final String VESPA_CONSUMER_ID = VespaMetrics.VESPA_CONSUMER_ID.id; - private static final MetricSet defaultConsumerMetrics = new MetricSet("default-consumer", + private static final MetricSet defaultConsumerMetrics = new MetricSet("vespa-consumer-metrics", emptyList(), ImmutableList.of(vespaMetricSet, systemMetricSet, networkMetricSet)); - public static MetricsConsumer getDefaultMetricsConsumer() { + public static MetricsConsumer getVespaMetricsConsumer() { return new MetricsConsumer(VESPA_CONSUMER_ID, defaultConsumerMetrics); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index fab1e90cc03..0ad0d57c1c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.admin.monitoring.builder.xml; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.text.XML; -import com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics; import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; @@ -15,7 +14,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 4b29af697f2..57e0b969929 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -72,6 +73,7 @@ import org.w3c.dom.Node; import java.net.URI; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -212,13 +214,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), deploymentSpec); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getRotations(), deploymentSpec); + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getRotations(), context.getDeployState().getEndpoints(), deploymentSpec); }); } - private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<Rotation> rotations, DeploymentSpec spec) { + private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<Rotation> rotations, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { cluster.getContainers().forEach(container -> { - setRotations(container, rotations, spec.globalServiceId(), cluster.getName()); + setRotations(container, rotations, endpoints, spec.globalServiceId(), cluster.getName()); container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); }); } @@ -229,13 +231,30 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { declaredZone.active()); } - private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { + private void setRotations(Container container, + Set<Rotation> rotations, + Set<ContainerEndpoint> endpoints, + Optional<String> globalServiceId, + String containerClusterName) { + final Set<String> rotationsProperty = new HashSet<>(); + // Add the legacy rotations to the list of available rotations. Using the same test + // as was used before to mirror the old business logic for global-service-id. if ( ! rotations.isEmpty() && globalServiceId.isPresent()) { if (containerClusterName.equals(globalServiceId.get())) { - container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); + rotations.stream().map(Rotation::getId).forEach(rotationsProperty::add); } } + + // For ContainerEndpoints this is more straight-forward, just add all that are present + endpoints.stream() + .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) + .flatMap(endpoint -> endpoint.names().stream()) + .forEach(rotationsProperty::add); + + // Build the comma delimited list of endpoints this container should be known as. + // Confusingly called 'rotations' for legacy reasons. + container.setProp("rotations", String.join(",", rotationsProperty)); } private void addRoutingAliases(ApplicationContainerCluster cluster, Element spec, Environment environment) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java index 34b727f9f4e..412264aec57 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java @@ -26,7 +26,7 @@ public class VespaMlModelTestCase { private final String expectedRankConfig = "constant(constant1).type : tensor(x[3])\n" + - "constant(constant1).value : tensor(x[3]):{{x:0}:0.5,{x:1}:1.5,{x:2}:2.5}\n" + + "constant(constant1).value : tensor(x[3]):[0.5, 1.5, 2.5]\n" + "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" + "rankingExpression(foo1).input2.type : tensor(x[3])\n" + "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index a08c5394dda..ff38a184eec 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -33,7 +33,7 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.g import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index bac9d674460..59b7110e96e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.test.VespaModelTester; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; import static org.junit.Assert.assertEquals; /** diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index c7816c23119..f787453dfb6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -33,6 +34,7 @@ import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; @@ -45,8 +47,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static org.hamcrest.CoreMatchers.is; @@ -611,6 +616,48 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void endpoints_are_added_to_containers() throws IOException, SAXException { + final var servicesXml = joinLines("", + "<container id='comics-search' version='1.0'>", + " <nodes>", + " <node hostalias='host1' />", + " </nodes>", + "</container>" + ); + + final var deploymentXml = joinLines("", + "<deployment version='1.0'>", + " <prod />", + "</deployment>" + ); + + final var applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withDeploymentSpec(deploymentXml) + .build(); + + final var deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))) + .endpoints(Set.of(new ContainerEndpoint("comics-search", List.of("nalle", "balle")))) + .properties(new TestProperties().setHostedVespa(true)) + .build(); + + final var model = new VespaModel(new NullConfigModelRegistry(), deployState); + final var containers = model.getContainerClusters().values().stream() + .flatMap(cluster -> cluster.getContainers().stream()) + .collect(Collectors.toList()); + + assertFalse("Missing container objects based on configuration", containers.isEmpty()); + + containers.forEach(container -> { + final var rotations = container.getServicePropertyString("rotations").split(","); + final var rotationsSet = Set.of(rotations); + assertEquals(Set.of("balle", "nalle"), rotationsSet); + }); + } + + @Test public void singlenode_servicespec_is_used_with_hosted_vespa() throws IOException, SAXException { String servicesXml = "<container id='default' version='1.0' />"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 18f4d317019..cf3f2d35bd7 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -391,18 +391,14 @@ "public boolean hasFastDisk()", "public double getBandwidth()", "public double getMinCpuCores()", - "public java.lang.String getDescription()", "public boolean isRetired()", "public com.yahoo.config.provision.Flavor$Type getType()", "public boolean isDocker()", - "public int getIdealHeadroom()", "public java.lang.String canonicalName()", "public boolean isCanonical()", "public java.util.List replaces()", "public boolean satisfies(com.yahoo.config.provision.Flavor)", - "public boolean hasAtLeast(com.yahoo.config.provision.NodeResources)", "public void freeze()", - "public boolean isLargerThan(com.yahoo.config.provision.Flavor)", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 8667707883d..b393d9ee22a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -5,10 +5,8 @@ import com.google.common.collect.ImmutableList; import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; /** * A host or node flavor. @@ -26,10 +24,8 @@ public class Flavor { private final boolean isStock; private final Type type; private final double bandwidth; - private final String description; private final boolean retired; private List<Flavor> replacesFlavors; - private int idealHeadroom; // Note: Not used after Vespa 6.282 /** The hardware resources of this flavor */ private NodeResources resources; @@ -46,10 +42,8 @@ public class Flavor { flavorConfig.minDiskAvailableGb(), flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow); this.bandwidth = flavorConfig.bandwidth(); - this.description = flavorConfig.description(); this.retired = flavorConfig.retired(); this.replacesFlavors = new ArrayList<>(); - this.idealHeadroom = flavorConfig.idealHeadroom(); } /** Creates a *node* flavor from a node resources spec */ @@ -64,10 +58,8 @@ public class Flavor { this.isStock = true; this.type = Type.DOCKER_CONTAINER; this.bandwidth = 1; - this.description = ""; this.retired = false; - this.replacesFlavors = Collections.emptyList(); - this.idealHeadroom = 0; + this.replacesFlavors = List.of(); this.resources = resources; } @@ -102,8 +94,6 @@ public class Flavor { public double getMinCpuCores() { return resources.vcpu(); } - public String getDescription() { return description; } - /** Returns whether the flavor is retired */ public boolean isRetired() { return retired; @@ -114,11 +104,6 @@ public class Flavor { /** Convenience, returns getType() == Type.DOCKER_CONTAINER */ public boolean isDocker() { return type == Type.DOCKER_CONTAINER; } - /** The free capacity we would like to preserve for this flavor */ - public int getIdealHeadroom() { - return idealHeadroom; - } - /** * Returns the canonical name of this flavor - which is the name which should be used as an interface to users. * The canonical name of this flavor is: @@ -164,23 +149,10 @@ public class Flavor { return false; } - /** - * Returns whether this flavor has at least the given resources, i.e if all resources of this are at least - * as large as the given resources. - */ - public boolean hasAtLeast(NodeResources resources) { - return this.resources.satisfies(resources); - } - /** Irreversibly freezes the content of this */ public void freeze() { replacesFlavors = ImmutableList.copyOf(replacesFlavors); } - - /** Returns whether this flavor has at least as much of each hardware resource as the given flavor */ - public boolean isLargerThan(Flavor other) { - return hasAtLeast(other.resources); - } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index 1e40f6f8f36..1cfb18d2cd2 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -43,12 +43,6 @@ flavor[].fastDisk bool default=true # Expected network interface bandwidth available for this flavor, in Mbit/s. flavor[].bandwidth double default=0.0 -# Human readable free text for description of node. -flavor[].description string default="" - # The flavor is retired and should no longer be used. flavor[].retired bool default=false -# The free capacity we would like to preserve for this flavor -# Note: Not used after Vespa 6.282 -flavor[].idealHeadroom int default=0 diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java index 81f3798a370..55ffa821e26 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java @@ -10,9 +10,7 @@ import java.util.ArrayList; import java.util.List; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; public class NodeFlavorsTest { @@ -59,16 +57,6 @@ public class NodeFlavorsTest { } @Test - public void testHasAtLeast() { - Flavor flavor = new Flavor(new NodeResources(1, 2, 3)); - assertTrue(flavor.hasAtLeast(new NodeResources(1, 2, 3))); - assertTrue(flavor.hasAtLeast(new NodeResources(1, 1.5, 2))); - assertFalse(flavor.hasAtLeast(new NodeResources(1, 1.5, 4))); - assertFalse(flavor.hasAtLeast(new NodeResources(2, 1.5, 4))); - assertFalse(flavor.hasAtLeast(new NodeResources(1, 2.1, 4))); - } - - @Test public void testRetiredFlavorWithoutReplacement() { FlavorsConfig.Builder builder = new FlavorsConfig.Builder(); List<FlavorsConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 97eed129f2f..d875385d14d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; @@ -127,6 +128,7 @@ public class ModelContextImpl implements ModelContext { private final boolean hostedVespa; private final Zone zone; private final Set<Rotation> rotations; + private final Set<ContainerEndpoint> endpoints; private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useDedicatedNodeForLogserver; @@ -144,6 +146,7 @@ public class ModelContextImpl implements ModelContext { boolean hostedVespa, Zone zone, Set<Rotation> rotations, + Set<ContainerEndpoint> endpoints, boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, @@ -157,6 +160,7 @@ public class ModelContextImpl implements ModelContext { this.hostedVespa = hostedVespa; this.zone = zone; this.rotations = rotations; + this.endpoints = endpoints; this.isBootstrap = isBootstrap; this.isFirstTimeDeployment = isFirstTimeDeployment; this.useDedicatedNodeForLogserver = Flags.USE_DEDICATED_NODE_FOR_LOGSERVER.bindTo(flagSource) @@ -202,6 +206,9 @@ public class ModelContextImpl implements ModelContext { public Set<Rotation> rotations() { return rotations; } @Override + public Set<ContainerEndpoint> endpoints() { return endpoints; } + + @Override public boolean isBootstrap() { return isBootstrap; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 08bc222a4c4..94cd30de28b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -25,6 +26,7 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; @@ -131,6 +133,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { configserverConfig.hostedVespa(), zone(), new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId), + ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating flagSource, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index a1570046df9..5bf70c55f9e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -10,7 +10,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; -import com.yahoo.vespa.config.server.tenant.ContainerEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import java.time.Clock; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index b224a218a9d..54c96c0461d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -22,7 +23,6 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; @@ -33,7 +33,7 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.tenant.ContainerEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; @@ -46,6 +46,7 @@ import javax.xml.transform.TransformerException; import java.io.IOException; import java.net.URI; import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -149,6 +150,7 @@ public class SessionPreparer { final Rotations rotations; // TODO: Remove this once we have migrated fully to container endpoints final ContainerEndpointsCache containerEndpoints; final Set<Rotation> rotationsSet; + final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; private final TlsSecretsKeys tlsSecretsKeys; private final Optional<TlsSecrets> tlsSecrets; @@ -174,6 +176,7 @@ public class SessionPreparer { this.rotationsSet = getRotations(params.rotations()); this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); + this.endpointsSet = getEndpoints(params.containerEndpoints()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), @@ -184,6 +187,7 @@ public class SessionPreparer { configserverConfig.hostedVespa(), zone, rotationsSet, + endpointsSet, params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), @@ -284,10 +288,17 @@ public class SessionPreparer { return rotations; } + private Set<ContainerEndpoint> getEndpoints(List<ContainerEndpoint> endpoints) { + if (endpoints == null || endpoints.isEmpty()) { + endpoints = this.containerEndpoints.read(applicationId); + } + return ImmutableSet.copyOf(endpoints); + } + } private static List<ContainerEndpoint> toContainerEndpoints(String globalServceId, Set<Rotation> rotations) { - return List.of(new ContainerEndpoint(new ClusterId(globalServceId), + return List.of(new ContainerEndpoint(globalServceId, rotations.stream() .map(Rotation::getId) .collect(Collectors.toUnmodifiableList()))); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index 91f9e3c8eed..4ffce8a697e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -1,11 +1,11 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.applicationmodel.ClusterId; import java.util.ArrayList; import java.util.List; @@ -49,7 +49,7 @@ public class ContainerEndpointSerializer { names.add(containerName); }); - return new ContainerEndpoint(new ClusterId(clusterId), names); + return new ContainerEndpoint(clusterId, names); } public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java index 7e29f9abc1d..9bce1224d96 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -1,6 +1,7 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.vespa.config.SlimeUtils; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index d7fafb2dace..860bbdc134c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.component.Version; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -14,6 +15,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Test; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -33,6 +35,10 @@ public class ModelContextImplTest { final Rotation rotation = new Rotation("this.is.a.mock.rotation"); final Set<Rotation> rotations = Collections.singleton(rotation); + + final ContainerEndpoint endpoint = new ContainerEndpoint("foo", List.of("a", "b")); + final Set<ContainerEndpoint> endpoints = Collections.singleton(endpoint); + final InMemoryFlagSource flagSource = new InMemoryFlagSource(); ModelContext context = new ModelContextImpl( @@ -53,6 +59,7 @@ public class ModelContextImplTest { false, Zone.defaultZone(), rotations, + endpoints, false, false, flagSource, @@ -72,6 +79,7 @@ public class ModelContextImplTest { assertNotNull(context.properties().zone()); assertFalse(context.properties().hostedVespa()); assertThat(context.properties().rotations(), equalTo(rotations)); + assertThat(context.properties().endpoints(), equalTo(endpoints)); assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); assertThat(context.properties().useDedicatedNodeForLogserver(), equalTo(true)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 395c1ecb80b..1f99f59eb8e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -20,11 +21,14 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.xml.sax.SAXException; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -33,20 +37,34 @@ import java.util.Random; import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * @author Ulf Lilleengen */ +@RunWith(Parameterized.class) public class LbServicesProducerTest { private static final String rotation1 = "rotation-1"; private static final String rotation2 = "rotation-2"; private static final String rotationString = rotation1 + "," + rotation2; private static final Set<Rotation> rotations = Collections.singleton(new Rotation(rotationString)); + private static final Set<ContainerEndpoint> endpoints = Set.of( + new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2")) + ); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final boolean useGlobalServiceId; + + @Parameterized.Parameters + public static Object[] useGlobalServiceId() { + return new Object[] { true, false }; + } + + public LbServicesProducerTest(boolean useGlobalServiceId) { + this.useGlobalServiceId = useGlobalServiceId; + } @Test public void testDeterministicGetConfig() throws IOException, SAXException { @@ -123,20 +141,40 @@ public class LbServicesProducerTest { @Test public void testConfigAliasesWithRotations() throws IOException, SAXException { + assumeTrue(useGlobalServiceId); + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() .rotations(rotations) .properties(new TestProperties().setHostedVespa(true))); RegionName regionName = RegionName.from("us-east-1"); - LbServicesConfig conf = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); - final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); - assertThat(services.servicealiases().size(), is(1)); - assertThat(services.endpointaliases().size(), is(4)); - assertThat(services.servicealiases(0), is("service1")); - assertThat(services.endpointaliases(0), is("foo1.bar1.com")); - assertThat(services.endpointaliases(1), is("foo2.bar2.com")); - assertThat(services.endpointaliases(2), is(rotation1)); - assertThat(services.endpointaliases(3), is(rotation2)); + var services = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel) + .tenants("foo") + .applications("foo:prod:" + regionName.value() + ":default") + .hosts("foo.foo.yahoo.com") + .services(QRSERVER.serviceName); + + assertThat(services.servicealiases(), contains("service1")); + assertThat("Missing rotations in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); + } + + @Test + public void testConfigAliasesWithEndpoints() throws IOException, SAXException { + assumeFalse(useGlobalServiceId); + + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() + .endpoints(endpoints) + .properties(new TestProperties().setHostedVespa(true))); + RegionName regionName = RegionName.from("us-east-1"); + + var services = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel) + .tenants("foo") + .applications("foo:prod:" + regionName.value() + ":default") + .hosts("foo.foo.yahoo.com") + .services(QRSERVER.serviceName); + + assertThat(services.servicealiases(), contains("service1")); + assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); } private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) { @@ -195,14 +233,32 @@ public class LbServicesProducerTest { " <search/>" + "</jdisc>" + "</services>"; - String deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + - "<deployment version='1.0'>" + - " <test />" + - " <prod global-service-id='mydisc'>" + - " <region active='true'>us-east-1</region>" + - " <region active='false'>us-east-2</region>" + - " </prod>" + - "</deployment>"; + + String deploymentInfo; + + if (useGlobalServiceId) { + deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod global-service-id='mydisc'>" + + " <region active='true'>us-east-1</region>" + + " <region active='false'>us-east-2</region>" + + " </prod>" + + "</deployment>"; + } else { + deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod>" + + " <region active='true'>us-east-1</region>" + + " <region active='false'>us-east-2</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint container-id='mydisc' />" + + " </endpoints>" + + "</deployment>"; + } + return new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withDeploymentSpec(deploymentInfo).build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 6eba85af37e..f5fd6053b07 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -6,8 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.config.server.tenant.ContainerEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import org.junit.Test; import java.net.URLEncoder; @@ -84,10 +83,10 @@ public class PrepareParamsTest { @Test public void testCorrectParsingWithContainerEndpoints() { - var endpoints = List.of(new ContainerEndpoint(new ClusterId("qrs1"), + var endpoints = List.of(new ContainerEndpoint("qrs1", List.of("c1.example.com", "c2.example.com")), - new ContainerEndpoint(new ClusterId("qrs2"), + new ContainerEndpoint("qrs2", List.of("c3.example.com", "c4.example.com"))); var param = "[\n" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 02f5dbeb4cb..88baf1b8d74 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -16,7 +16,6 @@ import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; @@ -29,7 +28,7 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.tenant.ContainerEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; @@ -222,7 +221,7 @@ public class SessionPreparerTest { var params = new PrepareParams.Builder().applicationId(applicationId).rotations(rotations).build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); - var expected = List.of(new ContainerEndpoint(new ClusterId("qrs"), + var expected = List.of(new ContainerEndpoint("qrs", List.of("app1.tenant1.global.vespa.example.com", "rotation-042.vespa.global.routing"))); assertEquals(expected, readContainerEndpoints(applicationId)); @@ -252,10 +251,10 @@ public class SessionPreparerTest { .build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); - var expected = List.of(new ContainerEndpoint(new ClusterId("foo"), + var expected = List.of(new ContainerEndpoint("foo", List.of("foo.app1.tenant1.global.vespa.example.com", "rotation-042.vespa.global.routing")), - new ContainerEndpoint(new ClusterId("bar"), + new ContainerEndpoint("bar", List.of("bar.app1.tenant1.global.vespa.example.com", "rotation-043.vespa.global.routing"))); assertEquals(expected, readContainerEndpoints(applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java index aac0b6d1a16..053a3f7a15d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.slime.Slime; -import com.yahoo.vespa.applicationmodel.ClusterId; import org.junit.Test; import java.util.List; @@ -30,7 +30,7 @@ public class ContainerEndpointSerializerTest { @Test public void writeReadSingleEndpoint() { - final var endpoint = new ContainerEndpoint(new ClusterId("foo"), List.of("a", "b")); + final var endpoint = new ContainerEndpoint("foo", List.of("a", "b")); final var serialized = new Slime(); ContainerEndpointSerializer.endpointToSlime(serialized.setObject(), endpoint); final var deserialized = ContainerEndpointSerializer.endpointFromSlime(serialized.get()); @@ -40,7 +40,7 @@ public class ContainerEndpointSerializerTest { @Test public void writeReadEndpoints() { - final var endpoints = List.of(new ContainerEndpoint(new ClusterId("foo"), List.of("a", "b"))); + final var endpoints = List.of(new ContainerEndpoint("foo", List.of("a", "b"))); final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java index 3598b6e63c3..4400b424d1b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java @@ -1,9 +1,9 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; -import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Test; @@ -17,7 +17,7 @@ public class ContainerEndpointsCacheTest { public void readWriteFromCache() { final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); final var endpoints = List.of( - new ContainerEndpoint(new ClusterId("the-cluster-1"), List.of("a", "b", "c")) + new ContainerEndpoint("the-cluster-1", List.of("a", "b", "c")) ); cache.write(ApplicationId.defaultId(), endpoints); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java new file mode 100644 index 00000000000..21f38084c22 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/ArtifactId.java @@ -0,0 +1,26 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import static java.util.Objects.requireNonNull; + +/** + * Identifier for an artifact. + * + * @author jonmv + */ +public class ArtifactId { + + private final String groupId; + private final String artifactId; + + public ArtifactId(String groupId, String artifactId) { + this.groupId = requireNonNull(groupId); + this.artifactId = requireNonNull(artifactId); + } + + /** Group ID of this. */ + public String groupId() { return groupId; } + + /** Artifact ID of this. */ + public String artifactId() { return artifactId; } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java new file mode 100644 index 00000000000..fb133f75654 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +/** + * A Maven repository which keeps released artifacts. + * + * @author jonmv + */ +public interface MavenRepository { + + /** Returns metadata about all releases of a specific artifact to this repository. */ + Metadata metadata(); + + /** Returns the id of the artifact whose releases this tracks. */ + ArtifactId artifactId(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java new file mode 100644 index 00000000000..fd84a05db6a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java @@ -0,0 +1,50 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import com.yahoo.component.Version; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; + +/** + * Metadata about a released artifact. + * + * @author jonmv + */ +public class Metadata { + + private final ArtifactId id; + private final List<Version> versions; + + public Metadata(ArtifactId id, List<Version> versions) { + this.id = requireNonNull(id); + this.versions = versions.stream().sorted().collect(Collectors.toUnmodifiableList()); + } + + /** Creates a new Metadata object from the given XML document. */ + public static Metadata fromXml(String xml) { + Element metadata = XML.getDocument(xml).getDocumentElement(); + ArtifactId id = new ArtifactId(XML.getValue(XML.getChild(metadata, "groupId")), + XML.getValue(XML.getChild(metadata, "artifactId"))); + List<Version> versions = new ArrayList<>(); + for (Element version : XML.getChildren(XML.getChild(XML.getChild(metadata, "versioning"), "versions"))) + versions.add(Version.fromString(XML.getValue(version))); + + return new Metadata(id, versions); + } + + /** Id of the metadata this concerns. */ + public ArtifactId id() { return id; } + + /** List of available versions of this, sorted by ascending version order. */ + public List<Version> versions() { return versions; } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java new file mode 100644 index 00000000000..d5abdf31f4b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java new file mode 100644 index 00000000000..f716458542c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java @@ -0,0 +1,12 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public interface Billing { + + void handleBilling(ApplicationId applicationId, String customerId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java new file mode 100644 index 00000000000..20b77703160 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public class MockBilling implements Billing { + + @Override + public void handleBilling(ApplicationId applicationId, String customerId) {} +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java new file mode 100644 index 00000000000..be1deb3997a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; + +import java.util.List; + +/** + * Mock repository for maven artifacts, that returns a static metadata. + * + * @author jonmv + */ +public class MockMavenRepository implements MavenRepository { + + public static final ArtifactId id = new ArtifactId("ai.vespa", "search"); + + @Override + public Metadata metadata() { + return new Metadata(id, List.of(Version.fromString("6.0"), + Version.fromString("6.1"), + Version.fromString("6.2"))); + } + + @Override + public ArtifactId artifactId() { + return id; + } + +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java new file mode 100644 index 00000000000..17d0694538c --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import com.yahoo.component.Version; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; + +public class MetadataTest { + + @Test + public void testParsing() { + Metadata metadata = Metadata.fromXml(metadataXml); + assertEquals("com.yahoo.vespa", metadata.id().groupId()); + assertEquals("tenant-base", metadata.id().artifactId()); + assertEquals(Version.fromString("6.297.80"), metadata.versions().get(0)); + assertEquals(Version.fromString("7.61.10"), metadata.versions().get(metadata.versions().size() - 1)); + } + + private static final String metadataXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<metadata>\n" + + " <groupId>com.yahoo.vespa</groupId>\n" + + " <artifactId>tenant-base</artifactId>\n" + + " <versioning>\n" + + " <latest>7.61.10</latest>\n" + + " <release>7.61.10</release>\n" + + " <versions>\n" + + " <version>6.297.80</version>\n" + + " <version>6.300.15</version>\n" + + " <version>6.301.8</version>\n" + + " <version>6.303.29</version>\n" + + " <version>6.304.14</version>\n" + + " <version>6.305.35</version>\n" + + " <version>6.328.65</version>\n" + + " <version>6.329.64</version>\n" + + " <version>6.330.51</version>\n" + + " <version>7.3.19</version>\n" + + " <version>7.18.17</version>\n" + + " <version>7.20.129</version>\n" + + " <version>7.21.18</version>\n" + + " <version>7.22.18</version>\n" + + " <version>7.38.38</version>\n" + + " <version>7.39.5</version>\n" + + " <version>7.40.41</version>\n" + + " <version>7.41.15</version>\n" + + " <version>7.57.40</version>\n" + + " <version>7.60.51</version>\n" + + " <version>7.61.10</version>\n" + + " </versions>\n" + + " <lastUpdated>20190619054245</lastUpdated>\n" + + " </versioning>\n" + + "</metadata>\n"; +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6f0ee75d098..d87d52f2c12 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; @@ -83,6 +84,7 @@ public class Controller extends AbstractComponent { private final AuditLogger auditLogger; private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; + private final MavenRepository mavenRepository; /** * Creates a controller @@ -95,11 +97,13 @@ public class Controller extends AbstractComponent { RoutingGenerator routingGenerator, Chef chef, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, - BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource) { + BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, + MavenRepository mavenRepository) { this(curator, rotationsConfig, gitHub, zoneRegistry, configServer, metricsService, routingGenerator, chef, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, - buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource); + buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, + mavenRepository); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, @@ -109,7 +113,7 @@ public class Controller extends AbstractComponent { AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier<String> hostnameSupplier, - Mailer mailer, FlagSource flagSource) { + Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -122,6 +126,7 @@ public class Controller extends AbstractComponent { this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.nameServiceForwarder = new NameServiceForwarder(curator); + this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud)); applicationController = new ApplicationController(this, curator, accessControl, @@ -164,9 +169,9 @@ public class Controller extends AbstractComponent { public ZoneRegistry zoneRegistry() { return zoneRegistry; } - public NameServiceForwarder nameServiceForwarder() { - return nameServiceForwarder; - } + public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } + + public MavenRepository mavenRepository() { return mavenRepository; } public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java new file mode 100644 index 00000000000..c6956293adf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; + +import java.time.Duration; +import java.util.EnumSet; + +/** + * @author olaa + */ +public class BillingMaintainer extends Maintainer { + + private final Billing billing; + + public BillingMaintainer(Controller controller, Duration interval, JobControl jobControl, Billing billing) { + super(controller, interval, jobControl, BillingMaintainer.class.getSimpleName(), EnumSet.of(SystemName.cd)); + this.billing = billing; + } + + @Override + public void maintain() { + controller().tenants().asList() + .stream() + .filter(tenant -> tenant instanceof CloudTenant) + .map(tenant -> (CloudTenant) tenant) + .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) + .stream() + .forEach( application -> { + billing.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); + }) + ); + } +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index c1f896e6593..e840deb062c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; @@ -13,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -55,6 +55,7 @@ public class ControllerMaintenance extends AbstractComponent { private final CostReportMaintainer costReportMaintainer; private final ResourceMeterMaintainer resourceMeterMaintainer; private final NameServiceDispatcher nameServiceDispatcher; + private final BillingMaintainer billingMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, @@ -64,6 +65,7 @@ public class ControllerMaintenance extends AbstractComponent { ContactRetriever contactRetriever, CostReportConsumer reportConsumer, ResourceSnapshotConsumer resourceSnapshotConsumer, + Billing billing, SelfHostedCostConfig selfHostedCostConfig) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; @@ -86,6 +88,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); + billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); } public Upgrader upgrader() { return upgrader; } @@ -114,6 +117,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer.deconstruct(); resourceMeterMaintainer.deconstruct(); nameServiceDispatcher.deconstruct(); + billingMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 2b26e93aeb8..30bca180c0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -12,7 +12,8 @@ import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfi import java.time.Clock; import java.time.Duration; -import java.util.*; +import java.util.EnumSet; +import java.util.Objects; import java.util.logging.Logger; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 405a2e452d0..207a5f8dcf9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -42,6 +42,7 @@ public class VersionStatusSerializer { private static final String committedAtField = "releasedAt"; private static final String isControllerVersionField = "isCurrentControllerVersion"; private static final String isSystemVersionField = "isCurrentSystemVersion"; + private static final String isReleasedField = "isReleased"; private static final String deploymentStatisticsField = "deploymentStatistics"; private static final String confidenceField = "confidence"; private static final String configServersField = "configServerHostnames"; @@ -73,6 +74,7 @@ public class VersionStatusSerializer { object.setLong(committedAtField, version.committedAt().toEpochMilli()); object.setBool(isControllerVersionField, version.isControllerVersion()); object.setBool(isSystemVersionField, version.isSystemVersion()); + object.setBool(isReleasedField, version.isReleased()); deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); object.setString(confidenceField, version.confidence().name()); configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField)); @@ -105,6 +107,7 @@ public class VersionStatusSerializer { Instant.ofEpochMilli(object.field(committedAtField).asLong()), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), + object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true, configServersFromSlime(object.field(configServersField)), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 3db8c447572..9f091061596 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -97,6 +97,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -104,6 +105,7 @@ import java.util.Scanner; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; +import java.util.stream.Collectors; import static java.util.stream.Collectors.joining; @@ -515,7 +517,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); // Compile version. The version that should be used when building an application - object.setString("compileVersion", controller.applications().oldestInstalledPlatform(application.id()).toFullString()); + object.setString("compileVersion", compileVersion(application.id()).toFullString()); application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); @@ -693,6 +695,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return controller.zoneRegistry().getMonitoringSystemUri(deploymentId); } + /** + * Returns a non-broken, released version at least as old as the oldest platform the given application is on. + * + * If no known version is applicable, the newest version at least as old as the oldest platform is selected, + * among all versions released for this system. If no such versions exists, throws an IllegalStateException. + */ + private Version compileVersion(ApplicationId id) { + Version oldestPlatform = controller.applications().oldestInstalledPlatform(id); + return controller.versionStatus().versions().stream() + .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low)) + .filter(VespaVersion::isReleased) + .map(VespaVersion::versionNumber) + .filter(version -> ! version.isAfter(oldestPlatform)) + .max(Comparator.naturalOrder()) + .orElseGet(() -> controller.mavenRepository().metadata().versions().stream() + .filter(version -> ! version.isAfter(oldestPlatform)) + .filter(version -> ! controller.versionStatus().versions().stream() + .map(VespaVersion::versionNumber) + .collect(Collectors.toSet()).contains(version)) + .max(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalStateException("No available releases of " + + controller.mavenRepository().artifactId()))); + } + private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); ZoneId zone = ZoneId.from(environment, region); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java new file mode 100644 index 00000000000..9f3addd4992 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -0,0 +1,64 @@ +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; +import com.yahoo.vespa.hosted.controller.maven.repository.config.MavenRepositoryConfig; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Http client implementation of a {@link MavenRepository}, which uses a configured repository and artifact ID. + * + * @author jonmv + */ +public class MavenRepositoryClient implements MavenRepository { + + private final HttpClient client; + private final URI apiUrl; + private final ArtifactId id; + + public MavenRepositoryClient(MavenRepositoryConfig config) { + this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + this.apiUrl = URI.create(config.apiUrl() + "/").normalize(); + this.id = new ArtifactId(config.groupId(), config.artifactId()); + } + + @Override + public Metadata metadata() { + try { + HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); + HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); + if (response.statusCode() != 200) + throw new RuntimeException("Status code '" + response.statusCode() + "' and body\n'''\n" + + response.body() + "\n'''\nfor request " + request); + + return Metadata.fromXml(response.body()); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public ArtifactId artifactId() { + return id; + } + + static URI withArtifactPath(URI baseUrl, ArtifactId id) { + List<String> parts = new ArrayList<>(List.of(id.groupId().split("\\."))); + parts.add(id.artifactId()); + parts.add("maven-metadata.xml"); + return baseUrl.resolve(String.join("/", parts)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 5e57e9ebe8e..87f35d3b2c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -129,14 +129,17 @@ public class VersionStatus { Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, controller.applications().asList()); List<VespaVersion> versions = new ArrayList<>(); + List<Version> releasedVersions = controller.mavenRepository().metadata().versions(); for (DeploymentStatistics statistics : deploymentStatistics) { if (statistics.version().isEmpty()) continue; try { + boolean isReleased = Collections.binarySearch(releasedVersions, statistics.version()) >= 0; VespaVersion vespaVersion = createVersion(statistics, statistics.version().equals(controllerVersion), statistics.version().equals(systemVersion), + isReleased, systemApplicationVersions.getList(statistics.version()), controller); versions.add(vespaVersion); @@ -145,6 +148,7 @@ public class VersionStatus { statistics.version().toFullString(), e); } } + Collections.sort(versions); return new VersionStatus(versions); @@ -238,10 +242,11 @@ public class VersionStatus { } return versionMap.values(); } - + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isControllerVersion, - boolean isSystemVersion, + boolean isSystemVersion, + boolean isReleased, Collection<HostName> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); @@ -260,6 +265,7 @@ public class VersionStatus { gitSha.sha, committedAt, isControllerVersion, isSystemVersion, + isReleased, configServerHostnames, confidence ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index ffbf24be12a..117ce80adaa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -27,12 +27,13 @@ public class VespaVersion implements Comparable<VespaVersion> { private final Instant committedAt; private final boolean isControllerVersion; private final boolean isSystemVersion; + private final boolean isReleased; private final DeploymentStatistics statistics; private final ImmutableSet<HostName> systemApplicationHostnames; private final Confidence confidence; public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt, - boolean isControllerVersion, boolean isSystemVersion, + boolean isControllerVersion, boolean isSystemVersion, boolean isReleased, Collection<HostName> systemApplicationHostnames, Confidence confidence) { this.statistics = statistics; @@ -40,6 +41,7 @@ public class VespaVersion implements Comparable<VespaVersion> { this.committedAt = committedAt; this.isControllerVersion = isControllerVersion; this.isSystemVersion = isSystemVersion; + this.isReleased = isReleased; this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames); this.confidence = confidence; } @@ -102,6 +104,9 @@ public class VespaVersion implements Comparable<VespaVersion> { */ public boolean isSystemVersion() { return isSystemVersion; } + /** Returns whether the artifacts of this release are available in the configured maven repository. */ + public boolean isReleased() { return isReleased; } + /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */ public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java new file mode 100644 index 00000000000..c6f2c1e427d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/resources/configdefinitions/maven-repository.def b/controller-server/src/main/resources/configdefinitions/maven-repository.def new file mode 100644 index 00000000000..0fd2d410e9b --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/maven-repository.def @@ -0,0 +1,15 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.maven.repository.config + + +# URL to the Maven repository API that holds artifacts for tenants in the controller's system +# +apiUrl string default=https://repo.maven.apache.org/maven2/ + +# Group ID of the artifact to list versions for +# +groupId string default=com.yahoo.vespa + +# Artifact ID of the artifact to list versions for +# +artifactId string default=tenant-base diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 2b0ee741e7e..5748ad4f55c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueH import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.config.provision.zone.ZoneId; @@ -351,7 +352,8 @@ public final class ControllerTester { new MockRunDataStore(), () -> "test-controller", new MockMailer(), - new InMemoryFlagSource()); + new InMemoryFlagSource(), + new MockMavenRepository()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 5e6f9811376..a1e22b4fc64 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -36,9 +36,9 @@ public class VersionStatusSerializerTest { ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, - asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true, - asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -51,6 +51,7 @@ public class VersionStatusSerializerTest { assertEquals(a.committedAt().truncatedTo(MILLIS), b.committedAt()); assertEquals(a.isControllerVersion(), b.isControllerVersion()); assertEquals(a.isSystemVersion(), b.isSystemVersion()); + assertEquals(a.isReleased(), b.isReleased()); assertEquals(a.statistics(), b.statistics()); assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames()); assertEquals(a.confidence(), b.confidence()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 76c505ff8f8..427428a3a94 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -54,7 +54,7 @@ public class ContainerControllerTester { public ContainerControllerTester(JDisc container, String responseFilePath) { containerTester = new ContainerTester(container, responseFilePath); - CuratorDb curatorDb = new MockCuratorDb(); + CuratorDb curatorDb = controller().curator(); curatorDb.writeUpgradesPerMinute(100); upgrader = new Upgrader(controller(), Duration.ofDays(1), new JobControl(curatorDb), curatorDb); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index e974f55c9f6..6f612005524 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -72,6 +72,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockBilling'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock'/>\n" + @@ -91,6 +92,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.integration.ApplicationStoreMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + " <binding>http://*/deployment/v1/*</binding>\n" + " </handler>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 71c4b41a276..16fd10277d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -61,6 +61,8 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; import org.junit.Test; @@ -369,6 +371,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT), ""); + // Set version 6.1 to broken to change compile version for. + controllerTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); + tester.computeVersionStatus(); setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 5d1819bf0f2..1d719133ac3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -217,7 +217,7 @@ ] } ], - "compileVersion": "(ignore)", + "compileVersion": "6.0.0", "globalRotations": [ "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index ba81c5cf4e4..74d637499bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -80,7 +80,7 @@ public class ControllerApiTest extends ControllerContainerTest { public void testUpgraderApi() { // Get current configuration tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", new byte[0], Request.Method.GET), - "{\"upgradesPerMinute\":0.125,\"confidenceOverrides\":[]}", + "{\"upgradesPerMinute\":100.0,\"confidenceOverrides\":[]}", 200); // Set invalid configuration diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 01b063c84e1..d4f3e20ac14 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -4,6 +4,9 @@ "name": "ApplicationOwnershipConfirmer" }, { + "name": "BillingMaintainer" + }, + { "name": "ClusterInfoMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index fa3848a6ba5..d6620733efe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -74,6 +74,7 @@ public class DeploymentApiTest extends ControllerContainerTest { version.committedAt(), version.isControllerVersion(), version.isSystemVersion(), + version.isReleased(), ImmutableSet.of("config1.test", "config2.test").stream() .map(HostName::from) .collect(Collectors.toSet()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java new file mode 100644 index 00000000000..026d174cb73 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java @@ -0,0 +1,22 @@ +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import org.junit.Test; + +import java.net.URI; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class MavenRepositoryClientTest { + + @Test + public void testUri() { + assertEquals(URI.create("https://domain:123/base/group/id/artifact-id/maven-metadata.xml"), + MavenRepositoryClient.withArtifactPath(URI.create("https://domain:123/base/"), + new ArtifactId("group.id", "artifact-id"))); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index a365285b752..8e3dc24193f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -254,7 +254,7 @@ public class VersionStatusTest { assertTrue("Status for version without applications is removed", tester.controller().versionStatus().versions().stream() .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1))); - + // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); tester.completeUpgrade(default9, version2, "default"); @@ -294,6 +294,11 @@ public class VersionStatusTest { assertEquals("6.2", versions.get(0).versionNumber().toString()); assertEquals("6.4", versions.get(1).versionNumber().toString()); assertEquals("6.5", versions.get(2).versionNumber().toString()); + + // Check release status is correct (static data in MockMavenRepository). + assertTrue(versions.get(0).isReleased()); + assertFalse(versions.get(1).isReleased()); + assertFalse(versions.get(2).isReleased()); } @Test diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index a3d2a157073..6bdac611fdc 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -3,6 +3,10 @@ package com.yahoo.document.json.readers; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; +import com.yahoo.lang.MutableInteger; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Type; +import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; @@ -11,54 +15,61 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.*; /** * Reads the tensor format described at * http://docs.vespa.ai/documentation/reference/document-json-format.html#tensor + * + * @author geirst + * @author bratseth */ public class TensorReader { public static final String TENSOR_ADDRESS = "address"; public static final String TENSOR_DIMENSIONS = "dimensions"; public static final String TENSOR_CELLS = "cells"; + public static final String TENSOR_VALUES = "values"; public static final String TENSOR_VALUE = "value"; - public static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { + static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { // TODO: Switch implementation to om.yahoo.tensor.serialization.JsonFormat.decode - Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); + Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); - // read tensor cell fields and ignore everything else for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - if (TensorReader.TENSOR_CELLS.equals(buffer.currentName())) - readTensorCells(buffer, tensorBuilder); + if (TENSOR_CELLS.equals(buffer.currentName())) + readTensorCells(buffer, builder); + else if (TENSOR_VALUES.equals(buffer.currentName())) + readTensorValues(buffer, builder); + else if (builder.type().dimensions().stream().anyMatch(d -> d.isIndexed())) // sparse can be empty + throw new IllegalArgumentException("Expected a tensor value to contain either 'cells' or 'values'"); } expectObjectEnd(buffer.currentToken()); - tensorFieldValue.assign(tensorBuilder.build()); + tensorFieldValue.assign(builder.build()); } - public static void readTensorCells(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + static void readTensorCells(TokenBuffer buffer, Tensor.Builder builder) { expectArrayStart(buffer.currentToken()); int initNesting = buffer.nesting(); for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) - readTensorCell(buffer, tensorBuilder); + readTensorCell(buffer, builder); expectCompositeEnd(buffer.currentToken()); } - public static void readTensorCell(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + private static void readTensorCell(TokenBuffer buffer, Tensor.Builder builder) { expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); + Tensor.Builder.CellBuilder cellBuilder = builder.cell(); for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { String currentName = buffer.currentName(); if (TensorReader.TENSOR_ADDRESS.equals(currentName)) { readTensorAddress(buffer, cellBuilder); } else if (TensorReader.TENSOR_VALUE.equals(currentName)) { - cellValue = Double.valueOf(buffer.currentText()); + cellValue = readDouble(buffer); } } expectObjectEnd(buffer.currentToken()); cellBuilder.value(cellValue); } - public static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) { + private static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) { expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { @@ -68,4 +79,28 @@ public class TensorReader { } expectObjectEnd(buffer.currentToken()); } + + private static void readTensorValues(TokenBuffer buffer, Tensor.Builder builder) { + if ( ! (builder instanceof IndexedTensor.BoundBuilder)) + throw new IllegalArgumentException("The 'values' field can only be used with dense tensors. " + + "Use 'cells' instead"); + expectArrayStart(buffer.currentToken()); + + IndexedTensor.BoundBuilder indexedBuilder = (IndexedTensor.BoundBuilder)builder; + int index = 0; + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + indexedBuilder.cellByDirectIndex(index++, readDouble(buffer)); + expectCompositeEnd(buffer.currentToken()); + } + + private static double readDouble(TokenBuffer buffer) { + try { + return Double.valueOf(buffer.currentText()); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected a number but got '" + buffer.currentText()); + } + } + } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index f8ee23e86ba..69be397595e 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -52,6 +52,7 @@ import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.text.Utf8; import org.junit.After; import org.junit.Before; @@ -63,6 +64,7 @@ import org.mockito.internal.matchers.Contains; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -1294,6 +1296,24 @@ public class JsonReaderTestCase { } @Test + public void testParsingOfDenseTensorOnDenseForm() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[3])")); + builder.cell().label("x", 0).label("y", 0).value(2.0); + builder.cell().label("x", 0).label("y", 1).value(3.0); + builder.cell().label("x", 0).label("y", 2).value(4.0); + builder.cell().label("x", 1).label("y", 0).value(5.0); + builder.cell().label("x", 1).label("y", 1).value(6.0); + builder.cell().label("x", 1).label("y", 2).value(7.0); + Tensor expected = builder.build(); + + Tensor tensor = assertTensorField(expected, + createPutWithTensor(inputJson("{", + " 'values': [2.0, 3.0, 4.0, 5.0, 6.0, 7.0]", + "}"), "dense_tensor"), "dense_tensor"); + assertTrue(tensor instanceof IndexedTensor); // this matters for performance + } + + @Test public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() { assertSparseTensorField("{{x:a,y:b}:2.0}", createPutWithSparseTensor(inputJson("{", @@ -1689,11 +1709,14 @@ public class JsonReaderTestCase { return assertTensorField(expectedTensor, put, "sparse_tensor"); } private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) { - final Document doc = put.getDocument(); + return assertTensorField(Tensor.from(expectedTensor), put, tensorFieldName); + } + private static Tensor assertTensorField(Tensor expectedTensor, DocumentPut put, String tensorFieldName) { + Document doc = put.getDocument(); assertEquals("testtensor", doc.getId().getDocType()); assertEquals(TENSOR_DOC_ID, doc.getId().toString()); TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField(tensorFieldName)); - assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get()); + assertEquals(expectedTensor, fieldValue.getTensor().get()); return fieldValue.getTensor().get(); } diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index 714eb870b3e..9112a8b1712 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -100,16 +100,10 @@ TEST_FF("require that compiled evaluation passes all conformance tests", MyEvalT //----------------------------------------------------------------------------- -TEST("require that invalid function evaluates to a error") { +TEST("require that invalid function is tagged with error") { std::vector<vespalib::string> params({"x", "y", "z", "w"}); Function function = Function::parse(params, "x & y"); EXPECT_TRUE(function.has_error()); - InterpretedFunction ifun(SimpleTensorEngine::ref(), function, NodeTypes()); - InterpretedFunction::Context ctx(ifun); - SimpleParams my_params({1,2,3,4}); - const Value &result = ifun.eval(ctx, my_params); - EXPECT_TRUE(result.is_error()); - EXPECT_EQUAL(error_value, result.as_double()); } //----------------------------------------------------------------------------- diff --git a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp index 741b756e46f..7bca3d14c28 100644 --- a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp +++ b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp @@ -15,13 +15,12 @@ using namespace vespalib::eval::tensor_function; struct EvalCtx { const TensorEngine &engine; Stash stash; - ErrorValue error; std::vector<Value::UP> tensors; std::vector<Value::CREF> params; InterpretedFunction::UP ifun; std::unique_ptr<InterpretedFunction::Context> ictx; EvalCtx(const TensorEngine &engine_in) - : engine(engine_in), stash(), error(), tensors(), params(), ifun(), ictx() {} + : engine(engine_in), stash(), tensors(), params(), ifun(), ictx() {} ~EvalCtx() {} size_t add_tensor(Value::UP tensor) { size_t id = params.size(); diff --git a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp index 8180a7daef8..5dfde15a0ee 100644 --- a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp +++ b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp @@ -39,11 +39,10 @@ void verify_tensor(const TensorSpec &expect, ConstantValue::UP actual) { } void verify_invalid(ConstantValue::UP actual) { - EXPECT_EQUAL(actual->type(), ValueType::double_type()); - EXPECT_EQUAL(actual->value().as_double(), 0.0); + EXPECT_TRUE(actual->type().is_error()); } -TEST_F("require that invalid types loads an empty double", ConstantTensorLoader(SimpleTensorEngine::ref())) { +TEST_F("require that invalid types gives bad constant value", ConstantTensorLoader(SimpleTensorEngine::ref())) { TEST_DO(verify_invalid(f1.create(TEST_PATH("dense.json"), "invalid type spec"))); } diff --git a/eval/src/vespa/eval/eval/basic_nodes.cpp b/eval/src/vespa/eval/eval/basic_nodes.cpp index 85e00e76803..6138f9ac073 100644 --- a/eval/src/vespa/eval/eval/basic_nodes.cpp +++ b/eval/src/vespa/eval/eval/basic_nodes.cpp @@ -21,8 +21,8 @@ struct Frame { }; struct NoParams : LazyParams { - const Value &resolve(size_t, Stash &stash) const override { - return stash.create<ErrorValue>(); + const Value &resolve(size_t, Stash &) const override { + abort(); } }; diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index e362faadf46..208b2db4c3a 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -91,9 +91,7 @@ InterpretedFunction::eval(Context &ctx, const LazyParams ¶ms) const while (state.program_offset < _program.size()) { _program[state.program_offset++].perform(state); } - if (state.stack.size() != 1) { - state.stack.push_back(state.stash.create<ErrorValue>()); - } + assert(state.stack.size() == 1); return state.stack.back(); } diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index d84d9f53749..ebf065f32a1 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -142,8 +142,8 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const If &node) override { make_if(node); } - void visit(const Error &node) override { - make_const(node, ErrorValue::instance); + void visit(const Error &) override { + abort(); } void visit(const TensorMap &node) override { const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index c8c9a82d267..25fc98fc00a 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -40,22 +40,14 @@ const Value &to_value(std::unique_ptr<SimpleTensor> tensor, Stash &stash) { if (tensor->type().is_tensor()) { return *stash.create<Value::UP>(std::move(tensor)); } - if (tensor->type().is_double()) { - return stash.create<DoubleValue>(tensor->as_double()); - } - assert(tensor->type().is_error()); - return ErrorValue::instance; + return stash.create<DoubleValue>(tensor->as_double()); } Value::UP to_value(std::unique_ptr<SimpleTensor> tensor) { if (tensor->type().is_tensor()) { return tensor; } - if (tensor->type().is_double()) { - return std::make_unique<DoubleValue>(tensor->as_double()); - } - assert(tensor->type().is_error()); - return std::make_unique<ErrorValue>(); + return std::make_unique<DoubleValue>(tensor->as_double()); } } // namespace vespalib::eval::<unnamed> diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 8cbc7507592..7e512bb5bf1 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -298,7 +298,6 @@ struct TestContext { } void test_tensor_create_type() { - TEST_DO(verify_create_type("error")); TEST_DO(verify_create_type("double")); TEST_DO(verify_create_type("tensor(x{})")); TEST_DO(verify_create_type("tensor(x{},y{})")); diff --git a/eval/src/vespa/eval/eval/value.cpp b/eval/src/vespa/eval/eval/value.cpp index 4bfd758f9cd..3629a5ad698 100644 --- a/eval/src/vespa/eval/eval/value.cpp +++ b/eval/src/vespa/eval/eval/value.cpp @@ -6,9 +6,6 @@ namespace vespalib { namespace eval { -ValueType ErrorValue::_type = ValueType::error_type(); -const ErrorValue ErrorValue::instance; - ValueType DoubleValue::_type = ValueType::double_type(); } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index f14034968be..15df44efbac 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -20,7 +20,6 @@ constexpr double error_value = 31212.0; struct Value { typedef std::unique_ptr<Value> UP; typedef std::reference_wrapper<const Value> CREF; - virtual bool is_error() const { return false; } virtual bool is_double() const { return false; } virtual bool is_tensor() const { return false; } virtual double as_double() const { return 0.0; } @@ -30,17 +29,6 @@ struct Value { virtual ~Value() {} }; -class ErrorValue : public Value -{ -private: - static ValueType _type; -public: - static const ErrorValue instance; - bool is_error() const override { return true; } - double as_double() const override { return error_value; } - const ValueType &type() const override { return _type; } -}; - class DoubleValue : public Value { private: diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index afc8471bdb4..2005caa18ec 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -75,13 +75,17 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin ValueType value_type = ValueType::from_spec(type); if (value_type.is_error()) { LOG(warning, "invalid type specification: %s", type.c_str()); - return std::make_unique<SimpleConstantValue>(_engine.from_spec(TensorSpec("double"))); + return std::make_unique<BadConstantValue>(); } if (ends_with(path, ".tbf")) { vespalib::MappedFileInput file(path); vespalib::Memory content = file.get(); vespalib::nbostream stream(content.data, content.size); - return std::make_unique<SimpleConstantValue>(_engine.decode(stream)); + try { + return std::make_unique<SimpleConstantValue>(_engine.decode(stream)); + } catch (std::exception &) { + return std::make_unique<BadConstantValue>(); + } } Slime slime; decode_json(path, slime); @@ -99,7 +103,11 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin cells[i]["address"].traverse(extractor); spec.add(address, cells[i]["value"].asDouble()); } - return std::make_unique<SimpleConstantValue>(_engine.from_spec(spec)); + try { + return std::make_unique<SimpleConstantValue>(_engine.from_spec(spec)); + } catch (std::exception &) { + return std::make_unique<BadConstantValue>(); + } } } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value_cache/constant_value.h b/eval/src/vespa/eval/eval/value_cache/constant_value.h index ba7fe6fcf3d..a288ad70b53 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_value.h +++ b/eval/src/vespa/eval/eval/value_cache/constant_value.h @@ -30,6 +30,15 @@ public: const Value &value() const override { return *_value; } }; +class BadConstantValue : public ConstantValue { +private: + const ValueType _type; +public: + BadConstantValue() : _type(ValueType::error_type()) {} + const ValueType &type() const override { return _type; } + const Value &value() const override { abort(); } +}; + /** * An abstract factory of constant values. The typical use-case for * this will be to load constant values from file with a cache on top diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 2206cde49a9..a265ae5ae85 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -33,7 +33,6 @@ namespace vespalib::tensor { using eval::Aggr; using eval::Aggregator; using eval::DoubleValue; -using eval::ErrorValue; using eval::TensorFunction; using eval::TensorSpec; using eval::Value; @@ -83,7 +82,7 @@ const Value &to_default(const Value &value, Stash &stash) { const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { if (!tensor) { - return ErrorValue::instance; + return stash.create<DoubleValue>(eval::error_value); } if (tensor->type().is_tensor()) { return *stash.create<Value::UP>(std::move(tensor)); @@ -93,7 +92,7 @@ const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { Value::UP to_value(std::unique_ptr<Tensor> tensor) { if (!tensor) { - return std::make_unique<ErrorValue>(); + return std::make_unique<DoubleValue>(eval::error_value); } if (tensor->type().is_tensor()) { return tensor; @@ -171,7 +170,7 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const { ValueType type = ValueType::from_spec(spec.type()); if (type.is_error()) { - return std::make_unique<ErrorValue>(); + bad_spec(spec); } else if (type.is_double()) { double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; return std::make_unique<DoubleValue>(value); @@ -273,9 +272,7 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const const Value & DefaultTensorEngine::map(const Value &a, map_fun_t function, Stash &stash) const { - if (a.is_double()) { - return stash.create<DoubleValue>(function(a.as_double())); - } else if (auto tensor = a.as_tensor()) { + if (auto tensor = a.as_tensor()) { assert(&tensor->engine() == this); const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor); if (!tensor::Tensor::supported({my_a.type()})) { @@ -284,63 +281,49 @@ DefaultTensorEngine::map(const Value &a, map_fun_t function, Stash &stash) const CellFunctionFunAdapter cell_function(function); return to_value(my_a.apply(cell_function), stash); } else { - return ErrorValue::instance; + return stash.create<DoubleValue>(function(a.as_double())); } } const Value & DefaultTensorEngine::join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const { - if (a.is_double()) { - if (b.is_double()) { - return stash.create<DoubleValue>(function(a.as_double(), b.as_double())); - } else if (auto tensor_b = b.as_tensor()) { + if (auto tensor_a = a.as_tensor()) { + assert(&tensor_a->engine() == this); + const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor_a); + if (auto tensor_b = b.as_tensor()) { assert(&tensor_b->engine() == this); const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(*tensor_b); - if (!tensor::Tensor::supported({my_b.type()})) { + if (!tensor::Tensor::supported({my_a.type(), my_b.type()})) { return fallback_join(a, b, function, stash); } - CellFunctionBindLeftAdapter cell_function(function, a.as_double()); - return to_value(my_b.apply(cell_function), stash); + return to_value(my_a.join(function, my_b), stash); } else { - return ErrorValue::instance; - } - } else if (auto tensor_a = a.as_tensor()) { - assert(&tensor_a->engine() == this); - const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor_a); - if (b.is_double()) { if (!tensor::Tensor::supported({my_a.type()})) { return fallback_join(a, b, function, stash); } CellFunctionBindRightAdapter cell_function(function, b.as_double()); return to_value(my_a.apply(cell_function), stash); - } else if (auto tensor_b = b.as_tensor()) { + } + } else { + if (auto tensor_b = b.as_tensor()) { assert(&tensor_b->engine() == this); const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(*tensor_b); - if (!tensor::Tensor::supported({my_a.type(), my_b.type()})) { + if (!tensor::Tensor::supported({my_b.type()})) { return fallback_join(a, b, function, stash); } - return to_value(my_a.join(function, my_b), stash); + CellFunctionBindLeftAdapter cell_function(function, a.as_double()); + return to_value(my_b.apply(cell_function), stash); } else { - return ErrorValue::instance; + return stash.create<DoubleValue>(function(a.as_double(), b.as_double())); } - } else { - return ErrorValue::instance; } } const Value & DefaultTensorEngine::reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const { - if (a.is_double()) { - if (dimensions.empty()) { - Aggregator &aggregator = Aggregator::create(aggr, stash); - aggregator.first(a.as_double()); - return stash.create<DoubleValue>(aggregator.result()); - } else { - return ErrorValue::instance; - } - } else if (auto tensor = a.as_tensor()) { + if (auto tensor = a.as_tensor()) { assert(&tensor->engine() == this); const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(*tensor); if (!tensor::Tensor::supported({my_a.type()})) { @@ -360,7 +343,13 @@ DefaultTensorEngine::reduce(const Value &a, Aggr aggr, const std::vector<vespali return fallback_reduce(a, aggr, dimensions, stash); } } else { - return ErrorValue::instance; + if (dimensions.empty()) { + Aggregator &aggregator = Aggregator::create(aggr, stash); + aggregator.first(a.as_double()); + return stash.create<DoubleValue>(aggregator.result()); + } else { + return stash.create<DoubleValue>(eval::error_value); + } } } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java new file mode 100644 index 00000000000..cdacbe1656a --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java @@ -0,0 +1,40 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.operations; + +import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.List; + +/** + * Convert imported 'softmax' operation to the Vespa softmax ranking function. + * + * @author lesters + */ +public class Softmax extends IntermediateOperation { + + public Softmax(String modelName, String nodeName, List<IntermediateOperation> inputs) { + super(modelName, nodeName, inputs); + } + + @Override + protected OrderedTensorType lazyGetType() { + if ( ! allInputTypesPresent(1)) return null; + return inputs.get(0).type().get(); + } + + @Override + protected TensorFunction lazyGetFunction() { + if ( ! allInputFunctionsPresent(1)) return null; + + OrderedTensorType inputType = inputs.get(0).type().get(); + String dimension = inputType.dimensions().get(0).name(); + if (inputType.rank() == 2) { + dimension = inputType.dimensions().get(1).name(); // assumption: first dimension is batch dimension + } + + TensorFunction inputFunction = inputs.get(0).function().get(); + return new com.yahoo.tensor.functions.Softmax(inputFunction, dimension); + } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/GraphImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/GraphImporter.java index 1abbd0063a1..357794faee2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/GraphImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/GraphImporter.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.tensorflow; +import ai.vespa.rankingexpression.importer.operations.Softmax; import ai.vespa.rankingexpression.importer.operations.Sum; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import ai.vespa.rankingexpression.importer.IntermediateGraph; @@ -112,6 +113,7 @@ class GraphImporter { case "elu": return new Map(modelName, nodeName, inputs, ScalarFunctions.elu()); case "relu": return new Map(modelName, nodeName, inputs, ScalarFunctions.relu()); case "selu": return new Map(modelName, nodeName, inputs, ScalarFunctions.selu()); + case "softmax": return new Softmax(modelName, nodeName, inputs); // state ops case "variable": return new Constant(modelName, nodeName, nodeType); diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/BatchNormImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/BatchNormImportTestCase.java index 1b8d06bf964..e75c7fd4da3 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/BatchNormImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/BatchNormImportTestCase.java @@ -22,7 +22,7 @@ public class BatchNormImportTestCase { "src/test/models/tensorflow/batch_norm/saved"); ImportedModel.Signature signature = model.get().signature("serving_default"); - assertEquals("Has skipped outputs", + assertEquals("Should have no skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/DropoutImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/DropoutImportTestCase.java index 5e5c81ddcf1..b9d767774be 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/DropoutImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/DropoutImportTestCase.java @@ -29,7 +29,7 @@ public class DropoutImportTestCase { ImportedModel.Signature signature = model.get().signature("serving_default"); - Assert.assertEquals("Has skipped outputs", + Assert.assertEquals("Should have no skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); ImportedMlFunction function = signature.outputFunction("y", "y"); diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/MnistImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/MnistImportTestCase.java index 6b3e9207fad..c13ed84f701 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/MnistImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/MnistImportTestCase.java @@ -19,7 +19,7 @@ public class MnistImportTestCase { public void testMnistImport() { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/models/tensorflow/mnist/saved"); ImportedModel.Signature signature = model.get().signature("serving_default"); - Assert.assertEquals("Has skipped outputs", + Assert.assertEquals("Should have no skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); ImportedMlFunction output = signature.outputFunction("y", "y"); diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/SoftmaxImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/SoftmaxImportTestCase.java new file mode 100644 index 00000000000..525f915b252 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/SoftmaxImportTestCase.java @@ -0,0 +1,29 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.tensorflow; + +import ai.vespa.rankingexpression.importer.ImportedModel; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlFunction; +import org.junit.Assert; +import org.junit.Test; + + +import static org.junit.Assert.assertNotNull; + +/** + * @author lesters + */ +public class SoftmaxImportTestCase { + + @Test + public void testSoftmaxImport() { + TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/models/tensorflow/softmax/saved", 1, 5); + ImportedModel.Signature signature = model.get().signature("serving_default"); + Assert.assertEquals("Should have no skipped outputs", + 0, model.get().signature("serving_default").skippedOutputs().size()); + + ImportedMlFunction output = signature.outputFunction("y", "y"); + assertNotNull(output); + model.assertEqualResult("input", "output"); + } + +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/TestableTensorFlowModel.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/TestableTensorFlowModel.java index 4ff0c96d369..9d2f8cf0692 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/TestableTensorFlowModel.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/tensorflow/TestableTensorFlowModel.java @@ -33,14 +33,20 @@ public class TestableTensorFlowModel { private ImportedModel model; // Sizes of the input vector - private final int d0Size = 1; - private final int d1Size = 784; + private int d0Size = 1; + private int d1Size = 784; public TestableTensorFlowModel(String modelName, String modelDir) { tensorFlowModel = SavedModelBundle.load(modelDir, "serve"); model = new TensorFlowImporter().importModel(modelName, modelDir, tensorFlowModel); } + public TestableTensorFlowModel(String modelName, String modelDir, int d0Size, int d1Size) { + this(modelName, modelDir); + this.d0Size = d0Size; + this.d1Size = d1Size; + } + public ImportedModel get() { return model; } /** Compare that summing the tensors produce the same result to within some tolerance delta */ diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java index c7210e6710a..faa603d0ab0 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java @@ -31,11 +31,11 @@ public class VespaImportTestCase { assertEquals("tensor(x[3])", model.inputs().get("input2").toString()); assertEquals(2, model.smallConstants().size()); - assertEquals("tensor(x[3]):{{x:0}:0.5,{x:1}:1.5,{x:2}:2.5}", model.smallConstants().get("constant1")); + assertEquals("tensor(x[3]):[0.5, 1.5, 2.5]", model.smallConstants().get("constant1")); assertEquals("tensor():{3.0}", model.smallConstants().get("constant2")); assertEquals(1, model.largeConstants().size()); - assertEquals("tensor(x[3]):{{x:0}:0.5,{x:1}:1.5,{x:2}:2.5}", model.largeConstants().get("constant1asLarge")); + assertEquals("tensor(x[3]):[0.5, 1.5, 2.5]", model.largeConstants().get("constant1asLarge")); assertEquals(2, model.expressions().size()); assertEquals("reduce(reduce(input1 * input2, sum, name) * constant1, max, x) * constant2", diff --git a/model-integration/src/test/models/tensorflow/softmax/saved/saved_model.pbtxt b/model-integration/src/test/models/tensorflow/softmax/saved/saved_model.pbtxt new file mode 100644 index 00000000000..11435ce3fa1 --- /dev/null +++ b/model-integration/src/test/models/tensorflow/softmax/saved/saved_model.pbtxt @@ -0,0 +1,1999 @@ +saved_model_schema_version: 1 +meta_graphs { + meta_info_def { + stripped_op_list { + op { + name: "Add" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_BFLOAT16 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_STRING + } + } + } + } + op { + name: "Assign" + input_arg { + name: "ref" + type_attr: "T" + is_ref: true + } + input_arg { + name: "value" + type_attr: "T" + } + output_arg { + name: "output_ref" + type_attr: "T" + is_ref: true + } + attr { + name: "T" + type: "type" + } + attr { + name: "validate_shape" + type: "bool" + default_value { + b: true + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: true + } + } + allows_uninitialized_input: true + } + op { + name: "Const" + output_arg { + name: "output" + type_attr: "dtype" + } + attr { + name: "value" + type: "tensor" + } + attr { + name: "dtype" + type: "type" + } + } + op { + name: "Identity" + input_arg { + name: "input" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + } + op { + name: "MatMul" + input_arg { + name: "a" + type_attr: "T" + } + input_arg { + name: "b" + type_attr: "T" + } + output_arg { + name: "product" + type_attr: "T" + } + attr { + name: "transpose_a" + type: "bool" + default_value { + b: false + } + } + attr { + name: "transpose_b" + type: "bool" + default_value { + b: false + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_BFLOAT16 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + } + op { + name: "MergeV2Checkpoints" + input_arg { + name: "checkpoint_prefixes" + type: DT_STRING + } + input_arg { + name: "destination_prefix" + type: DT_STRING + } + attr { + name: "delete_old_dirs" + type: "bool" + default_value { + b: true + } + } + is_stateful: true + } + op { + name: "Mul" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_BFLOAT16 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + is_commutative: true + } + op { + name: "NoOp" + } + op { + name: "Pack" + input_arg { + name: "values" + type_attr: "T" + number_attr: "N" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "T" + type: "type" + } + attr { + name: "axis" + type: "int" + default_value { + i: 0 + } + } + } + op { + name: "Placeholder" + output_arg { + name: "output" + type_attr: "dtype" + } + attr { + name: "dtype" + type: "type" + } + attr { + name: "shape" + type: "shape" + default_value { + shape { + unknown_rank: true + } + } + } + } + op { + name: "RandomUniform" + input_arg { + name: "shape" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "dtype" + } + attr { + name: "seed" + type: "int" + default_value { + i: 0 + } + } + attr { + name: "seed2" + type: "int" + default_value { + i: 0 + } + } + attr { + name: "dtype" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + is_stateful: true + } + op { + name: "Relu" + input_arg { + name: "features" + type_attr: "T" + } + output_arg { + name: "activations" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_INT64 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + type: DT_QINT8 + } + } + } + } + op { + name: "RestoreV2" + input_arg { + name: "prefix" + type: DT_STRING + } + input_arg { + name: "tensor_names" + type: DT_STRING + } + input_arg { + name: "shape_and_slices" + type: DT_STRING + } + output_arg { + name: "tensors" + type_list_attr: "dtypes" + } + attr { + name: "dtypes" + type: "list(type)" + has_minimum: true + minimum: 1 + } + is_stateful: true + } + op { + name: "SaveV2" + input_arg { + name: "prefix" + type: DT_STRING + } + input_arg { + name: "tensor_names" + type: DT_STRING + } + input_arg { + name: "shape_and_slices" + type: DT_STRING + } + input_arg { + name: "tensors" + type_list_attr: "dtypes" + } + attr { + name: "dtypes" + type: "list(type)" + has_minimum: true + minimum: 1 + } + is_stateful: true + } + op { + name: "ShardedFilename" + input_arg { + name: "basename" + type: DT_STRING + } + input_arg { + name: "shard" + type: DT_INT32 + } + input_arg { + name: "num_shards" + type: DT_INT32 + } + output_arg { + name: "filename" + type: DT_STRING + } + } + op { + name: "Softmax" + input_arg { + name: "logits" + type_attr: "T" + } + output_arg { + name: "softmax" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + } + op { + name: "StringJoin" + input_arg { + name: "inputs" + type: DT_STRING + number_attr: "N" + } + output_arg { + name: "output" + type: DT_STRING + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "separator" + type: "string" + default_value { + s: "" + } + } + } + op { + name: "Sub" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_BFLOAT16 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + } + op { + name: "VariableV2" + output_arg { + name: "ref" + type_attr: "dtype" + is_ref: true + } + attr { + name: "shape" + type: "shape" + } + attr { + name: "dtype" + type: "type" + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + } + is_stateful: true + } + } + tags: "serve" + tensorflow_version: "1.12.0" + tensorflow_git_version: "v1.12.0-rc2-3-ga6d8ffae09" + } + graph_def { + node { + name: "input" + op: "Placeholder" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 5 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: -1 + } + dim { + size: 5 + } + } + } + } + } + node { + name: "random_uniform/shape" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\005\000\000\000\003\000\000\000" + } + } + } + } + node { + name: "random_uniform/min" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 0.0 + } + } + } + } + node { + name: "random_uniform/max" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 1.0 + } + } + } + } + node { + name: "random_uniform/RandomUniform" + op: "RandomUniform" + input: "random_uniform/shape" + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "seed" + value { + i: 0 + } + } + attr { + key: "seed2" + value { + i: 0 + } + } + } + node { + name: "random_uniform/sub" + op: "Sub" + input: "random_uniform/max" + input: "random_uniform/min" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node { + name: "random_uniform/mul" + op: "Mul" + input: "random_uniform/RandomUniform" + input: "random_uniform/sub" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "random_uniform" + op: "Add" + input: "random_uniform/mul" + input: "random_uniform/min" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "weights" + op: "VariableV2" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + attr { + key: "container" + value { + s: "" + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + attr { + key: "shared_name" + value { + s: "" + } + } + } + node { + name: "weights/Assign" + op: "Assign" + input: "weights" + input: "random_uniform" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@weights" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + attr { + key: "use_locking" + value { + b: true + } + } + attr { + key: "validate_shape" + value { + b: true + } + } + } + node { + name: "weights/read" + op: "Identity" + input: "weights" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@weights" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "random_uniform_1/shape" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 3 + } + } + } + } + node { + name: "random_uniform_1/min" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 0.0 + } + } + } + } + node { + name: "random_uniform_1/max" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 1.0 + } + } + } + } + node { + name: "random_uniform_1/RandomUniform" + op: "RandomUniform" + input: "random_uniform_1/shape" + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "seed" + value { + i: 0 + } + } + attr { + key: "seed2" + value { + i: 0 + } + } + } + node { + name: "random_uniform_1/sub" + op: "Sub" + input: "random_uniform_1/max" + input: "random_uniform_1/min" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node { + name: "random_uniform_1/mul" + op: "Mul" + input: "random_uniform_1/RandomUniform" + input: "random_uniform_1/sub" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + } + node { + name: "random_uniform_1" + op: "Add" + input: "random_uniform_1/mul" + input: "random_uniform_1/min" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + } + node { + name: "bias" + op: "VariableV2" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + attr { + key: "container" + value { + s: "" + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 3 + } + } + } + } + attr { + key: "shared_name" + value { + s: "" + } + } + } + node { + name: "bias/Assign" + op: "Assign" + input: "bias" + input: "random_uniform_1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@bias" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + attr { + key: "use_locking" + value { + b: true + } + } + attr { + key: "validate_shape" + value { + b: true + } + } + } + node { + name: "bias/read" + op: "Identity" + input: "bias" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@bias" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + } + node { + name: "MatMul" + op: "MatMul" + input: "input" + input: "weights/read" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 3 + } + } + } + } + } + attr { + key: "transpose_a" + value { + b: false + } + } + attr { + key: "transpose_b" + value { + b: false + } + } + } + node { + name: "add" + op: "Add" + input: "MatMul" + input: "bias/read" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "Relu" + op: "Relu" + input: "add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "output" + op: "Softmax" + input: "Relu" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 3 + } + } + } + } + } + } + node { + name: "init" + op: "NoOp" + input: "^bias/Assign" + input: "^weights/Assign" + } + node { + name: "save/Const" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "model" + } + } + } + } + node { + name: "save/StringJoin/inputs_1" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "_temp_6341ee658682497a95c4fd82a2c87cc6/part" + } + } + } + } + node { + name: "save/StringJoin" + op: "StringJoin" + input: "save/Const" + input: "save/StringJoin/inputs_1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "separator" + value { + s: "" + } + } + } + node { + name: "save/num_shards" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } + } + node { + name: "save/ShardedFilename/shard" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } + } + node { + name: "save/ShardedFilename" + op: "ShardedFilename" + input: "save/StringJoin" + input: "save/ShardedFilename/shard" + input: "save/num_shards" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node { + name: "save/SaveV2/tensor_names" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "bias" + string_val: "weights" + } + } + } + } + node { + name: "save/SaveV2/shape_and_slices" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "" + string_val: "" + } + } + } + } + node { + name: "save/SaveV2" + op: "SaveV2" + input: "save/ShardedFilename" + input: "save/SaveV2/tensor_names" + input: "save/SaveV2/shape_and_slices" + input: "bias" + input: "weights" + device: "/device:CPU:0" + attr { + key: "dtypes" + value { + list { + type: DT_FLOAT + type: DT_FLOAT + } + } + } + } + node { + name: "save/control_dependency" + op: "Identity" + input: "save/ShardedFilename" + input: "^save/SaveV2" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_class" + value { + list { + s: "loc:@save/ShardedFilename" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node { + name: "save/MergeV2Checkpoints/checkpoint_prefixes" + op: "Pack" + input: "save/ShardedFilename" + input: "^save/control_dependency" + device: "/device:CPU:0" + attr { + key: "N" + value { + i: 1 + } + } + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + attr { + key: "axis" + value { + i: 0 + } + } + } + node { + name: "save/MergeV2Checkpoints" + op: "MergeV2Checkpoints" + input: "save/MergeV2Checkpoints/checkpoint_prefixes" + input: "save/Const" + device: "/device:CPU:0" + attr { + key: "delete_old_dirs" + value { + b: true + } + } + } + node { + name: "save/Identity" + op: "Identity" + input: "save/Const" + input: "^save/MergeV2Checkpoints" + input: "^save/control_dependency" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node { + name: "save/RestoreV2/tensor_names" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "bias" + string_val: "weights" + } + } + } + } + node { + name: "save/RestoreV2/shape_and_slices" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "" + string_val: "" + } + } + } + } + node { + name: "save/RestoreV2" + op: "RestoreV2" + input: "save/Const" + input: "save/RestoreV2/tensor_names" + input: "save/RestoreV2/shape_and_slices" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + shape { + unknown_rank: true + } + } + } + } + attr { + key: "dtypes" + value { + list { + type: DT_FLOAT + type: DT_FLOAT + } + } + } + } + node { + name: "save/Assign" + op: "Assign" + input: "bias" + input: "save/RestoreV2" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@bias" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + } + } + } + } + attr { + key: "use_locking" + value { + b: true + } + } + attr { + key: "validate_shape" + value { + b: true + } + } + } + node { + name: "save/Assign_1" + op: "Assign" + input: "weights" + input: "save/RestoreV2:1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@weights" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + dim { + size: 3 + } + } + } + } + } + attr { + key: "use_locking" + value { + b: true + } + } + attr { + key: "validate_shape" + value { + b: true + } + } + } + node { + name: "save/restore_shard" + op: "NoOp" + input: "^save/Assign" + input: "^save/Assign_1" + } + node { + name: "save/restore_all" + op: "NoOp" + input: "^save/restore_shard" + } + versions { + producer: 27 + } + } + saver_def { + filename_tensor_name: "save/Const:0" + save_tensor_name: "save/Identity:0" + restore_op_name: "save/restore_all" + max_to_keep: 5 + sharded: true + keep_checkpoint_every_n_hours: 10000.0 + version: V2 + } + collection_def { + key: "trainable_variables" + value { + bytes_list { + value: "\n\tweights:0\022\016weights/Assign\032\016weights/read:02\020random_uniform:08\001" + value: "\n\006bias:0\022\013bias/Assign\032\013bias/read:02\022random_uniform_1:08\001" + } + } + } + collection_def { + key: "variables" + value { + bytes_list { + value: "\n\tweights:0\022\016weights/Assign\032\016weights/read:02\020random_uniform:08\001" + value: "\n\006bias:0\022\013bias/Assign\032\013bias/read:02\022random_uniform_1:08\001" + } + } + } + signature_def { + key: "serving_default" + value { + inputs { + key: "x" + value { + name: "input:0" + dtype: DT_FLOAT + tensor_shape { + dim { + size: -1 + } + dim { + size: 5 + } + } + } + } + outputs { + key: "y" + value { + name: "output:0" + dtype: DT_FLOAT + tensor_shape { + dim { + size: -1 + } + dim { + size: 3 + } + } + } + } + method_name: "tensorflow/serving/predict" + } + } +} diff --git a/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.data-00000-of-00001 b/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.data-00000-of-00001 Binary files differnew file mode 100644 index 00000000000..a9edaf376d0 --- /dev/null +++ b/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.data-00000-of-00001 diff --git a/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.index b/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.index Binary files differnew file mode 100644 index 00000000000..0ae49491ce6 --- /dev/null +++ b/model-integration/src/test/models/tensorflow/softmax/saved/variables/variables.index diff --git a/model-integration/src/test/models/tensorflow/softmax/softmax.py b/model-integration/src/test/models/tensorflow/softmax/softmax.py new file mode 100644 index 00000000000..aab9956f914 --- /dev/null +++ b/model-integration/src/test/models/tensorflow/softmax/softmax.py @@ -0,0 +1,29 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import numpy as np +import tensorflow as tf + +# Creates simple random neural network that has softmax on output. No training. + +n_inputs = 5 +n_outputs = 3 + +input = tf.placeholder(tf.float32, shape=(None, n_inputs), name="input") +W = tf.Variable(tf.random.uniform([n_inputs, n_outputs]), name="weights") +b = tf.Variable(tf.random.uniform([n_outputs]), name="bias") +Z = tf.matmul(input, W) + b +hidden_layer = tf.nn.relu(Z) +output_layer = tf.nn.softmax(hidden_layer, name="output") + +init = tf.global_variables_initializer() + +with tf.Session() as sess: + init.run() + export_path = "saved" + builder = tf.saved_model.builder.SavedModelBuilder(export_path) + signature = tf.saved_model.signature_def_utils.predict_signature_def(inputs = {'x':input}, outputs = {'y':output_layer}) + builder.add_meta_graph_and_variables(sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map={'serving_default':signature}) + builder.save(as_text=True) + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 73c86fc8de1..59873d7956e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -116,16 +116,18 @@ public class ConfigServerApiImpl implements ConfigServerApi { if (configServers.size() == 1) break; // Failure to communicate with a config server is not abnormal during upgrades - if (e.getMessage().contains("(Connection refused)")) { - logger.info("Connection refused to " + configServer + " (upgrading?), will try next"); + if (ConnectionException.isKnownConnectionException(e)) { + logger.info("Failed to connect to " + configServer + " (upgrading?), will try next: " + e.getMessage()); } else { logger.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); } } } - throw HttpException.handleException( - "All requests against the config servers (" + configServers + ") failed, last as follows:", lastException); + String prefix = configServers.size() == 1 ? + "Request against " + configServers.get(0) + " failed: " : + "All requests against the config servers (" + configServers + ") failed, last as follows: "; + throw ConnectionException.handleException(prefix, lastException); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java new file mode 100644 index 00000000000..7e860bfb66b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java @@ -0,0 +1,43 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import org.apache.http.NoHttpResponseException; + +import java.io.EOFException; +import java.net.SocketException; +import java.net.SocketTimeoutException; + +/** + * @author freva + */ +@SuppressWarnings("serial") +public class ConnectionException extends ConvergenceException { + + private ConnectionException(String message) { + super(message); + } + + /** + * Returns {@link ConnectionException} if the given Throwable is of a known and well understood error or + * a RuntimeException with the given exception as cause otherwise. + */ + public static RuntimeException handleException(String prefix, Throwable t) { + if (isKnownConnectionException(t)) + return new ConnectionException(prefix + t.getMessage()); + + return new RuntimeException(prefix, t); + } + + static boolean isKnownConnectionException(Throwable t) { + for (; t != null; t = t.getCause()) { + if (t instanceof SocketException || + t instanceof SocketTimeoutException || + t instanceof NoHttpResponseException || + t instanceof EOFException) + return true; + } + + return false; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java index 3825107bfa6..a9493d4606e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java @@ -2,12 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import org.apache.http.NoHttpResponseException; import javax.ws.rs.core.Response; -import java.io.EOFException; -import java.net.SocketException; -import java.net.SocketTimeoutException; /** * @author hakonhall @@ -66,22 +62,6 @@ public class HttpException extends ConvergenceException { throw new HttpException(status, message, true); } - /** - * Returns {@link HttpException} if the given Throwable is of a known and well understood error or - * a RuntimeException with the given exception as cause otherwise. - */ - public static RuntimeException handleException(String prefix, Throwable t) { - for (; t != null; t = t.getCause()) { - if (t instanceof SocketException || - t instanceof SocketTimeoutException || - t instanceof NoHttpResponseException || - t instanceof EOFException) - return new HttpException(prefix + t.getMessage()); - } - - return new RuntimeException(prefix, t); - } - public static class NotFoundException extends HttpException { public NotFoundException(String message) { super(Response.Status.NOT_FOUND, message, false); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java index bb16e2bae63..22633f67463 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java @@ -19,19 +19,19 @@ public class NodeMembership { this.retired = retired; } - public String getClusterType() { + public String clusterType() { return clusterType; } - public String getClusterId() { + public String clusterId() { return clusterId; } - public String getGroup() { + public String group() { return group; } - public int getIndex() { + public int index() { return index; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java index c1900316bb9..c41e050d534 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java @@ -17,15 +17,15 @@ public class NodeOwner { this.instance = instance; } - public String getTenant() { + public String tenant() { return tenant; } - public String getApplication() { + public String application() { return application; } - public String getInstance() { + public String instance() { return instance; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 52d6f16dd78..6fb6d44bd6f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -17,7 +17,7 @@ import java.util.Set; public class NodeSpec { private final String hostname; private final NodeState state; - private final NodeType nodeType; + private final NodeType type; private final String flavor; private final String canonicalFlavor; @@ -25,7 +25,7 @@ public class NodeSpec { private final Optional<DockerImage> currentDockerImage; private final Optional<Version> wantedVespaVersion; - private final Optional<Version> vespaVersion; + private final Optional<Version> currentVespaVersion; private final Optional<Version> wantedOsVersion; private final Optional<Version> currentOsVersion; @@ -46,9 +46,9 @@ public class NodeSpec { private final Optional<NodeOwner> owner; private final Optional<NodeMembership> membership; - private final double minCpuCores; - private final double minMainMemoryAvailableGb; - private final double minDiskAvailableGb; + private final double vcpus; + private final double memoryGb; + private final double diskGb; private final boolean fastDisk; private final double bandwidth; @@ -64,11 +64,11 @@ public class NodeSpec { Optional<DockerImage> wantedDockerImage, Optional<DockerImage> currentDockerImage, NodeState state, - NodeType nodeType, + NodeType type, String flavor, String canonicalFlavor, Optional<Version> wantedVespaVersion, - Optional<Version> vespaVersion, + Optional<Version> currentVespaVersion, Optional<Version> wantedOsVersion, Optional<Version> currentOsVersion, Optional<Boolean> allowedToBeDown, @@ -82,9 +82,9 @@ public class NodeSpec { Optional<Instant> wantedFirmwareCheck, Optional<Instant> currentFirmwareCheck, Optional<String> modelName, - double minCpuCores, - double minMainMemoryAvailableGb, - double minDiskAvailableGb, + double vcpus, + double memoryGb, + double diskGb, boolean fastDisk, double bandwidth, Set<String> ipAddresses, @@ -102,12 +102,12 @@ public class NodeSpec { this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage); this.currentDockerImage = Objects.requireNonNull(currentDockerImage); this.state = Objects.requireNonNull(state); - this.nodeType = Objects.requireNonNull(nodeType); + this.type = Objects.requireNonNull(type); this.flavor = Objects.requireNonNull(flavor); this.canonicalFlavor = canonicalFlavor; this.modelName = modelName; this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion); - this.vespaVersion = Objects.requireNonNull(vespaVersion); + this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion); this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion); this.currentOsVersion = Objects.requireNonNull(currentOsVersion); this.allowedToBeDown = Objects.requireNonNull(allowedToBeDown); @@ -120,9 +120,9 @@ public class NodeSpec { this.currentRebootGeneration = currentRebootGeneration; this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck); this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck); - this.minCpuCores = minCpuCores; - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; - this.minDiskAvailableGb = minDiskAvailableGb; + this.vcpus = vcpus; + this.memoryGb = memoryGb; + this.diskGb = diskGb; this.fastDisk = fastDisk; this.bandwidth = bandwidth; this.ipAddresses = Objects.requireNonNull(ipAddresses); @@ -131,125 +131,125 @@ public class NodeSpec { this.parentHostname = Objects.requireNonNull(parentHostname); } - public String getHostname() { + public String hostname() { return hostname; } - public NodeState getState() { + public NodeState state() { return state; } - public NodeType getNodeType() { - return nodeType; + public NodeType type() { + return type; } - public String getFlavor() { + public String flavor() { return flavor; } - public String getCanonicalFlavor() { + public String canonicalFlavor() { return canonicalFlavor; } - public Optional<DockerImage> getWantedDockerImage() { + public Optional<DockerImage> wantedDockerImage() { return wantedDockerImage; } - public Optional<DockerImage> getCurrentDockerImage() { + public Optional<DockerImage> currentDockerImage() { return currentDockerImage; } - public Optional<Version> getWantedVespaVersion() { + public Optional<Version> wantedVespaVersion() { return wantedVespaVersion; } - public Optional<Version> getVespaVersion() { - return vespaVersion; + public Optional<Version> currentVespaVersion() { + return currentVespaVersion; } - public Optional<Version> getCurrentOsVersion() { + public Optional<Version> currentOsVersion() { return currentOsVersion; } - public Optional<Version> getWantedOsVersion() { + public Optional<Version> wantedOsVersion() { return wantedOsVersion; } - public Optional<Long> getWantedRestartGeneration() { + public Optional<Long> wantedRestartGeneration() { return wantedRestartGeneration; } - public Optional<Long> getCurrentRestartGeneration() { + public Optional<Long> currentRestartGeneration() { return currentRestartGeneration; } - public long getWantedRebootGeneration() { + public long wantedRebootGeneration() { return wantedRebootGeneration; } - public long getCurrentRebootGeneration() { + public long currentRebootGeneration() { return currentRebootGeneration; } - public Optional<Instant> getWantedFirmwareCheck() { + public Optional<Instant> wantedFirmwareCheck() { return wantedFirmwareCheck; } - public Optional<Instant> getCurrentFirmwareCheck() { + public Optional<Instant> currentFirmwareCheck() { return currentFirmwareCheck; } - public Optional<String> getModelName() { + public Optional<String> modelName() { return modelName; } - public Optional<Boolean> getAllowedToBeDown() { + public Optional<Boolean> allowedToBeDown() { return allowedToBeDown; } - public Optional<Boolean> getWantToDeprovision() { + public Optional<Boolean> wantToDeprovision() { return wantToDeprovision; } - public Optional<NodeOwner> getOwner() { + public Optional<NodeOwner> owner() { return owner; } - public Optional<NodeMembership> getMembership() { + public Optional<NodeMembership> membership() { return membership; } - public double getMinCpuCores() { - return minCpuCores; + public double vcpus() { + return vcpus; } - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; + public double memoryGb() { + return memoryGb; } - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; + public double diskGb() { + return diskGb; } public boolean isFastDisk() { return fastDisk; } - public double getBandwidth() { + public double bandwidth() { return bandwidth; } - public Set<String> getIpAddresses() { + public Set<String> ipAddresses() { return ipAddresses; } - public Set<String> getAdditionalIpAddresses() { + public Set<String> additionalIpAddresses() { return additionalIpAddresses; } - public NodeReports getReports() { return reports; } + public NodeReports reports() { return reports; } - public Optional<String> getParentHostname() { + public Optional<String> parentHostname() { return parentHostname; } @@ -264,11 +264,11 @@ public class NodeSpec { Objects.equals(wantedDockerImage, that.wantedDockerImage) && Objects.equals(currentDockerImage, that.currentDockerImage) && Objects.equals(state, that.state) && - Objects.equals(nodeType, that.nodeType) && + Objects.equals(type, that.type) && Objects.equals(flavor, that.flavor) && Objects.equals(canonicalFlavor, that.canonicalFlavor) && Objects.equals(wantedVespaVersion, that.wantedVespaVersion) && - Objects.equals(vespaVersion, that.vespaVersion) && + Objects.equals(currentVespaVersion, that.currentVespaVersion) && Objects.equals(wantedOsVersion, that.wantedOsVersion) && Objects.equals(currentOsVersion, that.currentOsVersion) && Objects.equals(allowedToBeDown, that.allowedToBeDown) && @@ -281,9 +281,9 @@ public class NodeSpec { Objects.equals(currentRebootGeneration, that.currentRebootGeneration) && Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) && Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) && - Objects.equals(minCpuCores, that.minCpuCores) && - Objects.equals(minMainMemoryAvailableGb, that.minMainMemoryAvailableGb) && - Objects.equals(minDiskAvailableGb, that.minDiskAvailableGb) && + Objects.equals(vcpus, that.vcpus) && + Objects.equals(memoryGb, that.memoryGb) && + Objects.equals(diskGb, that.diskGb) && Objects.equals(fastDisk, that.fastDisk) && Objects.equals(bandwidth, that.bandwidth) && Objects.equals(ipAddresses, that.ipAddresses) && @@ -299,11 +299,11 @@ public class NodeSpec { wantedDockerImage, currentDockerImage, state, - nodeType, + type, flavor, canonicalFlavor, wantedVespaVersion, - vespaVersion, + currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, @@ -316,9 +316,9 @@ public class NodeSpec { currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, - minCpuCores, - minMainMemoryAvailableGb, - minDiskAvailableGb, + vcpus, + memoryGb, + diskGb, fastDisk, bandwidth, ipAddresses, @@ -334,26 +334,26 @@ public class NodeSpec { + " wantedDockerImage=" + wantedDockerImage + " currentDockerImage=" + currentDockerImage + " state=" + state - + " nodeType=" + nodeType + + " type=" + type + " flavor=" + flavor + " canonicalFlavor=" + canonicalFlavor + " wantedVespaVersion=" + wantedVespaVersion - + " vespaVersion=" + vespaVersion + + " currentVespaVersion=" + currentVespaVersion + " wantedOsVersion=" + wantedOsVersion + " currentOsVersion=" + currentOsVersion + " allowedToBeDown=" + allowedToBeDown + " wantToDeprovision=" + wantToDeprovision + " owner=" + owner + " membership=" + membership - + " minCpuCores=" + minCpuCores + + " vcpus=" + vcpus + " wantedRestartGeneration=" + wantedRestartGeneration + " currentRestartGeneration=" + currentRestartGeneration + " wantedRebootGeneration=" + wantedRebootGeneration + " currentRebootGeneration=" + currentRebootGeneration + " wantedFirmwareCheck=" + wantedFirmwareCheck + " currentFirmwareCheck=" + currentFirmwareCheck - + " minMainMemoryAvailableGb=" + minMainMemoryAvailableGb - + " minDiskAvailableGb=" + minDiskAvailableGb + + " memoryGb=" + memoryGb + + " diskGb=" + diskGb + " fastDisk=" + fastDisk + " bandwidth=" + bandwidth + " ipAddresses=" + ipAddresses @@ -365,14 +365,14 @@ public class NodeSpec { public static class Builder { private String hostname; - private Optional<DockerImage> wantedDockerImage = Optional.empty(); - private Optional<DockerImage> currentDockerImage = Optional.empty(); private NodeState state; - private NodeType nodeType; + private NodeType type; private String flavor; private String canonicalFlavor; + private Optional<DockerImage> wantedDockerImage = Optional.empty(); + private Optional<DockerImage> currentDockerImage = Optional.empty(); private Optional<Version> wantedVespaVersion = Optional.empty(); - private Optional<Version> vespaVersion = Optional.empty(); + private Optional<Version> currentVespaVersion = Optional.empty(); private Optional<Version> wantedOsVersion = Optional.empty(); private Optional<Version> currentOsVersion = Optional.empty(); private Optional<Boolean> allowedToBeDown = Optional.empty(); @@ -386,10 +386,10 @@ public class NodeSpec { private Optional<Instant> wantedFirmwareCheck = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<String> modelName = Optional.empty(); - private double minCpuCores; - private double minMainMemoryAvailableGb; - private double minDiskAvailableGb; - private boolean fastDisk = false; + private double vcpus; + private double memoryGb; + private double diskGb; + private boolean fastDisk; private double bandwidth; private Set<String> ipAddresses = Set.of(); private Set<String> additionalIpAddresses = Set.of(); @@ -401,12 +401,12 @@ public class NodeSpec { public Builder(NodeSpec node) { hostname(node.hostname); state(node.state); - nodeType(node.nodeType); + type(node.type); flavor(node.flavor); canonicalFlavor(node.canonicalFlavor); - minCpuCores(node.minCpuCores); - minMainMemoryAvailableGb(node.minMainMemoryAvailableGb); - minDiskAvailableGb(node.minDiskAvailableGb); + vcpus(node.vcpus); + memoryGb(node.memoryGb); + diskGb(node.diskGb); fastDisk(node.fastDisk); bandwidth(node.bandwidth); ipAddresses(node.ipAddresses); @@ -418,7 +418,7 @@ public class NodeSpec { node.wantedDockerImage.ifPresent(this::wantedDockerImage); node.currentDockerImage.ifPresent(this::currentDockerImage); node.wantedVespaVersion.ifPresent(this::wantedVespaVersion); - node.vespaVersion.ifPresent(this::vespaVersion); + node.currentVespaVersion.ifPresent(this::currentVespaVersion); node.wantedOsVersion.ifPresent(this::wantedOsVersion); node.currentOsVersion.ifPresent(this::currentOsVersion); node.allowedToBeDown.ifPresent(this::allowedToBeDown); @@ -452,8 +452,8 @@ public class NodeSpec { return this; } - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; + public Builder type(NodeType nodeType) { + this.type = nodeType; return this; } @@ -472,8 +472,8 @@ public class NodeSpec { return this; } - public Builder vespaVersion(Version vespaVersion) { - this.vespaVersion = Optional.of(vespaVersion); + public Builder currentVespaVersion(Version vespaVersion) { + this.currentVespaVersion = Optional.of(vespaVersion); return this; } @@ -537,18 +537,18 @@ public class NodeSpec { return this; } - public Builder minCpuCores(double minCpuCores) { - this.minCpuCores = minCpuCores; + public Builder vcpus(double minCpuCores) { + this.vcpus = minCpuCores; return this; } - public Builder minMainMemoryAvailableGb(double minMainMemoryAvailableGb) { - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; + public Builder memoryGb(double minMainMemoryAvailableGb) { + this.memoryGb = minMainMemoryAvailableGb; return this; } - public Builder minDiskAvailableGb(double minDiskAvailableGb) { - this.minDiskAvailableGb = minDiskAvailableGb; + public Builder diskGb(double minDiskAvailableGb) { + this.diskGb = minDiskAvailableGb; return this; } @@ -603,127 +603,127 @@ public class NodeSpec { return this; } - public String getHostname() { + public String hostname() { return hostname; } - public Optional<DockerImage> getWantedDockerImage() { + public Optional<DockerImage> wantedDockerImage() { return wantedDockerImage; } - public Optional<DockerImage> getCurrentDockerImage() { + public Optional<DockerImage> currentDockerImage() { return currentDockerImage; } - public NodeState getState() { + public NodeState state() { return state; } - public NodeType getNodeType() { - return nodeType; + public NodeType type() { + return type; } - public String getFlavor() { + public String flavor() { return flavor; } - public String getCanonicalFlavor() { + public String canonicalFlavor() { return canonicalFlavor; } - public Optional<Version> getWantedVespaVersion() { + public Optional<Version> wantedVespaVersion() { return wantedVespaVersion; } - public Optional<Version> getVespaVersion() { - return vespaVersion; + public Optional<Version> currentVespaVersion() { + return currentVespaVersion; } - public Optional<Version> getWantedOsVersion() { + public Optional<Version> wantedOsVersion() { return wantedOsVersion; } - public Optional<Version> getCurrentOsVersion() { + public Optional<Version> currentOsVersion() { return currentOsVersion; } - public Optional<Boolean> getAllowedToBeDown() { + public Optional<Boolean> allowedToBeDown() { return allowedToBeDown; } - public Optional<Boolean> getWantToDeprovision() { + public Optional<Boolean> wantToDeprovision() { return wantToDeprovision; } - public Optional<NodeOwner> getOwner() { + public Optional<NodeOwner> owner() { return owner; } - public Optional<NodeMembership> getMembership() { + public Optional<NodeMembership> membership() { return membership; } - public Optional<Long> getWantedRestartGeneration() { + public Optional<Long> wantedRestartGeneration() { return wantedRestartGeneration; } - public Optional<Long> getCurrentRestartGeneration() { + public Optional<Long> currentRestartGeneration() { return currentRestartGeneration; } - public long getWantedRebootGeneration() { + public long wantedRebootGeneration() { return wantedRebootGeneration; } - public long getCurrentRebootGeneration() { + public long currentRebootGeneration() { return currentRebootGeneration; } - public double getMinCpuCores() { - return minCpuCores; + public double vcpus() { + return vcpus; } - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; + public double memoryGb() { + return memoryGb; } - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; + public double diskGb() { + return diskGb; } public boolean isFastDisk() { return fastDisk; } - public double getBandwidth() { + public double bandwidth() { return bandwidth; } - public Set<String> getIpAddresses() { + public Set<String> ipAddresses() { return ipAddresses; } - public Set<String> getAdditionalIpAddresses() { + public Set<String> additionalIpAddresses() { return additionalIpAddresses; } - public NodeReports getReports() { + public NodeReports reports() { return reports; } - public Optional<String> getParentHostname() { + public Optional<String> parentHostname() { return parentHostname; } public NodeSpec build() { - return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, nodeType, + return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor, canonicalFlavor, - wantedVespaVersion, vespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, wantToDeprovision, + wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, wantToDeprovision, owner, membership, wantedRestartGeneration, currentRestartGeneration, wantedRebootGeneration, currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, modelName, - minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb, + vcpus, memoryGb, diskGb, fastDisk, bandwidth, ipAddresses, additionalIpAddresses, reports, parentHostname); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java index 6124e1bdc0e..353abd64778 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.orchestrator.restapi.HostApi; import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; @@ -41,6 +43,8 @@ public class OrchestratorImpl implements Orchestrator { throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found"); } catch (HttpException e) { throw new OrchestratorException("Failed to suspend " + hostName + ": " + e.toString()); + } catch (ConnectionException e) { + throw new ConvergenceException("Failed to suspend " + hostName + ": " + e.getMessage()); } catch (RuntimeException e) { throw new RuntimeException("Got error on suspend", e); } @@ -60,6 +64,8 @@ public class OrchestratorImpl implements Orchestrator { batchOperationResult = configServerApi.put(url, Optional.empty(), BatchOperationResult.class); } catch (HttpException e) { throw new OrchestratorException("Failed to batch suspend for " + parentHostName + ": " + e.toString()); + } catch (ConnectionException e) { + throw new ConvergenceException("Failed to batch suspend for " + parentHostName + ": " + e.getMessage()); } catch (RuntimeException e) { throw new RuntimeException("Got error on batch suspend for " + parentHostName + ", with nodes " + hostNames, e); } @@ -78,7 +84,9 @@ public class OrchestratorImpl implements Orchestrator { } catch (HttpException.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found"); } catch (HttpException e) { - throw new OrchestratorException("Failed to suspend " + hostName + ": " + e.toString()); + throw new OrchestratorException("Failed to resume " + hostName + ": " + e.toString()); + } catch (ConnectionException e) { + throw new ConvergenceException("Failed to resume " + hostName + ": " + e.getMessage()); } catch (RuntimeException e) { throw new RuntimeException("Got error on resume", e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java index 2fe8d4b4792..e99a107cfe1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.state; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; @@ -20,7 +21,7 @@ public class StateImpl implements State { try { HealthResponse response = configServerApi.get("/state/v1/health", HealthResponse.class); return HealthCode.fromString(response.status.code); - } catch (HttpException e) { + } catch (ConnectionException | HttpException e) { return HealthCode.DOWN; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 954ba25895a..1a993b2687c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -64,13 +64,13 @@ public class DockerOperationsImpl implements DockerOperations { context.log(logger, "Creating container"); // IPv6 - Assume always valid - Inet6Address ipV6Address = ipAddresses.getIPv6Address(context.node().getHostname()).orElseThrow( - () -> new RuntimeException("Unable to find a valid IPv6 address for " + context.node().getHostname() + + Inet6Address ipV6Address = ipAddresses.getIPv6Address(context.node().hostname()).orElseThrow( + () -> new RuntimeException("Unable to find a valid IPv6 address for " + context.node().hostname() + ". Missing an AAAA DNS entry?")); Docker.CreateContainerCommand command = docker.createContainerCommand( - context.node().getWantedDockerImage().get(), context.containerName()) - .withHostName(context.node().getHostname()) + context.node().wantedDockerImage().get(), context.containerName()) + .withHostName(context.node().hostname()) .withResources(containerResources) .withManagedBy(MANAGER_NAME) .withUlimit("nofile", 262_144, 262_144) @@ -88,7 +88,7 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_ADMIN") // Needed for perf .withAddCapability("SYS_NICE"); // Needed for set_mempolicy to work - if (context.node().getMembership().map(NodeMembership::getClusterType).map("content"::equalsIgnoreCase).orElse(false)) { + if (context.node().membership().map(NodeMembership::clusterType).map("content"::equalsIgnoreCase).orElse(false)) { command.withSecurityOpts("seccomp=unconfined"); } @@ -101,20 +101,20 @@ public class DockerOperationsImpl implements DockerOperations { command.withIpAddress(ipV6Local); // IPv4 - Only present for some containers - Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().getHostname()) + Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname()) .map(ipV4Address -> { InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX); return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2); }); ipV4Local.ifPresent(command::withIpAddress); - addEtcHosts(containerData, context.node().getHostname(), ipV4Local, ipV6Local); + addEtcHosts(containerData, context.node().hostname(), ipV4Local, ipV6Local); } addMounts(context, command); // TODO: Enforce disk constraints - long minMainMemoryAvailableMb = (long) (context.node().getMinMainMemoryAvailableGb() * 1024); + long minMainMemoryAvailableMb = (long) (context.node().memoryGb() * 1024); if (minMainMemoryAvailableMb > 0) { // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine // only has this much physical memory (overrides total memory reported by `free -m`). diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 4972a306377..26e4dcda88e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -158,23 +158,23 @@ public class StorageMaintainer { private Map<String, Object> generateTags(NodeAgentContext context) { Map<String, String> tags = new LinkedHashMap<>(); tags.put("namespace", "Vespa"); - tags.put("role", nodeTypeToRole(context.node().getNodeType())); + tags.put("role", nodeTypeToRole(context.node().type())); tags.put("zone", context.zone().getId().value()); - context.node().getVespaVersion().ifPresent(version -> tags.put("vespaVersion", version.toFullString())); + context.node().currentVespaVersion().ifPresent(version -> tags.put("vespaVersion", version.toFullString())); if (! isConfigserverLike(context.nodeType())) { - tags.put("state", context.node().getState().toString()); - context.node().getParentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); - context.node().getOwner().ifPresent(owner -> { - tags.put("tenantName", owner.getTenant()); - tags.put("app", owner.getApplication() + "." + owner.getInstance()); - tags.put("applicationName", owner.getApplication()); - tags.put("instanceName", owner.getInstance()); - tags.put("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance()); + tags.put("state", context.node().state().toString()); + context.node().parentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); + context.node().owner().ifPresent(owner -> { + tags.put("tenantName", owner.tenant()); + tags.put("app", owner.application() + "." + owner.instance()); + tags.put("applicationName", owner.application()); + tags.put("instanceName", owner.instance()); + tags.put("applicationId", owner.tenant() + "." + owner.application() + "." + owner.instance()); }); - context.node().getMembership().ifPresent(membership -> { - tags.put("clustertype", membership.getClusterType()); - tags.put("clusterid", membership.getClusterId()); + context.node().membership().ifPresent(membership -> { + tags.put("clustertype", membership.clusterType()); + tags.put("clusterid", membership.clusterId()); }); } @@ -260,20 +260,20 @@ public class StorageMaintainer { private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) { Map<String, String> attributes = new HashMap<>(); - attributes.put("hostname", context.node().getHostname()); + attributes.put("hostname", context.node().hostname()); attributes.put("region", context.zone().getRegionName().value()); attributes.put("environment", context.zone().getEnvironment().value()); - attributes.put("flavor", context.node().getFlavor()); + attributes.put("flavor", context.node().flavor()); attributes.put("kernel_version", System.getProperty("os.version")); attributes.put("cpu_microcode_version", getMicrocodeVersion()); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); - context.node().getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); - context.node().getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version.toFullString())); - context.node().getOwner().ifPresent(owner -> { - attributes.put("tenant", owner.getTenant()); - attributes.put("application", owner.getApplication()); - attributes.put("instance", owner.getInstance()); + context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); + context.node().currentVespaVersion().ifPresent(version -> attributes.put("vespa_version", version.toFullString())); + context.node().owner().ifPresent(owner -> { + attributes.put("tenant", owner.tenant()); + attributes.put("application", owner.application()); + attributes.put("instance", owner.instance()); }); return Collections.unmodifiableMap(attributes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index f3302bd2359..4a76e0e0a5b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -125,7 +125,7 @@ public class NodeAdminStateUpdater { throw new ConvergenceException("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen")); } - boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).getState() == NodeState.active; + boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).state() == NodeState.active; switch (wantedState) { case RESUMED: if (hostIsActiveInNR) orchestrator.resume(hostHostname); @@ -164,7 +164,7 @@ public class NodeAdminStateUpdater { void adjustNodeAgentsToRunFromNodeRepository() { try { Map<String, NodeSpec> nodeSpecByHostname = nodeRepository.getNodes(hostHostname).stream() - .collect(Collectors.toMap(NodeSpec::getHostname, Function.identity())); + .collect(Collectors.toMap(NodeSpec::hostname, Function.identity())); Map<String, Acl> aclByHostname = Optional.of(cachedAclSupplier.get()) .filter(acls -> acls.keySet().containsAll(nodeSpecByHostname.keySet())) .orElseGet(cachedAclSupplier::invalidateAndGet); @@ -183,8 +183,8 @@ public class NodeAdminStateUpdater { private List<String> getNodesInActiveState() { return nodeRepository.getNodes(hostHostname) .stream() - .filter(node -> node.getState() == NodeState.active) - .map(NodeSpec::getHostname) + .filter(node -> node.state() == NodeState.active) + .map(NodeSpec::hostname) .collect(Collectors.toList()); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index a7cdd7e655d..f1fd97f6e4c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -26,11 +26,11 @@ public interface NodeAgentContext extends TaskContext { /** @return hostname of the docker container this context applies to */ default HostName hostname() { - return HostName.from(node().getHostname()); + return HostName.from(node().hostname()); } default NodeType nodeType() { - return node().getNodeType(); + return node().type(); } AthenzIdentity identity(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 8435fe34770..ef8ea60bee3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -1,9 +1,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; @@ -47,7 +45,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { String vespaUser, String vespaUserOnHost) { this.node = Objects.requireNonNull(node); this.acl = Objects.requireNonNull(acl); - this.containerName = ContainerName.fromHostname(node.getHostname()); + this.containerName = ContainerName.fromHostname(node.hostname()); this.identity = Objects.requireNonNull(identity); this.dockerNetworking = Objects.requireNonNull(dockerNetworking); this.zone = Objects.requireNonNull(zone); @@ -181,12 +179,12 @@ public class NodeAgentContextImpl implements NodeAgentContext { this.nodeSpecBuilder .hostname(hostname) .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("d-2-8-50"); } public Builder nodeType(NodeType nodeType) { - this.nodeSpecBuilder.nodeType(nodeType); + this.nodeSpecBuilder.type(nodeType); return this; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2716cc8cc59..977f1016ed8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -118,7 +118,7 @@ public class NodeAgentImpl implements NodeAgent { this.healthChecker = healthChecker; this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource) - .with(FetchVector.Dimension.HOSTNAME, contextSupplier.currentContext().node().getHostname()); + .with(FetchVector.Dimension.HOSTNAME, contextSupplier.currentContext().node().hostname()); this.loopThread = new Thread(() -> { while (!terminated.get()) { @@ -172,20 +172,20 @@ public class NodeAgentImpl implements NodeAgent { final NodeAttributes currentNodeAttributes = new NodeAttributes(); final NodeAttributes newNodeAttributes = new NodeAttributes(); - if (context.node().getWantedRestartGeneration().isPresent() && - !Objects.equals(context.node().getCurrentRestartGeneration(), currentRestartGeneration)) { - currentNodeAttributes.withRestartGeneration(context.node().getCurrentRestartGeneration()); + if (context.node().wantedRestartGeneration().isPresent() && + !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) { + currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration()); newNodeAttributes.withRestartGeneration(currentRestartGeneration); } - if (!Objects.equals(context.node().getCurrentRebootGeneration(), currentRebootGeneration)) { - currentNodeAttributes.withRebootGeneration(context.node().getCurrentRebootGeneration()); + if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration)) { + currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration()); newNodeAttributes.withRebootGeneration(currentRebootGeneration); } - Optional<DockerImage> actualDockerImage = context.node().getWantedDockerImage().filter(n -> containerState == UNKNOWN); - if (!Objects.equals(context.node().getCurrentDockerImage(), actualDockerImage)) { - DockerImage currentImage = context.node().getCurrentDockerImage().orElse(DockerImage.EMPTY); + Optional<DockerImage> actualDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); + if (!Objects.equals(context.node().currentDockerImage(), actualDockerImage)) { + DockerImage currentImage = context.node().currentDockerImage().orElse(DockerImage.EMPTY); DockerImage newImage = actualDockerImage.orElse(DockerImage.EMPTY); currentNodeAttributes.withDockerImage(currentImage); @@ -228,7 +228,7 @@ public class NodeAgentImpl implements NodeAgent { shouldRestartServices(context.node()).ifPresent(restartReason -> { context.log(logger, "Will restart services: " + restartReason); restartServices(context, existingContainer.get()); - currentRestartGeneration = context.node().getWantedRestartGeneration(); + currentRestartGeneration = context.node().wantedRestartGeneration(); }); } @@ -236,18 +236,18 @@ public class NodeAgentImpl implements NodeAgent { } private Optional<String> shouldRestartServices(NodeSpec node) { - if (!node.getWantedRestartGeneration().isPresent()) return Optional.empty(); + if (!node.wantedRestartGeneration().isPresent()) return Optional.empty(); // Restart generation is only optional because it does not exist for unallocated nodes - if (currentRestartGeneration.get() < node.getWantedRestartGeneration().get()) { + if (currentRestartGeneration.get() < node.wantedRestartGeneration().get()) { return Optional.of("Restart requested - wanted restart generation has been bumped: " - + currentRestartGeneration.get() + " -> " + node.getWantedRestartGeneration().get()); + + currentRestartGeneration.get() + " -> " + node.wantedRestartGeneration().get()); } return Optional.empty(); } private void restartServices(NodeAgentContext context, Container existingContainer) { - if (existingContainer.state.isRunning() && context.node().getState() == NodeState.active) { + if (existingContainer.state.isRunning() && context.node().state() == NodeState.active) { context.log(logger, "Restarting services"); // Since we are restarting the services we need to suspend the node. orchestratorSuspendNode(context); @@ -290,22 +290,22 @@ public class NodeAgentImpl implements NodeAgent { } private Optional<String> shouldRemoveContainer(NodeAgentContext context, Container existingContainer) { - final NodeState nodeState = context.node().getState(); + final NodeState nodeState = context.node().state(); if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) { return Optional.of("Node in state " + nodeState + ", container should no longer be running"); } - if (context.node().getWantedDockerImage().isPresent() && - !context.node().getWantedDockerImage().get().equals(existingContainer.image)) { + if (context.node().wantedDockerImage().isPresent() && + !context.node().wantedDockerImage().get().equals(existingContainer.image)) { return Optional.of("The node is supposed to run a new Docker image: " - + existingContainer.image.asString() + " -> " + context.node().getWantedDockerImage().get().asString()); + + existingContainer.image.asString() + " -> " + context.node().wantedDockerImage().get().asString()); } if (!existingContainer.state.isRunning()) { return Optional.of("Container no longer running"); } - if (currentRebootGeneration < context.node().getWantedRebootGeneration()) { + if (currentRebootGeneration < context.node().wantedRebootGeneration()) { return Optional.of(String.format("Container reboot wanted. Current: %d, Wanted: %d", - currentRebootGeneration, context.node().getWantedRebootGeneration())); + currentRebootGeneration, context.node().wantedRebootGeneration())); } // Even though memory can be easily changed with docker update, we need to restart the container @@ -330,7 +330,7 @@ public class NodeAgentImpl implements NodeAgent { } try { - if (context.node().getState() != NodeState.dirty) { + if (context.node().state() != NodeState.dirty) { suspend(); } stopServices(); @@ -341,7 +341,7 @@ public class NodeAgentImpl implements NodeAgent { storageMaintainer.handleCoreDumpsForContainer(context, Optional.of(existingContainer)); dockerOperations.removeContainer(context, existingContainer); - currentRebootGeneration = context.node().getWantedRebootGeneration(); + currentRebootGeneration = context.node().wantedRebootGeneration(); containerState = ABSENT; context.log(logger, "Container successfully removed, new containerState is " + containerState); } @@ -361,13 +361,13 @@ public class NodeAgentImpl implements NodeAgent { private ContainerResources getContainerResources(NodeAgentContext context) { double cpuCap = noCpuCap(context.zone()) ? 0 : - context.node().getOwner() + context.node().owner() .map(NodeOwner::asApplicationId) .map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())) .orElse(containerCpuCap) - .value() * context.node().getMinCpuCores(); + .value() * context.node().vcpus(); - return ContainerResources.from(cpuCap, context.node().getMinCpuCores(), context.node().getMinMainMemoryAvailableGb()); + return ContainerResources.from(cpuCap, context.node().vcpus(), context.node().memoryGb()); } private boolean noCpuCap(ZoneApi zone) { @@ -376,9 +376,9 @@ public class NodeAgentImpl implements NodeAgent { } private boolean downloadImageIfNeeded(NodeSpec node, Optional<Container> container) { - if (node.getWantedDockerImage().equals(container.map(c -> c.image))) return false; + if (node.wantedDockerImage().equals(container.map(c -> c.image))) return false; - return node.getWantedDockerImage().map(dockerOperations::pullImageAsyncIfNeeded).orElse(false); + return node.wantedDockerImage().map(dockerOperations::pullImageAsyncIfNeeded).orElse(false); } public void converge(NodeAgentContext context) { @@ -406,14 +406,14 @@ public class NodeAgentImpl implements NodeAgent { logChangesToNodeSpec(context, lastNode, node); // Current reboot generation uninitialized or incremented from outside to cancel reboot - if (currentRebootGeneration < node.getCurrentRebootGeneration()) - currentRebootGeneration = node.getCurrentRebootGeneration(); + if (currentRebootGeneration < node.currentRebootGeneration()) + currentRebootGeneration = node.currentRebootGeneration(); // Either we have changed allocation status (restart gen. only available to allocated nodes), or // restart generation has been incremented from outside to cancel restart - if (currentRestartGeneration.isPresent() != node.getCurrentRestartGeneration().isPresent() || - currentRestartGeneration.map(current -> current < node.getCurrentRestartGeneration().get()).orElse(false)) - currentRestartGeneration = node.getCurrentRestartGeneration(); + if (currentRestartGeneration.isPresent() != node.currentRestartGeneration().isPresent() || + currentRestartGeneration.map(current -> current < node.currentRestartGeneration().get()).orElse(false)) + currentRestartGeneration = node.currentRestartGeneration(); // Every time the node spec changes, we should clear the metrics for this container as the dimensions // will change and we will be reporting duplicate metrics. @@ -424,7 +424,7 @@ public class NodeAgentImpl implements NodeAgent { lastNode = node; } - switch (node.getState()) { + switch (node.state()) { case ready: case reserved: case parked: @@ -437,12 +437,12 @@ public class NodeAgentImpl implements NodeAgent { storageMaintainer.handleCoreDumpsForContainer(context, container); storageMaintainer.getDiskUsageFor(context) - .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.getMinDiskAvailableGb()) + .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.diskGb()) .filter(diskUtil -> diskUtil >= 0.8) .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context)); if (downloadImageIfNeeded(node, container)) { - context.log(logger, "Waiting for image to download " + context.node().getWantedDockerImage().get().asString()); + context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); return; } container = removeContainerIfNeededUpdateContainerState(context, container); @@ -479,20 +479,20 @@ public class NodeAgentImpl implements NodeAgent { break; case dirty: removeContainerIfNeededUpdateContainerState(context, container); - context.log(logger, "State is " + node.getState() + ", will delete application storage and mark node as ready"); + context.log(logger, "State is " + node.state() + ", will delete application storage and mark node as ready"); credentialsMaintainer.ifPresent(maintainer -> maintainer.clearCredentials(context)); storageMaintainer.archiveNodeStorage(context); updateNodeRepoWithCurrentAttributes(context); nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); break; default: - throw new ConvergenceException("UNKNOWN STATE " + node.getState().name()); + throw new ConvergenceException("UNKNOWN STATE " + node.state().name()); } } private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) { StringBuilder builder = new StringBuilder(); - appendIfDifferent(builder, "state", lastNode, node, NodeSpec::getState); + appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state); if (builder.length() > 0) { context.log(logger, LogLevel.INFO, "Changes to node: " + builder.toString()); } @@ -525,9 +525,9 @@ public class NodeAgentImpl implements NodeAgent { Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() .add("host", context.hostname().value()) .add("role", SecretAgentCheckConfig.nodeTypeToRole(context.nodeType())) - .add("state", node.getState().toString()); - node.getParentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); - node.getAllowedToBeDown().ifPresent(allowed -> + .add("state", node.state().toString()); + node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); + node.allowedToBeDown().ifPresent(allowed -> dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS")); Dimensions dimensions = dimensionsBuilder.build(); @@ -540,13 +540,13 @@ public class NodeAgentImpl implements NodeAgent { final long memoryTotalBytes = stats.getMemoryStats().getLimit(); final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage(); final long memoryTotalBytesCache = stats.getMemoryStats().getCache(); - final long diskTotalBytes = (long) (node.getMinDiskAvailableGb() * BYTES_IN_GB); + final long diskTotalBytes = (long) (node.diskGb() * BYTES_IN_GB); final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context); lastCpuMetric.updateCpuDeltas(cpuSystemTotalTime, cpuContainerTotalTime, cpuContainerKernelTime); // Ratio of CPU cores allocated to this container to total number of CPU cores on this host - final double allocatedCpuRatio = node.getMinCpuCores() / totalNumCpuCores; + final double allocatedCpuRatio = node.vcpus() / totalNumCpuCores; double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio; double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio; @@ -564,7 +564,7 @@ public class NodeAgentImpl implements NodeAgent { .withMetric("mem_total.util", 100 * memoryTotalUsageRatio) .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated) .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated) - .withMetric("cpu.vcpus", node.getMinCpuCores()) + .withMetric("cpu.vcpus", node.vcpus()) .withMetric("disk.limit", diskTotalBytes); diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed)); @@ -597,7 +597,7 @@ public class NodeAgentImpl implements NodeAgent { // Push metrics to the metrics proxy in each container. // TODO Remove port selection logic when all hosted apps have upgraded to Vespa 7. - int port = context.node().getVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095; + int port = context.node().currentVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095; String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:" + port, "setExtraMetrics", wrappedMetrics}; dockerOperations.executeCommandInContainerAsRoot(context, 5L, command); } catch (JsonProcessingException | DockerExecTimeoutException e) { @@ -666,7 +666,7 @@ public class NodeAgentImpl implements NodeAgent { // to allow the node admin to make decisions that depend on the docker image. Or, each docker image // needs to contain routines for drain and suspend. For many images, these can just be dummy routines. private void orchestratorSuspendNode(NodeAgentContext context) { - if (context.node().getState() != NodeState.active) return; + if (context.node().state() != NodeState.active) return; context.log(logger, "Ask Orchestrator for permission to suspend node"); try { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index fb443ed14c4..0938eb23b49 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -105,14 +105,14 @@ public class RealNodeRepositoryTest { List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); assertThat(containersToRun.size(), is(1)); NodeSpec node = containersToRun.get(0); - assertThat(node.getHostname(), is("host4.yahoo.com")); - assertThat(node.getWantedDockerImage().get(), is(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"))); - assertThat(node.getState(), is(NodeState.active)); - assertThat(node.getWantedRestartGeneration().get(), is(0L)); - assertThat(node.getCurrentRestartGeneration().get(), is(0L)); - assertEquals(1, node.getMinCpuCores(), delta); - assertEquals(1, node.getMinMainMemoryAvailableGb(), delta); - assertEquals(100, node.getMinDiskAvailableGb(), delta); + assertThat(node.hostname(), is("host4.yahoo.com")); + assertThat(node.wantedDockerImage().get(), is(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"))); + assertThat(node.state(), is(NodeState.active)); + assertThat(node.wantedRestartGeneration().get(), is(0L)); + assertThat(node.currentRestartGeneration().get(), is(0L)); + assertEquals(1, node.vcpus(), delta); + assertEquals(1, node.memoryGb(), delta); + assertEquals(100, node.diskGb(), delta); } @Test @@ -120,7 +120,7 @@ public class RealNodeRepositoryTest { String hostname = "host4.yahoo.com"; Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); assertTrue(node.isPresent()); - assertEquals(hostname, node.get().getHostname()); + assertEquals(hostname, node.get().hostname()); } @Test @@ -176,8 +176,8 @@ public class RealNodeRepositoryTest { NodeSpec hostSpecInNodeRepo = nodeRepositoryApi.getOptionalNode("host123.domain.tld") .orElseThrow(RuntimeException::new); - assertEquals(host.nodeFlavor, hostSpecInNodeRepo.getFlavor()); - assertEquals(host.nodeType, hostSpecInNodeRepo.getNodeType()); + assertEquals(host.nodeFlavor, hostSpecInNodeRepo.flavor()); + assertEquals(host.nodeType, hostSpecInNodeRepo.type()); assertTrue(nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").isPresent()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java index 14755ebf9cc..a3256b6955b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.state; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; +import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; import org.junit.Test; @@ -29,7 +29,8 @@ public class StateImplTest { @Test public void connectException() { - RuntimeException exception = HttpException.handleException("Error: ", new ConnectException("connection refused")); + RuntimeException exception = + ConnectionException.handleException("Error: ", new ConnectException("connection refused")); when(api.get(any(), any())).thenThrow(exception); HealthCode code = state.getHealth(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index f3e334fff73..aacb2cafd30 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -29,13 +29,13 @@ public class DockerFailTest { .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) + .vcpus(1) + .memoryGb(1) + .diskGb(1) .build()); tester.inOrder(tester.docker).createContainerCommand(eq(dockerImage), eq(containerName)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 7f0f3fd37f6..22b3949755f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -81,7 +81,7 @@ public class DockerTester implements AutoCloseable { NodeSpec hostSpec = new NodeSpec.Builder() .hostname(HOST_HOSTNAME.value()) .state(NodeState.active) - .nodeType(NodeType.host) + .type(NodeType.host) .flavor("default") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index 27b11c3c1ba..8163f90e31f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -28,15 +28,15 @@ public class MultiDockerTest { tester.addChildNodeRepositoryNode( new NodeSpec.Builder(nodeSpec2) .state(NodeState.dirty) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) + .vcpus(1) + .memoryGb(1) + .diskGb(1) .build()); tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2"))); tester.inOrder(tester.storageMaintainer).archiveNodeStorage( argThat(context -> context.containerName().equals(new ContainerName("host2")))); - tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(NodeState.ready)); + tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready)); addAndWaitForNode(tester, "host3.test.yahoo.com", DockerImage.fromString("image1")); } @@ -47,13 +47,13 @@ public class MultiDockerTest { .hostname(hostName) .wantedDockerImage(dockerImage) .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) - .minCpuCores(2) - .minMainMemoryAvailableGb(4) - .minDiskAvailableGb(1) + .vcpus(2) + .memoryGb(4) + .diskGb(1) .build(); tester.addChildNodeRepositoryNode(nodeSpec); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index ebf9d72ff1b..625166a10d2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -32,7 +32,7 @@ public class NodeRepoMock implements NodeRepository { public List<NodeSpec> getNodes(String baseHostName) { synchronized (monitor) { return nodeRepositoryNodesByHostname.values().stream() - .filter(node -> baseHostName.equals(node.getParentHostname().orElse(null))) + .filter(node -> baseHostName.equals(node.parentHostname().orElse(null))) .collect(Collectors.toList()); } } @@ -69,7 +69,7 @@ public class NodeRepoMock implements NodeRepository { void updateNodeRepositoryNode(NodeSpec nodeSpec) { synchronized (monitor) { - nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec); + nodeRepositoryNodesByHostname.put(nodeSpec.hostname(), nodeSpec); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index 674c562cd88..4a232a5b2bd 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -52,9 +52,9 @@ public class RebootTest { .hostname(hostname) .wantedDockerImage(dockerImage) .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") - .vespaVersion(Version.fromString("6.50.0")) + .currentVespaVersion(Version.fromString("6.50.0")) .wantedRestartGeneration(1L) .currentRestartGeneration(1L) .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 82e5eca042c..bfc54cac045 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -30,7 +30,7 @@ public class RestartTest { .hostname(hostname) .state(NodeState.active) .wantedDockerImage(dockerImage) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1) .currentRestartGeneration(1) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 36169a2b283..57b18606def 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -157,12 +157,12 @@ public class StorageMaintainerTest { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname("host123-5.test.domain.tld") - .nodeType(nodeType) + .type(nodeType) .state(NodeState.active) .parentHostname("host123.test.domain.tld") .owner(new NodeOwner("tenant", "application", "instance")) .membership(new NodeMembership("clusterType", "clusterId", null, 0, false)) - .vespaVersion(Version.fromString("6.305.12")) + .currentVespaVersion(Version.fromString("6.305.12")) .flavor("d-2-8-50") .canonicalFlavor("d-2-8-50") .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 6e645e6c70f..ca9b05a3ff6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -160,7 +160,7 @@ public class NodeAdminImplTest { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname(hostname) .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("default") .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index b8894bbf814..bb18e261301 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -59,7 +59,7 @@ public class NodeAdminStateUpdaterTest { public void state_convergence() { mockNodeRepo(NodeState.active, 4); List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::getHostname) + .map(NodeSpec::hostname) .collect(Collectors.toList()); List<String> suspendHostnames = new ArrayList<>(activeHostnames); suspendHostnames.add(hostHostname.value()); @@ -170,7 +170,7 @@ public class NodeAdminStateUpdaterTest { // When doing batch suspend, only suspend the containers if the host is not active List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::getHostname) + .map(NodeSpec::hostname) .collect(Collectors.toList()); updater.converge(SUSPENDED); verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); @@ -206,9 +206,9 @@ public class NodeAdminStateUpdaterTest { updater.adjustNodeAgentsToRunFromNodeRepository(); updater.adjustNodeAgentsToRunFromNodeRepository(); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host3.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(acl)); verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); verify(nodeRepository, times(1)).getAcls(eq(hostHostname.value())); } @@ -224,9 +224,9 @@ public class NodeAdminStateUpdaterTest { updater.adjustNodeAgentsToRunFromNodeRepository(); updater.adjustNodeAgentsToRunFromNodeRepository(); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.getHostname().equals("host3.yahoo.com")), eq(Acl.EMPTY)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(Acl.EMPTY)); verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); verify(nodeRepository, times(2)).getAcls(eq(hostHostname.value())); // During the first tick, the cache is invalidated and retried } @@ -241,8 +241,8 @@ public class NodeAdminStateUpdaterTest { updater.adjustNodeAgentsToRunFromNodeRepository(); updater.adjustNodeAgentsToRunFromNodeRepository(); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.getHostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); verify(nodeRepository, times(1)).getAcls(eq(hostHostname.value())); } @@ -261,11 +261,11 @@ public class NodeAdminStateUpdaterTest { .mapToObj(i -> new NodeSpec.Builder() .hostname("host" + i + ".yahoo.com") .state(NodeState.active) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) + .vcpus(1) + .memoryGb(1) + .diskGb(1) .build()) .collect(Collectors.toList()); @@ -274,11 +274,11 @@ public class NodeAdminStateUpdaterTest { when(nodeRepository.getNode(eq(hostHostname.value()))).thenReturn(new NodeSpec.Builder() .hostname(hostHostname.value()) .state(hostState) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("default") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) + .vcpus(1) + .memoryGb(1) + .diskGb(1) .build()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index f754d1798ec..b4db8ff40d5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -64,11 +64,11 @@ public class NodeAgentImplTest { private final String hostName = "host1.test.yahoo.com"; private final NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() .hostname(hostName) - .nodeType(NodeType.tenant) + .type(NodeType.tenant) .flavor("docker") - .minCpuCores(MIN_CPU_CORES) - .minMainMemoryAvailableGb(MIN_MAIN_MEMORY_AVAILABLE_GB) - .minDiskAvailableGb(MIN_DISK_AVAILABLE_GB); + .vcpus(MIN_CPU_CORES) + .memoryGb(MIN_MAIN_MEMORY_AVAILABLE_GB) + .diskGb(MIN_DISK_AVAILABLE_GB); private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); private final DockerImage dockerImage = DockerImage.fromString("dockerImage"); @@ -90,7 +90,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -119,7 +119,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -140,7 +140,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -214,7 +214,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -241,7 +241,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion); + .currentVespaVersion(vespaVersion); NodeAgentContext firstContext = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -250,9 +250,9 @@ public class NodeAgentImplTest { when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(firstContext); - NodeAgentContext secondContext = createContext(specBuilder.minDiskAvailableGb(200).build()); + NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build()); nodeAgent.doConverge(secondContext); - NodeAgentContext thirdContext = createContext(specBuilder.minCpuCores(4).build()); + NodeAgentContext thirdContext = createContext(specBuilder.vcpus(4).build()); nodeAgent.doConverge(thirdContext); ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); mockGetContainer(dockerImage, resourcesAfterThird, true); @@ -288,7 +288,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion); + .currentVespaVersion(vespaVersion); NodeAgentContext firstContext = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -297,7 +297,7 @@ public class NodeAgentImplTest { when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(firstContext); - NodeAgentContext secondContext = createContext(specBuilder.minMainMemoryAvailableGb(20).build()); + NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); nodeAgent.doConverge(secondContext); ContainerResources resourcesAfterThird = ContainerResources.from(0, 2, 20); mockGetContainer(dockerImage, resourcesAfterThird, true); @@ -325,7 +325,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .wantedRestartGeneration(wantedRestartGeneration) .currentRestartGeneration(currentRestartGeneration) .build(); @@ -357,7 +357,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .wantedRebootGeneration(wantedRebootGeneration) .currentRebootGeneration(currentRebootGeneration) .build(); @@ -400,7 +400,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.failed) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -446,7 +446,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .state(NodeState.inactive) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -548,7 +548,7 @@ public class NodeAgentImplTest { .currentDockerImage(dockerImage) .wantedDockerImage(dockerImage) .state(NodeState.active) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -570,7 +570,7 @@ public class NodeAgentImplTest { .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(NodeState.active) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .build(); NodeAgentContext context = createContext(node); @@ -651,10 +651,10 @@ public class NodeAgentImplTest { .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(NodeState.active) - .vespaVersion(vespaVersion) + .currentVespaVersion(vespaVersion) .owner(owner) .membership(membership) - .minMainMemoryAvailableGb(2) + .memoryGb(2) .allowedToBeDown(true) .parentHostname("parent.host.name.yahoo.com") .build(); @@ -713,7 +713,7 @@ public class NodeAgentImplTest { @Test public void testRunningConfigServer() { final NodeSpec node = nodeBuilder - .nodeType(NodeType.config) + .type(NodeType.config) .wantedDockerImage(dockerImage) .state(NodeState.active) .wantedVespaVersion(vespaVersion) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index bedfbc5bdc1..9b78f558a7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -399,10 +399,7 @@ public class NodeRepository extends AbstractComponent { public void deactivate(ApplicationId application, NestedTransaction transaction) { try (Mutex lock = lock(application)) { - db.writeTo(Node.State.inactive, - db.getNodes(application, Node.State.reserved, Node.State.active), - Agent.application, Optional.empty(), transaction - ); + deactivate(db.getNodes(application, Node.State.reserved, Node.State.active), transaction); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index 58c576d3f44..26f5a148b76 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; +import java.time.Instant; import java.util.Objects; /** @@ -14,12 +15,14 @@ public class LoadBalancer { private final LoadBalancerId id; private final LoadBalancerInstance instance; - private final boolean inactive; + private final State state; + private final Instant changedAt; - public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, boolean inactive) { + public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, State state, Instant changedAt) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.instance = Objects.requireNonNull(instance, "instance must be non-null"); - this.inactive = inactive; + this.state = Objects.requireNonNull(state, "state must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); } /** An identifier for this load balancer. The ID is unique inside the zone */ @@ -32,17 +35,44 @@ public class LoadBalancer { return instance; } - /** - * Returns whether this load balancer is inactive. Inactive load balancers are eventually removed by - * {@link LoadBalancerExpirer}. Inactive load balancers may be reactivated if a deleted cluster is redeployed. - */ - public boolean inactive() { - return inactive; + /** The current state of this */ + public State state() { + return state; } - /** Return a copy of this that is set inactive */ - public LoadBalancer deactivate() { - return new LoadBalancer(id, instance, true); + /** Returns when this was last changed */ + public Instant changedAt() { + return changedAt; + } + + /** Returns a copy of this with state set to given state */ + public LoadBalancer with(State state, Instant changedAt) { + if (this.state != State.reserved && state == State.reserved) { + throw new IllegalArgumentException("Invalid state transition: " + this.state + " -> " + state); + } + return new LoadBalancer(id, instance, state, changedAt); + } + + /** Returns a copy of this with instance set to given instance */ + public LoadBalancer with(LoadBalancerInstance instance) { + return new LoadBalancer(id, instance, state, changedAt); + } + + public enum State { + + /** This load balancer has been provisioned and reserved for an application */ + reserved, + + /** + * The load balancer has been deactivated and is ready to be removed. Inactive load balancers are eventually + * removed by {@link LoadBalancerExpirer}. Inactive load balancers may be reactivated if a deleted cluster is + * redeployed. + */ + inactive, + + /** The load balancer is in active use by an application */ + active, + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index ba7a83169ad..7fd50bf0930 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -3,18 +3,20 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; +import java.time.Instant; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; /** - * A filterable load balancer list. + * A filterable load balancer list. This is immutable. * * @author mpolden */ -public class LoadBalancerList { +public class LoadBalancerList implements Iterable<LoadBalancer> { private final List<LoadBalancer> loadBalancers; @@ -27,9 +29,14 @@ public class LoadBalancerList { return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application))); } - /** Returns the subset of load balancers that are inactive */ - public LoadBalancerList inactive() { - return of(loadBalancers.stream().filter(LoadBalancer::inactive)); + /** Returns the subset of load balancers that are in given state */ + public LoadBalancerList in(LoadBalancer.State state) { + return of(loadBalancers.stream().filter(lb -> lb.state() == state)); + } + + /** Returns the subset of load balancers that last changed before given instant */ + public LoadBalancerList changedBefore(Instant instant) { + return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant))); } public List<LoadBalancer> asList() { @@ -40,4 +47,9 @@ public class LoadBalancerList { return new LoadBalancerList(stream.collect(Collectors.toUnmodifiableList())); } + @Override + public Iterator<LoadBalancer> iterator() { + return loadBalancers.iterator(); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 684f6dbcd50..d7f41c4d8e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; */ public class FailedExpirer extends Maintainer { - private static final Logger log = Logger.getLogger(NodeRetirer.class.getName()); + private static final Logger log = Logger.getLogger(FailedExpirer.class.getName()); private static final int maxAllowedFailures = 5; // Stop recycling nodes after this number of failures private final NodeRepository nodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index d6b392c4d64..7d7f8d479fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -6,6 +6,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer.State; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -17,15 +18,21 @@ import java.util.Objects; import java.util.stream.Collectors; /** - * Periodically remove inactive load balancers permanently. + * Periodically expire load balancers. * - * When an application is removed, any associated load balancers are only deactivated. This maintainer ensures that - * underlying load balancer instances are eventually freed. + * Load balancers expire from the following states: + * + * {@link LoadBalancer.State#inactive}: An application is removed and load balancers are deactivated. + * {@link LoadBalancer.State#reserved}: An prepared application is never successfully activated, thus never activating + * any prepared load balancers. * * @author mpolden */ public class LoadBalancerExpirer extends Maintainer { + private static final Duration reservedExpiry = Duration.ofHours(1); + private static final Duration inactiveExpiry = Duration.ofHours(1); + private final LoadBalancerService service; private final CuratorDatabaseClient db; @@ -37,22 +44,39 @@ public class LoadBalancerExpirer extends Maintainer { @Override protected void maintain() { + expireReserved(); removeInactive(); } + private void expireReserved() { + try (Lock lock = db.lockLoadBalancers()) { + var now = nodeRepository().clock().instant(); + var expirationTime = now.minus(reservedExpiry); + var expired = nodeRepository().loadBalancers() + .in(State.reserved) + .changedBefore(expirationTime); + expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); + } + } + private void removeInactive() { List<LoadBalancerId> failed = new ArrayList<>(); Exception lastException = null; try (Lock lock = db.lockLoadBalancers()) { - for (LoadBalancer loadBalancer : nodeRepository().loadBalancers().inactive().asList()) { - if (hasNodes(loadBalancer.id().application())) { // Defer removal if there are still nodes allocated to application + var now = nodeRepository().clock().instant(); + var expirationTime = now.minus(inactiveExpiry); + var expired = nodeRepository().loadBalancers() + .in(State.inactive) + .changedBefore(expirationTime); + for (var lb : expired) { + if (hasNodes(lb.id().application())) { // Defer removal if there are still nodes allocated to application continue; } try { - service.remove(loadBalancer.id().application(), loadBalancer.id().cluster()); - db.removeLoadBalancer(loadBalancer.id()); + service.remove(lb.id().application(), lb.id().cluster()); + db.removeLoadBalancer(lb.id()); } catch (Exception e) { - failed.add(loadBalancer.id()); + failed.add(lb.id()); lastException = e; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 25549abe9ed..0ecbfab2b99 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -7,16 +7,10 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.InfraDeployer; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareCount; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -47,7 +41,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final DirtyExpirer dirtyExpirer; private final ProvisionedExpirer provisionedExpirer; private final NodeRebooter nodeRebooter; - private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; private final InfrastructureProvisioner infrastructureProvisioner; private final Optional<LoadBalancerExpirer> loadBalancerExpirer; @@ -91,11 +84,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); - - RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone)); - FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker( - NodeRetirer.SPARE_NODES_POLICY, FlavorSpareCount.constructFlavorSpareCountGraph(zone.nodeFlavors().get().getFlavors())); - nodeRetirer = new NodeRetirer(nodeRepository, flavorSpareChecker, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), deployer, policy); } @Override @@ -109,7 +97,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failedExpirer.deconstruct(); dirtyExpirer.deconstruct(); nodeRebooter.deconstruct(); - nodeRetirer.deconstruct(); provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); infrastructureProvisioner.deconstruct(); @@ -153,7 +140,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration dirtyExpiry; private final Duration provisionedExpiry; private final Duration rebootInterval; - private final Duration nodeRetirerInterval; private final Duration metricsInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; @@ -172,11 +158,10 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failedExpirerInterval = Duration.ofMinutes(10); provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); - nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; - loadBalancerExpiry = Duration.ofHours(1); + loadBalancerExpiry = Duration.ofMinutes(10); reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions hostProvisionerInterval = Duration.ofMinutes(5); hostDeprovisionerInterval = Duration.ofMinutes(5); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java deleted file mode 100644 index 0245f2a92a3..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Deployer; -import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeType; -import com.yahoo.log.LogLevel; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker; - -import java.time.Duration; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Automatically retires ready and active nodes if they meet a certain criteria given by the {@link RetirementPolicy} - * and if there are enough remaining nodes to both replace the retiring node as well as to keep enough in spare. - * - * @author freva - */ -public class NodeRetirer extends Maintainer { - - public static final FlavorSpareChecker.SpareNodesPolicy SPARE_NODES_POLICY = flavorSpareCount -> - flavorSpareCount.getNumReadyAmongReplacees() > 2; - - private static final long MAX_SIMULTANEOUS_RETIRES_PER_CLUSTER = 1; - private static final Logger log = Logger.getLogger(NodeRetirer.class.getName()); - - private final Deployer deployer; - private final FlavorSpareChecker flavorSpareChecker; - private final RetirementPolicy retirementPolicy; - - NodeRetirer(NodeRepository nodeRepository, FlavorSpareChecker flavorSpareChecker, Duration interval, - Deployer deployer, RetirementPolicy retirementPolicy) { - super(nodeRepository, interval); - this.deployer = deployer; - this.retirementPolicy = retirementPolicy; - this.flavorSpareChecker = flavorSpareChecker; - } - - @Override - protected void maintain() { - if (! retirementPolicy.isActive()) return; - - if (retireUnallocated()) { - retireAllocated(); - } - } - - /** - * Retires unallocated nodes by moving them directly to parked. - * Returns true iff all there are no unallocated nodes that match the retirement policy - */ - boolean retireUnallocated() { - try (Mutex lock = nodeRepository().lockAllocation()) { - List<Node> allNodes = nodeRepository().getNodes(NodeType.tenant); - Map<Flavor, Map<Node.State, Long>> numSpareNodesByFlavorByState = getNumberOfNodesByFlavorByNodeState(allNodes); - flavorSpareChecker.updateReadyAndActiveCountsByFlavor(numSpareNodesByFlavorByState); - - long numFlavorsWithUnsuccessfullyRetiredNodes = allNodes.stream() - .filter(node -> node.state() == Node.State.ready) - .filter(node -> retirementPolicy.shouldRetire(node).isPresent()) - .collect(Collectors.groupingBy( - Node::flavor, - Collectors.toSet())) - .entrySet().stream() - .filter(entry -> { - Set<Node> nodesThatShouldBeRetiredForFlavor = entry.getValue(); - for (Iterator<Node> iter = nodesThatShouldBeRetiredForFlavor.iterator(); iter.hasNext(); ) { - Node nodeToRetire = iter.next(); - if (! flavorSpareChecker.canRetireUnallocatedNodeWithFlavor(nodeToRetire.flavor())) break; - - retirementPolicy.shouldRetire(nodeToRetire).ifPresent(reason -> { - nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToDeprovision(true)), lock); - nodeRepository().park(nodeToRetire.hostname(), false, Agent.NodeRetirer, reason); - iter.remove(); - }); - } - - if (! nodesThatShouldBeRetiredForFlavor.isEmpty()) { - String commaSeparatedHostnames = nodesThatShouldBeRetiredForFlavor.stream().map(Node::hostname) - .collect(Collectors.joining(", ")); - log.info(String.format("Failed to retire %s, wanted to retire %d nodes (%s), but there are no spare nodes left.", - entry.getKey(), nodesThatShouldBeRetiredForFlavor.size(), commaSeparatedHostnames)); - } - return ! nodesThatShouldBeRetiredForFlavor.isEmpty(); - }).count(); - - return numFlavorsWithUnsuccessfullyRetiredNodes == 0; - } - } - - void retireAllocated() { - List<Node> allNodes = nodeRepository().getNodes(NodeType.tenant); - List<ApplicationId> activeApplications = getActiveApplicationIds(allNodes); - Map<Flavor, Map<Node.State, Long>> numSpareNodesByFlavorByState = getNumberOfNodesByFlavorByNodeState(allNodes); - flavorSpareChecker.updateReadyAndActiveCountsByFlavor(numSpareNodesByFlavorByState); - - // Get all the nodes that we could retire along with their deployments - Map<Deployment, Set<Node>> nodesToRetireByDeployment = new HashMap<>(); - for (ApplicationId applicationId : activeApplications) { - Map<ClusterSpec.Id, Set<Node>> nodesByCluster = getNodesBelongingToApplication(allNodes, applicationId).stream() - .collect(Collectors.groupingBy( - node -> node.allocation().get().membership().cluster().id(), - Collectors.toSet())); - Map<ClusterSpec.Id, Set<Node>> retireableNodesByCluster = nodesByCluster.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> filterRetireableNodes(entry.getValue()))); - if (retireableNodesByCluster.values().stream().mapToInt(Set::size).sum() == 0) continue; - - Optional<Deployment> deployment = deployer.deployFromLocalActive(applicationId); - if ( ! deployment.isPresent()) continue; // this will be done at another config server - - Set<Node> replaceableNodes = retireableNodesByCluster.entrySet().stream() - .flatMap(entry -> entry.getValue().stream() - .filter(node -> flavorSpareChecker.canRetireAllocatedNodeWithFlavor(node.flavor())) - .limit(getNumberNodesAllowToRetireForCluster(nodesByCluster.get(entry.getKey()), MAX_SIMULTANEOUS_RETIRES_PER_CLUSTER))) - .collect(Collectors.toSet()); - if (! replaceableNodes.isEmpty()) nodesToRetireByDeployment.put(deployment.get(), replaceableNodes); - } - - nodesToRetireByDeployment.forEach(((deployment, nodes) -> { - ApplicationId app = nodes.iterator().next().allocation().get().owner(); - Set<Node> nodesToRetire; - - // While under application lock, get up-to-date node, and make sure that the state and the owner of the - // node has not changed in the meantime, mutate the up-to-date node (so to not overwrite other fields - // that may have changed) with wantToRetire and wantToDeprovision. - try (Mutex lock = nodeRepository().lock(app)) { - nodesToRetire = nodes.stream() - .map(node -> - nodeRepository().getNode(node.hostname()) - .filter(upToDateNode -> node.state() == Node.State.active) - .filter(upToDateNode -> node.allocation().get().owner().equals(upToDateNode.allocation().get().owner()))) - .flatMap(node -> node.map(Stream::of).orElseGet(Stream::empty)) - .collect(Collectors.toSet()); - - nodesToRetire.forEach(node -> - retirementPolicy.shouldRetire(node).ifPresent(reason -> { - log.info("Setting wantToRetire and wantToDeprovision for host " + node.hostname() + - " with flavor " + node.flavor().name() + - " allocated to " + node.allocation().get().owner() + ". Reason: " + reason); - - Node updatedNode = node.with(node.status() - .withWantToRetire(true) - .withWantToDeprovision(true)); - nodeRepository().write(updatedNode, lock); - })); - } - - // This takes a while, so do it outside of the application lock - if (! nodesToRetire.isEmpty()) { - try { - deployment.activate(); - } catch (Exception e) { - log.log(LogLevel.INFO, "Failed to redeploy " + app.serializedForm() + ", will be redeployed later by application maintainer", e); - } - } - })); - } - - private List<Node> getNodesBelongingToApplication(Collection<Node> allNodes, ApplicationId applicationId) { - return allNodes.stream() - .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(applicationId)) - .collect(Collectors.toList()); - } - - /** - * Returns a list of ApplicationIds sorted by number of active nodes the application has allocated to it - */ - List<ApplicationId> getActiveApplicationIds(Collection<Node> nodes) { - return nodes.stream() - .filter(node -> node.state() == Node.State.active) - .collect(Collectors.groupingBy( - node -> node.allocation().get().owner(), - Collectors.counting())) - .entrySet().stream() - .sorted((c1, c2) -> c2.getValue().compareTo(c1.getValue())) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - /** - * @param nodes Collection of nodes that are considered for retirement - * @return Set of nodes that all should eventually be retired - */ - Set<Node> filterRetireableNodes(Collection<Node> nodes) { - return nodes.stream() - .filter(node -> node.state() == Node.State.active) - .filter(node -> !node.status().wantToRetire()) - .filter(node -> retirementPolicy.shouldRetire(node).isPresent()) - .collect(Collectors.toSet()); - } - - /** - * @param clusterNodes All the nodes allocated to an application belonging to a single cluster - * @return number of nodes we can safely start retiring - */ - long getNumberNodesAllowToRetireForCluster(Collection<Node> clusterNodes, long maxSimultaneousRetires) { - long numNodesInWantToRetire = clusterNodes.stream() - .filter(node -> node.status().wantToRetire()) - .filter(node -> node.state() != Node.State.parked) - .count(); - return Math.max(0, maxSimultaneousRetires - numNodesInWantToRetire); - } - - private Map<Flavor, Map<Node.State, Long>> getNumberOfNodesByFlavorByNodeState(Collection<Node> allNodes) { - return allNodes.stream() - .collect(Collectors.groupingBy( - Node::flavor, - Collectors.groupingBy(Node::state, Collectors.counting()))); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java deleted file mode 100644 index 6562a89c2d6..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance.retire; - -import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.provision.Node; - -import java.net.Inet4Address; -import java.util.Optional; - -/** - * @author freva - */ -public class RetireIPv4OnlyNodes implements RetirementPolicy { - private final Zone zone; - - public RetireIPv4OnlyNodes(Zone zone) { - this.zone = zone; - } - - @Override - public boolean isActive() { - if(zone.system() == SystemName.cd) { - return zone.environment() == Environment.dev || zone.environment() == Environment.prod; - } - - if (zone.system() == SystemName.main) { - if (zone.region().equals(RegionName.from("us-east-3"))) { - return zone.environment() == Environment.perf || zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("us-west-1"))) { - return zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("us-central-1"))) { - return zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("ap-southeast-1"))) { - return zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("ap-northeast-1"))) { - return zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("ap-northeast-2"))) { - return zone.environment() == Environment.prod; - } else if (zone.region().equals(RegionName.from("eu-west-1"))) { - return zone.environment() == Environment.prod; - } - } - - return false; - } - - @Override - public Optional<String> shouldRetire(Node node) { - if (node.flavor().getType() == Flavor.Type.VIRTUAL_MACHINE) return Optional.empty(); - boolean shouldRetire = node.ipAddresses().stream() - .map(InetAddresses::forString) - .allMatch(address -> address instanceof Inet4Address); - - return shouldRetire ? Optional.of("Node is IPv4-only") : Optional.empty(); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java deleted file mode 100644 index ca0419f11c3..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance.retire; - -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.Optional; - -/** - * @author freva - */ -public interface RetirementPolicy { - - /** - * Returns whether the policy is currently active. NodeRetirer ask every time before executing. - */ - boolean isActive(); - - /** - * Returns reason for retiring the node, empty if node should not be retired - */ - Optional<String> shouldRetire(Node node); -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyCache.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyCache.java deleted file mode 100644 index c112daadcc9..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyCache.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance.retire; - -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.Optional; - -/** - * @author freva - */ -public class RetirementPolicyCache implements RetirementPolicy { - private final RetirementPolicy policy; - private final boolean isActiveCached; - - RetirementPolicyCache(RetirementPolicy policy) { - this.policy = policy; - this.isActiveCached = policy.isActive(); - } - - @Override - public boolean isActive() { - return isActiveCached; - } - - public Optional<String> shouldRetire(Node node) { - return policy.shouldRetire(node); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyList.java deleted file mode 100644 index 5f4d887b029..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicyList.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance.retire; - -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class RetirementPolicyList implements RetirementPolicy { - private final List<RetirementPolicy> retirementPolicies; - - public RetirementPolicyList(RetirementPolicy... retirementPolicies) { - this.retirementPolicies = Stream.of(retirementPolicies) - .map(RetirementPolicyCache::new) - .collect(Collectors.toList()); - } - - @Override - public boolean isActive() { - return retirementPolicies.stream().anyMatch(RetirementPolicy::isActive); - } - - @Override - public Optional<String> shouldRetire(Node node) { - List<String> retirementReasons = retirementPolicies.stream() - .filter(RetirementPolicy::isActive) - .map(retirementPolicy -> retirementPolicy.shouldRetire(node)) - .flatMap(reason -> reason.map(Stream::of).orElse(Stream.empty())) - .collect(Collectors.toList()); - - return retirementReasons.isEmpty() ? Optional.empty() : - Optional.of("[" + String.join(", ", retirementReasons) + "]"); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 371ed4d2496..61ca19a4cb9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -484,7 +484,7 @@ public class CuratorDatabaseClient { } private Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { - return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson); + return read(loadBalancerPath(id), (data) -> LoadBalancerSerializer.fromJson(data, clock.instant())); } public void writeLoadBalancer(LoadBalancer loadBalancer) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index fd2294c1b5d..d04dd2b5c18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -15,9 +15,9 @@ import com.yahoo.vespa.hosted.provision.lb.Real; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import java.util.LinkedHashSet; import java.util.Optional; -import java.util.Set; import java.util.function.Function; /** @@ -36,12 +36,13 @@ public class LoadBalancerSerializer { private static final String idField = "id"; private static final String hostnameField = "hostname"; + private static final String stateField = "state"; + private static final String changedAtField = "changedAt"; private static final String dnsZoneField = "dnsZone"; private static final String inactiveField = "inactive"; private static final String portsField = "ports"; private static final String networksField = "networks"; private static final String realsField = "reals"; - private static final String nameField = "name"; private static final String ipAddressField = "ipAddress"; private static final String portField = "port"; @@ -51,6 +52,8 @@ public class LoadBalancerSerializer { root.setString(idField, loadBalancer.id().serializedForm()); root.setString(hostnameField, loadBalancer.instance().hostname().toString()); + root.setString(stateField, asString(loadBalancer.state())); + root.setLong(changedAtField, loadBalancer.changedAt().toEpochMilli()); loadBalancer.instance().dnsZone().ifPresent(dnsZone -> root.setString(dnsZoneField, dnsZone.id())); Cursor portArray = root.setArray(portsField); loadBalancer.instance().ports().forEach(portArray::addLong); @@ -63,8 +66,6 @@ public class LoadBalancerSerializer { realObject.setString(ipAddressField, real.ipAddress()); realObject.setLong(portField, real.port()); }); - root.setBool(inactiveField, loadBalancer.inactive()); - try { return SlimeUtils.toJsonBytes(slime); } catch (IOException e) { @@ -72,10 +73,10 @@ public class LoadBalancerSerializer { } } - public static LoadBalancer fromJson(byte[] data) { + public static LoadBalancer fromJson(byte[] data, Instant defaultChangedAt) { Cursor object = SlimeUtils.jsonToSlime(data).get(); - Set<Real> reals = new LinkedHashSet<>(); + var reals = new LinkedHashSet<Real>(); object.field(realsField).traverse((ArrayTraverser) (i, realObject) -> { reals.add(new Real(HostName.from(realObject.field(hostnameField).asString()), realObject.field(ipAddressField).asString(), @@ -83,25 +84,61 @@ public class LoadBalancerSerializer { }); - Set<Integer> ports = new LinkedHashSet<>(); + var ports = new LinkedHashSet<Integer>(); object.field(portsField).traverse((ArrayTraverser) (i, port) -> ports.add((int) port.asLong())); - Set<String> networks = new LinkedHashSet<>(); + var networks = new LinkedHashSet<String>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), new LoadBalancerInstance( HostName.from(object.field(hostnameField).asString()), - optionalField(object.field(dnsZoneField), DnsZone::new), + optionalString(object.field(dnsZoneField), DnsZone::new), ports, networks, reals ), - object.field(inactiveField).asBool()); + stateFromSlime(object), + instantFromSlime(object.field(changedAtField), defaultChangedAt)); + } + + private static Instant instantFromSlime(Cursor field, Instant defaultValue) { + return optionalValue(field, (value) -> Instant.ofEpochMilli(value.asLong())).orElse(defaultValue); + } + + private static LoadBalancer.State stateFromSlime(Inspector object) { + var inactiveValue = optionalValue(object.field(inactiveField), Inspector::asBool); + if (inactiveValue.isPresent()) { // TODO(mpolden): Remove reading of "inactive" field after June 2019 + return inactiveValue.get() ? LoadBalancer.State.inactive : LoadBalancer.State.active; + } else { + return stateFromString(object.field(stateField).asString()); + } + } + + private static <T> Optional<T> optionalValue(Inspector field, Function<Inspector, T> fieldMapper) { + return Optional.of(field).filter(Inspector::valid).map(fieldMapper); + } + + private static <T> Optional<T> optionalString(Inspector field, Function<String, T> fieldMapper) { + return optionalValue(field, Inspector::asString).map(fieldMapper); } - private static <T> Optional<T> optionalField(Inspector field, Function<String, T> fieldMapper) { - return Optional.of(field).filter(Inspector::valid).map(Inspector::asString).map(fieldMapper); + private static String asString(LoadBalancer.State state) { + switch (state) { + case active: return "active"; + case inactive: return "inactive"; + case reserved: return "reserved"; + default: throw new IllegalArgumentException("No serialization defined for state enum '" + state + "'"); + } + } + + private static LoadBalancer.State stateFromString(String state) { + switch (state) { + case "active": return LoadBalancer.State.active; + case "inactive": return LoadBalancer.State.inactive; + case "reserved": return LoadBalancer.State.reserved; + default: throw new IllegalArgumentException("No serialization defined for state string '" + state + "'"); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 424889caf72..45fb1e050a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -385,7 +385,7 @@ public class NodeSerializer { case "application" : return Agent.application; case "system" : return Agent.system; case "operator" : return Agent.operator; - case "NodeRetirer" : return Agent.NodeRetirer; + case "NodeRetirer" : return Agent.system; // TODO: Remove after 7.67 case "NodeFailer" : return Agent.NodeFailer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); @@ -395,7 +395,7 @@ public class NodeSerializer { case application : return "application"; case system : return "system"; case operator : return "operator"; - case NodeRetirer : return "NodeRetirer"; + case NodeRetirer : return "system"; // TODO: Remove after 7.67 case NodeFailer : return "NodeFailer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 4626a600d2c..1e83c2c9176 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.transaction.Mutex; @@ -22,16 +24,26 @@ import java.util.function.Function; import java.util.stream.Collectors; /** - * Performs activation of nodes for an application + * Performs activation of resources for an application. E.g. nodes or load balancers. * * @author bratseth */ class Activator { private final NodeRepository nodeRepository; + private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; - public Activator(NodeRepository nodeRepository) { + public Activator(NodeRepository nodeRepository, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { this.nodeRepository = nodeRepository; + this.loadBalancerProvisioner = loadBalancerProvisioner; + } + + /** Activate required resources for given application */ + public void activate(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction) { + try (Mutex lock = nodeRepository.lock(application)) { + activateNodes(application, hosts, transaction, lock); + activateLoadBalancers(application, hosts, lock); + } } /** @@ -46,36 +58,50 @@ class Activator { * @param transaction Transaction with operations to commit together with any operations done within the repository. * @param application the application to allocate nodes for * @param hosts the hosts to make the set of active nodes of this + * @param applicationLock application lock that must be held when calling this */ - public void activate(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction) { - try (Mutex lock = nodeRepository.lock(application)) { - Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); - NodeList allNodes = nodeRepository.list(); - NodeList applicationNodes = allNodes.owner(application); - - List<Node> reserved = applicationNodes.state(Node.State.reserved).asList(); - List<Node> reservedToActivate = retainHostsInList(hostnames, reserved); - List<Node> active = applicationNodes.state(Node.State.active).asList(); - List<Node> continuedActive = retainHostsInList(hostnames, active); - List<Node> allActive = new ArrayList<>(continuedActive); - allActive.addAll(reservedToActivate); - if ( ! containsAll(hostnames, allActive)) - throw new IllegalArgumentException("Activation of " + application + " failed. " + - "Could not find all requested hosts." + - "\nRequested: " + hosts + - "\nReserved: " + toHostNames(reserved) + - "\nActive: " + toHostNames(active) + - "\nThis might happen if the time from reserving host to activation takes " + - "longer time than reservation expiry (the hosts will then no longer be reserved)"); - - validateParentHosts(application, allNodes, reservedToActivate); - - List<Node> activeToRemove = removeHostsFromList(hostnames, active); - activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired - nodeRepository.deactivate(activeToRemove, transaction); - nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes - nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); - } + private void activateNodes(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, + @SuppressWarnings("unused") Mutex applicationLock) { + Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); + NodeList allNodes = nodeRepository.list(); + NodeList applicationNodes = allNodes.owner(application); + + List<Node> reserved = applicationNodes.state(Node.State.reserved).asList(); + List<Node> reservedToActivate = retainHostsInList(hostnames, reserved); + List<Node> active = applicationNodes.state(Node.State.active).asList(); + List<Node> continuedActive = retainHostsInList(hostnames, active); + List<Node> allActive = new ArrayList<>(continuedActive); + allActive.addAll(reservedToActivate); + if (!containsAll(hostnames, allActive)) + throw new IllegalArgumentException("Activation of " + application + " failed. " + + "Could not find all requested hosts." + + "\nRequested: " + hosts + + "\nReserved: " + toHostNames(reserved) + + "\nActive: " + toHostNames(active) + + "\nThis might happen if the time from reserving host to activation takes " + + "longer time than reservation expiry (the hosts will then no longer be reserved)"); + + validateParentHosts(application, allNodes, reservedToActivate); + + List<Node> activeToRemove = removeHostsFromList(hostnames, active); + activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired + nodeRepository.deactivate(activeToRemove, transaction); + nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes + nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); + } + + /** Activate load balancers */ + private void activateLoadBalancers(ApplicationId application, Collection<HostSpec> hosts, + @SuppressWarnings("unused") Mutex applicationLock) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, clustersOf(hosts))); + } + + private static List<ClusterSpec> clustersOf(Collection<HostSpec> hosts) { + return hosts.stream() + .map(HostSpec::membership) + .flatMap(Optional::stream) + .map(ClusterMembership::cluster) + .collect(Collectors.toUnmodifiableList()); } private static void validateParentHosts(ApplicationId application, NodeList nodes, List<Node> potentialChildren) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index 77872fc1435..fbf97ba25d9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provisioning.FlavorsConfig; /** * Simplifies creation of a node-repository config containing flavors. @@ -22,7 +22,6 @@ public class FlavorConfigBuilder { public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); - flavor.description("Flavor-name-is-" + flavorName); flavor.minDiskAvailableGb(disk); flavor.minCpuCores(cpu); flavor.minMainMemoryAvailableGb(mem); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java deleted file mode 100644 index 5f81fed2a04..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * This class helps answer the question if there are enough nodes to retire a node with flavor f by: - * <ul> - * <li>Finding all the possible flavors that the replacement node could end up on</li> - * <li>Making sure that regardless of which flavor it ends up on, there is still enough spare nodes - * to handle at unexpected node failures.</li> - * </ul> - * <p> - * Definitions: - * <ul> - * <li>Wanted flavor: The flavor that is the node prefers, for example by specifying in services.xml</li> - * <li>Node-repo flavor: The flavor that the node actually has (Either the wanted flavor or a flavor that transitively - * replaces the wanted flavor)</li> - * <li>Replacee flavor: Flavor x is replacee of y iff x transitively replaces y</li> - * <li>Immediate replacee flavor: Flavor x is an immediate replacee of flavor y iff x directly replaces y.</li> - * </ul> - * - * @author freva - */ -public class FlavorSpareChecker { - - private final SpareNodesPolicy spareNodesPolicy; - private final Map<Flavor, FlavorSpareCount> spareCountByFlavor; - - public FlavorSpareChecker(SpareNodesPolicy spareNodesPolicy, Map<Flavor, FlavorSpareCount> spareCountByFlavor) { - this.spareNodesPolicy = spareNodesPolicy; - this.spareCountByFlavor = spareCountByFlavor; - } - - public void updateReadyAndActiveCountsByFlavor(Map<Flavor, Map<Node.State, Long>> numberOfNodesByFlavorByState) { - spareCountByFlavor.forEach((flavor, flavorSpareCount) -> { - Map<Node.State, Long> numberOfNodesByState = numberOfNodesByFlavorByState.getOrDefault(flavor, Collections.emptyMap()); - flavorSpareCount.updateReadyAndActiveCounts( - numberOfNodesByState.getOrDefault(Node.State.ready, 0L), - numberOfNodesByState.getOrDefault(Node.State.active, 0L)); - }); - } - - public boolean canRetireAllocatedNodeWithFlavor(Flavor flavor) { - Set<FlavorSpareCount> possibleNewFlavors = findPossibleReplacementFlavorFor(spareCountByFlavor.get(flavor)); - possibleNewFlavors.forEach(FlavorSpareCount::decrementNumberOfReady); - return !possibleNewFlavors.isEmpty(); - } - - public boolean canRetireUnallocatedNodeWithFlavor(Flavor flavor) { - FlavorSpareCount flavorSpareCount = spareCountByFlavor.get(flavor); - if (flavorSpareCount.hasReady() && spareNodesPolicy.hasSpare(flavorSpareCount)) { - flavorSpareCount.decrementNumberOfReady(); - return true; - } - - return false; - } - - - /** - * Returns a set of possible new flavors that can replace this flavor given current node allocation. - * If the set is empty, there are not enough spare nodes to safely retire this flavor. - * <p> - * The algorithm is: - * for all possible wanted flavor, check: - * <ul> - * <li>1: Sum of ready nodes of flavor f and all replacee flavors of f is > reserved (set by {@link SpareNodesPolicy}</li> - * <li>2a: Number of ready nodes of flavor f is > 0</li> - * <li>2b: Verify 1 & 2 for all immediate replacee of f, f_i, where sum of ready nodes of f_i and all - * replacee flavors of f_i is > 0</li> - * </ul> - * Only 2a OR 2b need to be satisfied. - */ - private Set<FlavorSpareCount> findPossibleReplacementFlavorFor(FlavorSpareCount flavorSpareCount) { - Set<FlavorSpareCount> possibleReplacementFlavors = new HashSet<>(); - for (FlavorSpareCount possibleWantedFlavor : flavorSpareCount.getPossibleWantedFlavors()) { - Set<FlavorSpareCount> replacementFlavors = verifyReplacementConditions(possibleWantedFlavor); - if (replacementFlavors.isEmpty()) return Collections.emptySet(); - else possibleReplacementFlavors.addAll(replacementFlavors); - } - - return possibleReplacementFlavors; - } - - private Set<FlavorSpareCount> verifyReplacementConditions(FlavorSpareCount flavorSpareCount) { - Set<FlavorSpareCount> possibleReplacementFlavors = new HashSet<>(); - // Breaks condition 1, end - if (! spareNodesPolicy.hasSpare(flavorSpareCount)) return Collections.emptySet(); - - // Condition 2a - if (flavorSpareCount.hasReady()) { - possibleReplacementFlavors.add(flavorSpareCount); - - // Condition 2b - } else { - for (FlavorSpareCount possibleNewFlavor : flavorSpareCount.getImmediateReplacees()) { - if (possibleNewFlavor.getNumReadyAmongReplacees() == 0) continue; - - Set<FlavorSpareCount> replacementFlavors = verifyReplacementConditions(possibleNewFlavor); - if (replacementFlavors.isEmpty()) return Collections.emptySet(); - else possibleReplacementFlavors.addAll(replacementFlavors); - } - } - return possibleReplacementFlavors; - } - - public interface SpareNodesPolicy { - boolean hasSpare(FlavorSpareCount flavorSpareCount); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java deleted file mode 100644 index 217f4999bfb..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Keeps track of number of ready and active nodes for a flavor and its replaces neighbors - * - * @author freva - */ -public class FlavorSpareCount { - - private final Flavor flavor; - private Set<FlavorSpareCount> possibleWantedFlavors; - private Set<FlavorSpareCount> immediateReplacees; - private long numReady; - private long numActive; - - public static Map<Flavor, FlavorSpareCount> constructFlavorSpareCountGraph(List<Flavor> flavors) { - Map<Flavor, FlavorSpareCount> spareCountByFlavor = new HashMap<>(); - Map<Flavor, Set<Flavor>> immediateReplaceeFlavorsByFlavor = new HashMap<>(); - for (Flavor flavor : flavors) { - for (Flavor replaces : flavor.replaces()) { - if (! immediateReplaceeFlavorsByFlavor.containsKey(replaces)) { - immediateReplaceeFlavorsByFlavor.put(replaces, new HashSet<>()); - } - immediateReplaceeFlavorsByFlavor.get(replaces).add(flavor); - } - - spareCountByFlavor.put(flavor, new FlavorSpareCount(flavor)); - } - - spareCountByFlavor.forEach((flavor, flavorSpareCount) -> { - flavorSpareCount.immediateReplacees = ! immediateReplaceeFlavorsByFlavor.containsKey(flavor) ? - Collections.emptySet() : - immediateReplaceeFlavorsByFlavor.get(flavor).stream().map(spareCountByFlavor::get).collect(Collectors.toSet()); - flavorSpareCount.possibleWantedFlavors = recursiveReplacements(flavor, new HashSet<>()) - .stream().map(spareCountByFlavor::get).collect(Collectors.toSet()); - }); - - return spareCountByFlavor; - } - - private static Set<Flavor> recursiveReplacements(Flavor flavor, Set<Flavor> replacements) { - replacements.add(flavor); - for (Flavor replaces : flavor.replaces()) { - recursiveReplacements(replaces, replacements); - } - - return replacements; - } - - private FlavorSpareCount(Flavor flavor) { - this.flavor = flavor; - } - - public Flavor getFlavor() { - return flavor; - } - - void updateReadyAndActiveCounts(long numReady, long numActive) { - this.numReady = numReady; - this.numActive = numActive; - } - - boolean hasReady() { - return numReady > 0; - } - - public long getNumReadyAmongReplacees() { - long sumReadyNodes = numReady; - for (FlavorSpareCount replacee : immediateReplacees) { - sumReadyNodes += replacee.getNumReadyAmongReplacees(); - } - - return sumReadyNodes; - } - - Set<FlavorSpareCount> getPossibleWantedFlavors() { - return possibleWantedFlavors; - } - - Set<FlavorSpareCount> getImmediateReplacees() { - return immediateReplacees; - } - - void decrementNumberOfReady() { - numReady--; - } - - @Override - public String toString() { - return flavor.name() + " has " + numReady + " ready nodes and " + numActive + " active nodes"; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 372dca84a53..b74972458de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -18,21 +19,26 @@ import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; -import java.util.Collections; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Provides provisioning of load balancers for applications. + * Provisions and configures application load balancers. * * @author mpolden */ +// Load balancer state transitions: +// 1) (new) -> reserved -> active +// 2) active | reserved -> inactive +// 3) inactive -> active | (removed) public class LoadBalancerProvisioner { + private static final Logger log = Logger.getLogger(LoadBalancerProvisioner.class.getName()); + private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; @@ -44,43 +50,72 @@ public class LoadBalancerProvisioner { } /** - * Provision load balancer(s) for given application. + * Prepare a load balancer for given application and cluster. + * + * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated + * nodes. It's state will remain unchanged. + * + * If no load balancer exists, a new one will be provisioned in {@link LoadBalancer.State#reserved}. * - * If the application has multiple container clusters, one load balancer will be provisioned for each cluster. + * Calling this for irrelevant node or cluster types is a no-op. */ - public Map<LoadBalancerId, LoadBalancer> provision(ApplicationId application) { - try (Mutex applicationLock = nodeRepository.lock(application)) { - try (Mutex loadBalancersLock = db.lockLoadBalancers()) { - Map<LoadBalancerId, LoadBalancer> loadBalancers = new LinkedHashMap<>(); - for (Map.Entry<ClusterSpec, List<Node>> kv : activeContainers(application).entrySet()) { - LoadBalancerId id = new LoadBalancerId(application, kv.getKey().id()); - LoadBalancerInstance instance = create(application, kv.getKey().id(), kv.getValue()); - // Load balancer is always re-activated here to avoid reallocation if an application/cluster is - // deleted and then redeployed. - LoadBalancer loadBalancer = new LoadBalancer(id, instance, false); - loadBalancers.put(loadBalancer.id(), loadBalancer); - db.writeLoadBalancer(loadBalancer); - } - return Collections.unmodifiableMap(loadBalancers); - } + public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { + if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type + if (cluster.type() != ClusterSpec.Type.container) return; // Nothing to provision for this cluster type + provision(application, cluster.id(), false); + } + + /** + * Activate load balancer for given application and cluster. + * + * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated + * nodes and the load balancer itself will be moved to {@link LoadBalancer.State#active}. + * + * Calling this when no load balancer has been prepared for given cluster is a no-op. + */ + public void activate(ApplicationId application, List<ClusterSpec> clusters) { + for (var clusterId : containerClusterIdsOf(clusters)) { + // Provision again to ensure that load balancer instance re-configured with correct nodes + provision(application, clusterId, true); } } /** * Deactivate all load balancers assigned to given application. This is a no-op if an application does not have any - * load balancer(s) + * load balancer(s). */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (Mutex applicationLock = nodeRepository.lock(application)) { try (Mutex loadBalancersLock = db.lockLoadBalancers()) { - List<LoadBalancer> deactivatedLoadBalancers = nodeRepository.loadBalancers().owner(application).asList().stream() - .map(LoadBalancer::deactivate) - .collect(Collectors.toList()); + var now = nodeRepository.clock().instant(); + var deactivatedLoadBalancers = nodeRepository.loadBalancers().owner(application).asList().stream() + .map(lb -> lb.with(LoadBalancer.State.inactive, now)) + .collect(Collectors.toList()); db.writeLoadBalancers(deactivatedLoadBalancers, transaction); } } } + /** Idempotently provision a load balancer for given application and cluster */ + private void provision(ApplicationId application, ClusterSpec.Id clusterId, boolean activate) { + try (var applicationLock = nodeRepository.lock(application)) { + try (var loadBalancersLock = db.lockLoadBalancers()) { + var id = new LoadBalancerId(application, clusterId); + var now = nodeRepository.clock().instant(); + var instance = create(application, clusterId, allocatedContainers(application, clusterId)); + var loadBalancer = db.readLoadBalancers().get(id); + if (loadBalancer == null) { + if (activate) return; // Nothing to activate as this load balancer was never prepared + loadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); + } else { + var newState = activate ? LoadBalancer.State.active : loadBalancer.state(); + loadBalancer = loadBalancer.with(instance).with(newState, now); + } + db.writeLoadBalancer(loadBalancer); + } + } + } + private LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes) { Map<HostName, Set<String>> hostnameToIpAdresses = nodes.stream() .collect(Collectors.toMap(node -> HostName.from(node.hostname()), @@ -89,18 +124,19 @@ public class LoadBalancerProvisioner { hostnameToIpAdresses.forEach((hostname, ipAddresses) -> { ipAddresses.forEach(ipAddress -> reals.add(new Real(hostname, ipAddress))); }); + log.log(LogLevel.INFO, "Creating load balancer for " + cluster + " in " + application.toShortString() + + ", targeting: " + nodes); return service.create(application, cluster, reals); } - /** Returns a list of active containers for given application, grouped by cluster spec */ - private Map<ClusterSpec, List<Node>> activeContainers(ApplicationId application) { - return new NodeList(nodeRepository.getNodes(NodeType.tenant, Node.State.active)) + /** Returns a list of active and reserved nodes of type container in given cluster */ + private List<Node> allocatedContainers(ApplicationId application, ClusterSpec.Id clusterId) { + return new NodeList(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) .owner(application) .filter(node -> node.state().isAllocated()) .type(ClusterSpec.Type.container) - .asList() - .stream() - .collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster())); + .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterId)) + .asList(); } /** Find IP addresses reachable by the load balancer service */ @@ -118,4 +154,11 @@ public class LoadBalancerProvisioner { return reachable; } + private static List<ClusterSpec.Id> containerClusterIdsOf(List<ClusterSpec> clusters) { + return clusters.stream() + .filter(c -> c.type() == ClusterSpec.Type.container) + .map(ClusterSpec::id) + .collect(Collectors.toUnmodifiableList()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 21bfc1b6886..90ca8ef4d33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -62,14 +62,14 @@ public class NodeRepositoryProvisioner implements Provisioner { this.nodeRepository = nodeRepository; this.capacityPolicies = new CapacityPolicies(zone, flavors); this.zone = zone; + this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); this.preparer = new Preparer(nodeRepository, zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, - provisionServiceProvider.getHostProvisioner(), - provisionServiceProvider.getHostResourcesCalculator(), - Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource)); - this.activator = new Activator(nodeRepository); - this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> - new LoadBalancerProvisioner(nodeRepository, lbService)); + provisionServiceProvider.getHostProvisioner(), + provisionServiceProvider.getHostResourcesCalculator(), + Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource), + loadBalancerProvisioner); + this.activator = new Activator(nodeRepository, loadBalancerProvisioner); } /** @@ -112,14 +112,6 @@ public class NodeRepositoryProvisioner implements Provisioner { public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { validate(hosts); activator.activate(application, hosts, transaction); - transaction.onCommitted(() -> { - try { - loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.provision(application)); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to provision load balancer for application " + - application.toShortString(), e); - } - }); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index ca958f15c69..31ec964dceb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -24,15 +24,24 @@ class Preparer { private final NodeRepository nodeRepository; private final GroupPreparer groupPreparer; + private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; private final int spareCount; public Preparer(NodeRepository nodeRepository, int spareCount, Optional<HostProvisioner> hostProvisioner, - HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabled) { + HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabled, + Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { this.nodeRepository = nodeRepository; this.spareCount = spareCount; + this.loadBalancerProvisioner = loadBalancerProvisioner; this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, hostResourcesCalculator, dynamicProvisioningEnabled); } + /** Prepare all required resources for the given application and cluster */ + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { + prepareLoadBalancer(application, cluster, requestedNodes); + return prepareNodes(application, cluster, requestedNodes, wantedGroups); + } + /** * Ensure sufficient nodes are reserved or active for the given application and cluster * @@ -41,7 +50,7 @@ class Preparer { // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { + public List<Node> prepareNodes(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { List<Node> surplusNodes = findNodesInRemovableGroups(application, cluster, wantedGroups); MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster)); @@ -58,6 +67,11 @@ class Preparer { return acceptedNodes; } + /** Prepare a load balancer for given application and cluster */ + public void prepareLoadBalancer(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.prepare(application, cluster, requestedNodes)); + } + /** * Returns a list of the nodes which are * in groups with index number above or equal the group count diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index d31834567ab..bfbf7775031 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -55,6 +55,8 @@ public class LoadBalancersResponse extends HttpResponse { loadBalancers().forEach(lb -> { Cursor lbObject = loadBalancerArray.addObject(); lbObject.setString("id", lb.id().serializedForm()); + lbObject.setString("state", lb.state().name()); + lbObject.setLong("changedAt", lb.changedAt().toEpochMilli()); lbObject.setString("application", lb.id().application().application().value()); lbObject.setString("tenant", lb.id().application().tenant().value()); lbObject.setString("instance", lb.id().application().instance().value()); @@ -76,9 +78,9 @@ public class LoadBalancersResponse extends HttpResponse { realObject.setLong("port", real.port()); }); - lbObject.setArray("rotations"); // To avoid changing the API. This can be removed when clients stop expecting this - - lbObject.setBool("inactive", lb.inactive()); + // TODO(mpolden): The following fields preserves API compatibility. These can be removed once clients stop expecting them + lbObject.setArray("rotations"); + lbObject.setBool("inactive", lb.state() == LoadBalancer.State.inactive); }); new JsonFormat(true).encode(stream, slime); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 2ab916e6375..a591217f5d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -148,8 +148,6 @@ class NodesResponse extends HttpResponse { object.setString("canonicalFlavor", node.flavor().canonicalName()); object.setDouble("minDiskAvailableGb", node.flavor().getMinDiskAvailableGb()); object.setDouble("minMainMemoryAvailableGb", node.flavor().getMinMainMemoryAvailableGb()); - if (node.flavor().getDescription() != null && ! node.flavor().getDescription().isEmpty()) - object.setString("description", node.flavor().getDescription()); object.setDouble("minCpuCores", node.flavor().getMinCpuCores()); if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index d7942cdb6e7..33faf1eaf76 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -21,6 +21,8 @@ import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -31,7 +33,7 @@ public class LoadBalancerExpirerTest { private ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test - public void test_maintain() { + public void test_remove_inactive() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); @@ -49,25 +51,67 @@ public class LoadBalancerExpirerTest { // Remove one application deactivates load balancers for that application removeApplication(app1); - assertTrue(loadBalancers.get().get(lb1).inactive()); - assertFalse(loadBalancers.get().get(lb2).inactive()); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state()); + assertNotSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb2).state()); // Expirer defers removal while nodes are still allocated to application expirer.maintain(); assertEquals(2, tester.loadBalancerService().instances().size()); - - // Expirer removes load balancers once nodes are deallocated dirtyNodesOf(app1); + + // Expirer defers removal until expiration time passes + expirer.maintain(); + assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1)); + + // Expirer removes load balancers once expiration time passes + tester.clock().advance(Duration.ofHours(1)); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1)); // Active load balancer is left alone - assertFalse(loadBalancers.get().get(lb2).inactive()); + assertSame(LoadBalancer.State.active, loadBalancers.get().get(lb2).state()); assertTrue("Active load balancer is not removed", tester.loadBalancerService().instances().containsKey(lb2)); } + @Test + public void test_expire_reserved() { + LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), + Duration.ofDays(1), + tester.loadBalancerService()); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + + + // Prepare application + ClusterSpec.Id cluster = ClusterSpec.Id.from("qrs"); + ApplicationId app = tester.makeApplicationId(); + LoadBalancerId lb = new LoadBalancerId(app, cluster); + deployApplication(app, cluster, false); + + // Provisions load balancer in reserved + assertSame(LoadBalancer.State.reserved, loadBalancers.get().get(lb).state()); + + // Expirer does nothing + expirer.maintain(); + assertSame(LoadBalancer.State.reserved, loadBalancers.get().get(lb).state()); + + // Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout + dirtyNodesOf(app); + tester.clock().advance(Duration.ofHours(1)); + expirer.maintain(); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); + + // Expirer does nothing as inactive expiration time has not yet passed + expirer.maintain(); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); + + // Expirer removes inactive load balancer + tester.clock().advance(Duration.ofHours(1)); + expirer.maintain(); + assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb)); + } + private void dirtyNodesOf(ApplicationId application) { - tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, "unit-test"); + tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName()); } private void removeApplication(ApplicationId application) { @@ -77,12 +121,18 @@ public class LoadBalancerExpirerTest { } private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) { + deployApplication(application, cluster, true); + } + + private void deployApplication(ApplicationId application, ClusterSpec.Id cluster, boolean activate) { tester.makeReadyNodes(10, "d-1-1-1"); List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, Vtag.currentVersion, false, Collections.emptySet()), 2, 1, new NodeResources(1, 1, 1)); - tester.activate(application, hosts); + if (activate) { + tester.activate(application, hosts); + } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java deleted file mode 100644 index 93e44164f40..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; -import com.yahoo.vespa.hosted.provision.node.Agent; -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class NodeRetirerTest { - - private NodeRetirerTester tester; - private NodeRetirer retirer; - private final RetirementPolicy policy = mock(RetirementPolicy.class); - - @Before - public void setup() { - doAnswer(invoke -> { - boolean shouldRetire = ((Node) invoke.getArguments()[0]).ipAddresses().equals(Collections.singleton("::1")); - return shouldRetire ? Optional.of("Some reason") : Optional.empty(); - }).when(policy).shouldRetire(any(Node.class)); - when(policy.isActive()).thenReturn(true); - - NodeFlavors nodeFlavors = NodeRetirerTester.makeFlavors(5); - tester = new NodeRetirerTester(nodeFlavors); - retirer = spy(tester.makeNodeRetirer(policy)); - - tester.createReadyNodesByFlavor(21, 42, 27, 15, 8); - tester.deployApp("vespa", "calendar", new int[]{3}, new int[]{7}); - tester.deployApp("vespa", "notes", new int[]{0}, new int[]{3}); - tester.deployApp("sports", "results", new int[]{0}, new int[]{6}); - tester.deployApp("search", "images", new int[]{3}, new int[]{4}); - tester.deployApp("search", "videos", new int[]{2}, new int[]{2}); - tester.deployApp("tester", "my-app", new int[]{1, 2}, new int[]{4, 6}); - } - - @Test - public void testRetireUnallocated() { - tester.assertCountsForStateByFlavor(Node.State.ready, 12, 38, 19, 4, 8); - tester.setNumberAllowedUnallocatedRetirementsPerFlavor(6, 30, 15, 2, 4); - assertFalse(retirer.retireUnallocated()); - tester.assertCountsForStateByFlavor(Node.State.parked, 6, 30, 15, 2, 4); - - tester.assertCountsForStateByFlavor(Node.State.ready, 6, 8, 4, 2, 4); - tester.setNumberAllowedUnallocatedRetirementsPerFlavor(10, 20, 5, 5, 4); - assertTrue(retirer.retireUnallocated()); - tester.assertCountsForStateByFlavor(Node.State.parked, 12, 38, 19, 4, 8); - - tester.nodeRepository.getNodes().forEach(node -> - assertEquals(node.status().wantToDeprovision(), node.state() == Node.State.parked)); - } - - @Test - public void testRetireAllocated() { - // Update IP addresses on ready nodes so that when they are deployed to, we wont retire them - tester.nodeRepository.getNodes(Node.State.ready) - .forEach(node -> tester.nodeRepository.write(node.with(node.ipConfig().with(Set.of("::2"))), () -> {})); - - tester.assertCountsForStateByFlavor(Node.State.active, 9, 4, 8, 11, -1); - - tester.setNumberAllowedAllocatedRetirementsPerFlavor(3, 2, 4, 2); - retirer.retireAllocated(); - tester.assertParkedCountsByApplication(-1, -1, -1, -1, -1, -1); // Nodes should be in retired, but not yet parked - tester.assertRetiringCountsByApplication(1, 1, 1, 1, 1, 2); - - // Until the nodes we set to retire are fully retired and moved to parked, we should not attempt to retire any other nodes - retirer.retireAllocated(); - retirer.retireAllocated(); - tester.assertRetiringCountsByApplication(1, 1, 1, 1, 1, 2); - - tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 1, 1, 1, 2); - - // We can retire 1 more of flavor-0, 1 more of flavor-1, 2 more of flavor-2: - // app 6 has the most nodes, so it gets to retire flavor-1 and flavor-2 - // app 3 is the largest that is on flavor-0, so it gets the last node - // app 5 is gets the last node with flavor-2 - retirer.retireAllocated(); - tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 2, 1, 2, 4); - - // No more retirements are possible - retirer.retireAllocated(); - tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 2, 1, 2, 4); - - tester.nodeRepository.getNodes().forEach(node -> - assertEquals(node.status().wantToDeprovision(), node.state() == Node.State.parked)); - } - - @Test - public void testGetActiveApplicationIds() { - List<String> expectedOrder = Arrays.asList( - "tester.my-app", "vespa.calendar", "sports.results", "search.images", "vespa.notes", "search.videos"); - List<String> actualOrder = retirer.getActiveApplicationIds(tester.nodeRepository.getNodes()).stream() - .map(applicationId -> applicationId.toShortString().replace(":default", "")) - .collect(Collectors.toList()); - assertEquals(expectedOrder, actualOrder); - } - - @Test - public void testGetRetireableNodesForApplication() { - ApplicationId app = new ApplicationId.Builder().tenant("vespa").applicationName("calendar").build(); - - List<Node> nodes = tester.nodeRepository.getNodes(app); - Set<String> actual = retirer.filterRetireableNodes(nodes).stream().map(Node::hostname).collect(Collectors.toSet()); - Set<String> expected = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); - assertEquals(expected, actual); - - Node nodeWantToRetire = tester.nodeRepository.getNode("host3.test.yahoo.com").orElseThrow(RuntimeException::new); - tester.nodeRepository.write(nodeWantToRetire.with(nodeWantToRetire.status().withWantToRetire(true)), () -> {}); - Node nodeToFail = tester.nodeRepository.getNode("host5.test.yahoo.com").orElseThrow(RuntimeException::new); - tester.nodeRepository.fail(nodeToFail.hostname(), Agent.system, "Failed for unit testing"); - Node nodeToUpdate = tester.nodeRepository.getNode("host8.test.yahoo.com").orElseThrow(RuntimeException::new); - tester.nodeRepository.write(nodeToUpdate.with(nodeToUpdate.ipConfig().with(Set.of("::2"))), () -> {}); - - nodes = tester.nodeRepository.getNodes(app); - Set<String> excluded = Stream.of(nodeWantToRetire, nodeToFail, nodeToUpdate).map(Node::hostname).collect(Collectors.toSet()); - Set<String> actualAfterUpdates = retirer.filterRetireableNodes(nodes).stream().map(Node::hostname).collect(Collectors.toSet()); - Set<String> expectedAfterUpdates = nodes.stream().map(Node::hostname).filter(node -> !excluded.contains(node)).collect(Collectors.toSet()); - assertEquals(expectedAfterUpdates, actualAfterUpdates); - } - - @Test - public void testGetNumberNodesAllowToRetireForCluster() { - ApplicationId app = new ApplicationId.Builder().tenant("vespa").applicationName("calendar").build(); - long actualAllActive = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); - assertEquals(2, actualAllActive); - - // Lets put 3 random nodes in wantToRetire - List<Node> nodesToRetire = tester.nodeRepository.getNodes(app).stream().limit(3).collect(Collectors.toList()); - nodesToRetire.forEach(node -> tester.nodeRepository.write(node.with(node.status().withWantToRetire(true)), () -> {})); - long actualOneWantToRetire = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); - assertEquals(0, actualOneWantToRetire); - - // Now 2 of those finish retiring and go to parked - nodesToRetire.stream().limit(2).forEach(node -> - tester.nodeRepository.park(node.hostname(), false, Agent.system, "Parked for unit testing")); - long actualOneRetired = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); - assertEquals(1, actualOneRetired); - } - - @Test - public void inactivePolicyDoesNothingTest() { - when(policy.isActive()).thenReturn(false); - retirer.maintain(); - - verify(retirer, never()).retireUnallocated(); - verify(retirer, never()).retireAllocated(); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java deleted file mode 100644 index 832c2fc512b..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker; -import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; -import com.yahoo.vespa.orchestrator.OrchestrationException; -import com.yahoo.vespa.orchestrator.Orchestrator; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.LongStream; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class NodeRetirerTester { - public static final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - - // Components with state - public final ManualClock clock = new ManualClock(); - public final NodeRepository nodeRepository; - private final FlavorSpareChecker flavorSpareChecker = mock(FlavorSpareChecker.class); - private final MockDeployer deployer; - private final List<Flavor> flavors; - - // Use LinkedHashMap to keep order in which applications were deployed - private final Map<ApplicationId, MockDeployer.ApplicationContext> apps = new LinkedHashMap<>(); - - private final Orchestrator orchestrator = mock(Orchestrator.class); - private RetiredExpirer retiredExpirer; - private InactiveExpirer inactiveExpirer; - private int nextNodeId = 0; - - NodeRetirerTester(NodeFlavors nodeFlavors) { - Curator curator = new MockCurator(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); - deployer = new MockDeployer(provisioner, clock, apps); - flavors = nodeFlavors.getFlavors().stream().sorted(Comparator.comparing(Flavor::name)).collect(Collectors.toList()); - - try { - doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); - } catch (OrchestrationException e) { - e.printStackTrace(); - } - } - - NodeRetirer makeNodeRetirer(RetirementPolicy policy) { - return new NodeRetirer(nodeRepository, flavorSpareChecker, Duration.ofDays(1), deployer, policy); - } - - void createReadyNodesByFlavor(int... nums) { - List<Node> nodes = new ArrayList<>(); - for (int i = 0; i < nums.length; i++) { - Flavor flavor = flavors.get(i); - for (int j = 0; j < nums[i]; j++) { - int id = nextNodeId++; - nodes.add(nodeRepository.createNode("node" + id, "host" + id + ".test.yahoo.com", - new IP.Config(Set.of("::1"), Set.of()), Optional.empty(), - Optional.empty(), flavor, NodeType.tenant)); - } - } - - nodes = nodeRepository.addNodes(nodes); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - void deployApp(String tenantName, String applicationName, int[] flavorIds, int[] numNodes) { - final ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); - final List<MockDeployer.ClusterContext> clusterContexts = new ArrayList<>(); - - for (int i = 0; i < flavorIds.length; i++) { - Flavor flavor = flavors.get(flavorIds[i]); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("cluster-" + i), Version.fromString("6.99"), false, Collections.emptySet()); - Capacity capacity = Capacity.fromNodeCount(numNodes[i], Optional.of(flavor.name()), false, true); - // If the number of node the app wants is divisible by 2, make it into 2 groups, otherwise as 1 - int numGroups = numNodes[i] % 2 == 0 ? 2 : 1; - clusterContexts.add(new MockDeployer.ClusterContext(applicationId, cluster, capacity, numGroups)); - } - - apps.put(applicationId, new MockDeployer.ApplicationContext(applicationId, clusterContexts)); - deployer.deployFromLocalActive(applicationId, Duration.ZERO).get().activate(); - } - - void iterateMaintainers() { - if (retiredExpirer == null) { - retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, clock, Duration.ofDays(30), Duration.ofMinutes(10)); - inactiveExpirer = new InactiveExpirer(nodeRepository, clock, Duration.ofMinutes(10)); - } - - clock.advance(Duration.ofMinutes(11)); - retiredExpirer.maintain(); - - clock.advance(Duration.ofMinutes(11)); - inactiveExpirer.maintain(); - } - - void setNumberAllowedUnallocatedRetirementsPerFlavor(int... numAllowed) { - for (int i = 0; i < numAllowed.length; i++) { - Boolean[] responses = new Boolean[numAllowed[i]]; - Arrays.fill(responses, true); - responses[responses.length - 1 ] = false; - when(flavorSpareChecker.canRetireUnallocatedNodeWithFlavor(eq(flavors.get(i)))).thenReturn(true, responses); - } - } - - void setNumberAllowedAllocatedRetirementsPerFlavor(int... numAllowed) { - for (int i = 0; i < numAllowed.length; i++) { - Boolean[] responses = new Boolean[numAllowed[i]]; - Arrays.fill(responses, true); - responses[responses.length - 1] = false; - when(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(eq(flavors.get(i)))).thenReturn(true, responses); - } - } - - void assertCountsForStateByFlavor(Node.State state, long... nums) { - Map<Flavor, Long> expected = expectedCountsByFlavor(nums); - Map<Flavor, Long> actual = nodeRepository.getNodes(state).stream() - .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); - assertEquals(expected, actual); - } - - void assertParkedCountsByApplication(long... nums) { - // Nodes lose allocation when parked, so just do a sum. - long expected = LongStream.of(nums).filter(value -> value > 0L).sum(); - long actual = (long) nodeRepository.getNodes(Node.State.parked).size(); - assertEquals(expected, actual); - } - - // Nodes that are being retired or about to be retired (wantToRetire flag set), but are not yet fully retired (not parked) - void assertRetiringCountsByApplication(long... nums) { - Map<ApplicationId, Long> expected = expectedCountsByApplication(nums); - Map<ApplicationId, Long> actual = nodeRepository.getNodes().stream() - .filter(node -> node.status().wantToRetire()) - .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().membership().retired()) - .filter(node -> node.state() != Node.State.parked) - .collect(Collectors.groupingBy(node -> node.allocation().get().owner(), Collectors.counting())); - assertEquals(expected, actual); - } - - private Map<Flavor, Long> expectedCountsByFlavor(long... nums) { - Map<Flavor, Long> countsByFlavor = new HashMap<>(); - for (int i = 0; i < nums.length; i++) { - if (nums[i] < 0) continue; - Flavor flavor = flavors.get(i); - countsByFlavor.put(flavor, nums[i]); - } - return countsByFlavor; - } - - private Map<ApplicationId, Long> expectedCountsByApplication(long... nums) { - Map<ApplicationId, Long> countsByApplicationId = new HashMap<>(); - Iterator<ApplicationId> iterator = apps.keySet().iterator(); - for (int i = 0; iterator.hasNext(); i++) { - ApplicationId applicationId = iterator.next(); - if (nums[i] < 0) continue; - countsByApplicationId.put(applicationId, nums[i]); - } - return countsByApplicationId; - } - - static NodeFlavors makeFlavors(int numFlavors) { - FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); - for (int i = 0; i < numFlavors; i++) { - flavorConfigBuilder.addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); - } - return new NodeFlavors(flavorConfigBuilder.build()); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java deleted file mode 100644 index b40d091b346..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance.retire; - -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class RetireIPv4OnlyNodesTest { - private final RetireIPv4OnlyNodes policy = new RetireIPv4OnlyNodes(null); - private final List<Flavor> nodeFlavors = initFlavors(); - - @Test - public void testSingleIPv4Address() { - Node node = createNodeWithAddresses("127.0.0.1"); - assertTrue(policy.shouldRetire(node).isPresent()); - } - - @Test - public void testSingleIPv6Address() { - Node node = createNodeWithAddresses("::1"); - assertFalse(policy.shouldRetire(node).isPresent()); - } - - @Test - public void testMultipleIPv4Address() { - Node node = createNodeWithAddresses("127.0.0.1", "10.0.0.1", "192.168.0.1"); - assertTrue(policy.shouldRetire(node).isPresent()); - } - - @Test - public void testMultipleIPv6Address() { - Node node = createNodeWithAddresses("::1", "::2", "1234:5678:90ab::cdef"); - assertFalse(policy.shouldRetire(node).isPresent()); - } - - @Test - public void testCombinationAddress() { - Node node = createNodeWithAddresses("127.0.0.1", "::1", "10.0.0.1", "::2"); - assertFalse(policy.shouldRetire(node).isPresent()); - } - - @Test - public void testNeverRetireVMs() { - Node node = createVMWithAddresses("127.0.0.1", "10.0.0.1", "192.168.0.1"); - assertFalse(policy.shouldRetire(node).isPresent()); - - node = createNodeWithAddresses("::1", "::2", "1234:5678:90ab::cdef"); - assertFalse(policy.shouldRetire(node).isPresent()); - - node = createNodeWithAddresses("127.0.0.1", "::1", "10.0.0.1", "::2"); - assertFalse(policy.shouldRetire(node).isPresent()); - } - - private Node createNodeWithAddresses(String... addresses) { - Set<String> ipAddresses = Arrays.stream(addresses).collect(Collectors.toSet()); - return Node.create("openstackid", ipAddresses, Collections.emptySet(), "hostname", Optional.empty(), - Optional.empty(), nodeFlavors.get(0), NodeType.tenant); - } - - private Node createVMWithAddresses(String... addresses) { - Set<String> ipAddresses = Arrays.stream(addresses).collect(Collectors.toSet()); - return Node.create("openstackid", ipAddresses, Collections.emptySet(), "hostname", Optional.empty(), - Optional.empty(), nodeFlavors.get(1), NodeType.tenant); - } - - private List<Flavor> initFlavors() { - FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); - flavorConfigBuilder.addFlavor("default", 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); - flavorConfigBuilder.addFlavor("vm", 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.VIRTUAL_MACHINE); - return flavorConfigBuilder.build().flavor().stream().map(Flavor::new).collect(Collectors.toList()); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 460764b50db..b78b4120b81 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -12,9 +12,13 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.Real; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Optional; +import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; /** * @author mpolden @@ -23,31 +27,61 @@ public class LoadBalancerSerializerTest { @Test public void test_serialization() { - LoadBalancer loadBalancer = new LoadBalancer(new LoadBalancerId(ApplicationId.from("tenant1", - "application1", - "default"), - ClusterSpec.Id.from("qrs")), - new LoadBalancerInstance( - HostName.from("lb-host"), - Optional.of(new DnsZone("zone-id-1")), - ImmutableSet.of(4080, 4443), - ImmutableSet.of("10.2.3.4/24"), - ImmutableSet.of(new Real(HostName.from("real-1"), - "127.0.0.1", - 4080), - new Real(HostName.from("real-2"), - "127.0.0.2", - 4080))), - false); - - LoadBalancer serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + var now = Instant.now(); + var loadBalancer = new LoadBalancer(new LoadBalancerId(ApplicationId.from("tenant1", + "application1", + "default"), + ClusterSpec.Id.from("qrs")), + new LoadBalancerInstance( + HostName.from("lb-host"), + Optional.of(new DnsZone("zone-id-1")), + ImmutableSet.of(4080, 4443), + ImmutableSet.of("10.2.3.4/24"), + ImmutableSet.of(new Real(HostName.from("real-1"), + "127.0.0.1", + 4080), + new Real(HostName.from("real-2"), + "127.0.0.2", + 4080))), + LoadBalancer.State.active, + now); + + var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer), now); assertEquals(loadBalancer.id(), serialized.id()); assertEquals(loadBalancer.instance().hostname(), serialized.instance().hostname()); assertEquals(loadBalancer.instance().dnsZone(), serialized.instance().dnsZone()); assertEquals(loadBalancer.instance().ports(), serialized.instance().ports()); assertEquals(loadBalancer.instance().networks(), serialized.instance().networks()); - assertEquals(loadBalancer.inactive(), serialized.inactive()); + assertEquals(loadBalancer.state(), serialized.state()); + assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); assertEquals(loadBalancer.instance().reals(), serialized.instance().reals()); } + @Test + public void test_serialization_legacy() { // TODO(mpolden): Remove after June 2019 + var now = Instant.now(); + + var deserialized = LoadBalancerSerializer.fromJson(legacyJson(true).getBytes(StandardCharsets.UTF_8), now); + assertSame(LoadBalancer.State.inactive, deserialized.state()); + assertEquals(now, deserialized.changedAt()); + + deserialized = LoadBalancerSerializer.fromJson(legacyJson(false).getBytes(StandardCharsets.UTF_8), now); + assertSame(LoadBalancer.State.active, deserialized.state()); + } + + private static String legacyJson(boolean inactive) { + return "{\n" + + " \"id\": \"tenant1:application1:default:qrs\",\n" + + " \"hostname\": \"lb-host\",\n" + + " \"dnsZone\": \"zone-id-1\",\n" + + " \"ports\": [\n" + + " 4080,\n" + + " 4443\n" + + " ],\n" + + " \"networks\": [],\n" + + " \"reals\": [],\n" + + " \"inactive\": " + inactive + "\n" + + "}\n"; + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 50e19e15da5..1d8ca6aa57a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -386,7 +386,6 @@ public class DynamicDockerAllocationTest { tester.activate(application, hosts1); NodeResources resources = new NodeResources(1.5, 8, 50); - System.out.println("Redeploying with " + resources); List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.fromCount(2, resources), 1); tester.activate(application, hosts2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCheckerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCheckerTest.java deleted file mode 100644 index c60e1d94cac..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCheckerTest.java +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class FlavorSpareCheckerTest { - /* Creates flavors where 'replaces' graph that looks like this (largest flavor at the bottom): - * 5 - * | - * | - * 3 4 8 - * \ / \ | - * \ / \ | - * 1 6 7 - * / \ - * / \ - * 0 2 - */ - private static final List<Flavor> flavors = FlavorSpareCountTest.makeFlavors( - Collections.singletonList(1), // 0 -> {1} - Arrays.asList(3, 4), // 1 -> {3, 4} - Collections.singletonList(1), // 2 -> {1} - Collections.singletonList(5), // 3 -> {5} - Collections.emptyList(), // 4 -> {} - Collections.emptyList(), // 5 -> {} - Collections.singletonList(4), // 6 -> {4} - Collections.singletonList(8), // 7 -> {8} - Collections.emptyList()); // 8 -> {} - - private final Map<Flavor, FlavorSpareCount> flavorSpareCountByFlavor = flavors.stream() - .collect(Collectors.toMap( - i -> i, - i -> mock(FlavorSpareCount.class))); - - private final FlavorSpareChecker.SpareNodesPolicy spareNodesPolicy = mock(FlavorSpareChecker.SpareNodesPolicy.class); - private FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker(spareNodesPolicy, flavorSpareCountByFlavor); - - - @Test - public void canRetireUnallocated_Successfully() { - Flavor flavorToRetire = flavors.get(0); - FlavorSpareCount flavorSpareCount = flavorSpareCountByFlavor.get(flavorToRetire); - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - - assertTrue(flavorSpareChecker.canRetireUnallocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(0); - } - - @Test - public void canRetireUnallocated_NoReadyForFlavor() { - Flavor flavorToRetire = flavors.get(0); - FlavorSpareCount flavorSpareCount = flavorSpareCountByFlavor.get(flavorToRetire); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - - assertFalse(flavorSpareChecker.canRetireUnallocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(); - } - - @Test - public void canRetireUnallocated_NoSpareForFlavor() { - Flavor flavorToRetire = flavors.get(0); - FlavorSpareCount flavorSpareCount = flavorSpareCountByFlavor.get(flavorToRetire); - when(flavorSpareCount.hasReady()).thenReturn(true); - - assertFalse(flavorSpareChecker.canRetireUnallocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(); - } - - @Test - public void canRetireAllocated_LeafFlavor_Successfully() { - Flavor flavorToRetire = flavors.get(0); - - // If we want to retire flavor 0, then we must have enough spares & ready of flavor 0 and all - // other flavor that it replaces transitively - Stream.of(0, 1, 3, 4, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - - assertTrue(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(0, 1, 3, 4, 5); - } - - @Test - public void canRetireAllocated_LeafFlavor_NoSparesForPossibleWantedFlavor() { - Flavor flavorToRetire = flavors.get(0); - - // Flavor 4 is transitively replaced by flavor 0, even though we have enough spares of flavor 0, - // we cannot retire it if there are not enough spares of flavor 4 - Stream.of(0, 1, 3, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - - assertFalse(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(); - } - - @Test - public void canRetireAllocated_CenterNode_Successfully() { - Flavor flavorToRetire = flavors.get(1); - - Stream.of(1, 3, 4, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - - assertTrue(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(1, 3, 4, 5); - } - - @Test - public void canRetireAllocated_CenterNode_NoNodeRepoFlavorNodes_Successfully() { - Flavor flavorToRetire = flavors.get(1); - - // If we want to retire a node with node-repo flavor 1, but there are no ready nodes of flavor-1, - // we must ensure there are spare nodes of flavors that replace flavor 1 - Stream.of(0, 1, 2, 3, 4, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - when(flavorSpareCountByFlavor.get(flavorToRetire).hasReady()).thenReturn(false); - when(flavorSpareCountByFlavor.get(flavors.get(0)).getNumReadyAmongReplacees()).thenReturn(1L); - when(flavorSpareCountByFlavor.get(flavors.get(2)).getNumReadyAmongReplacees()).thenReturn(1L); - - assertTrue(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(0, 2, 3, 4, 5); - } - - @Test - public void canRetireAllocated_CenterNode_NoNodeRepoFlavorNodes_NoImmediateSpare() { - Flavor flavorToRetire = flavors.get(1); - - // Same as above, but now one of the flavors that could replace flavor 1 (flavor 2) does not have enough spares - Stream.of(0, 1, 3, 4, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - when(flavorSpareCountByFlavor.get(flavorToRetire).hasReady()).thenReturn(false); - when(flavorSpareCountByFlavor.get(flavors.get(0)).getNumReadyAmongReplacees()).thenReturn(1L); - when(flavorSpareCountByFlavor.get(flavors.get(2)).getNumReadyAmongReplacees()).thenReturn(1L); - - assertFalse(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(); - } - - @Test - public void canRetireAllocated_CenterNode_NoNodeRepoFlavorNodes_SkipEmptyImmediate() { - Flavor flavorToRetire = flavors.get(1); - - // Flavor 2 still has no spares, but also the sum of ready nodes in its replaces tree is 0, so we should - // be able to continue - Stream.of(0, 1, 3, 4, 5) - .map(flavors::get) - .map(flavorSpareCountByFlavor::get) - .forEach(flavorSpareCount -> { - when(flavorSpareCount.hasReady()).thenReturn(true); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(true); - }); - when(flavorSpareCountByFlavor.get(flavorToRetire).hasReady()).thenReturn(false); - when(flavorSpareCountByFlavor.get(flavors.get(0)).getNumReadyAmongReplacees()).thenReturn(1L); - when(flavorSpareCountByFlavor.get(flavors.get(2)).getNumReadyAmongReplacees()).thenReturn(0L); - - assertTrue(flavorSpareChecker.canRetireAllocatedNodeWithFlavor(flavorToRetire)); - verifyDecrement(0, 3, 4, 5); - } - - private void verifyDecrement(int... decrementFlavorIds) { - Set<Flavor> decrementedFlavors = Arrays.stream(decrementFlavorIds).boxed().map(flavors::get).collect(Collectors.toSet()); - for (Flavor flavor : flavors) { - int times = decrementedFlavors.contains(flavor) ? 1 : 0; - verify(flavorSpareCountByFlavor.get(flavor), times(times)).decrementNumberOfReady(); - } - } - - @Before - public void setup() { - Map<Flavor, FlavorSpareCount> flavorSpareCountGraph = FlavorSpareCount.constructFlavorSpareCountGraph(flavors); - flavorSpareCountByFlavor.forEach((flavor, flavorSpareCount) -> { - Set<FlavorSpareCount> possibleWantedFlavors = flavorSpareCountGraph.get(flavor).getPossibleWantedFlavors() - .stream().map(FlavorSpareCount::getFlavor).map(flavorSpareCountByFlavor::get).collect(Collectors.toSet()); - Set<FlavorSpareCount> immediateReplacees = flavorSpareCountGraph.get(flavor).getImmediateReplacees() - .stream().map(FlavorSpareCount::getFlavor).map(flavorSpareCountByFlavor::get).collect(Collectors.toSet()); - - doNothing().when(flavorSpareCount).decrementNumberOfReady(); - when(flavorSpareCount.hasReady()).thenReturn(false); - when(flavorSpareCount.getPossibleWantedFlavors()).thenReturn(possibleWantedFlavors); - when(flavorSpareCount.getImmediateReplacees()).thenReturn(immediateReplacees); - when(spareNodesPolicy.hasSpare(flavorSpareCount)).thenReturn(false); - }); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCountTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCountTest.java deleted file mode 100644 index cb9c5c02c65..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCountTest.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provisioning.FlavorsConfig; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class FlavorSpareCountTest { - /* Creates flavors where 'replaces' graph that looks like this (largest flavor at the bottom): - * 5 - * | - * | - * 3 4 8 - * \ / \ | - * \ / \ | - * 1 6 7 - * / \ - * / \ - * 0 2 - */ - private final List<Flavor> flavors = makeFlavors( - Collections.singletonList(1), // 0 -> {1} - Arrays.asList(3, 4), // 1 -> {3, 4} - Collections.singletonList(1), // 2 -> {1} - Collections.singletonList(5), // 3 -> {5} - Collections.emptyList(), // 4 -> {} - Collections.emptyList(), // 5 -> {} - Collections.singletonList(4), // 6 -> {4} - Collections.singletonList(8), // 7 -> {8} - Collections.emptyList()); // 8 -> {} - - private final Map<Flavor, FlavorSpareCount> flavorSpareCountByFlavor = - FlavorSpareCount.constructFlavorSpareCountGraph(flavors); - - @Test - public void testFlavorSpareCountGraph() { - List<List<Integer>> expectedPossibleWantedFlavorsByFlavorId = Arrays.asList( - Arrays.asList(0, 1, 3, 4, 5), - Arrays.asList(1, 3, 4, 5), - Arrays.asList(1, 2, 3, 4, 5), - Arrays.asList(3, 5), - Collections.singletonList(4), - Collections.singletonList(5), - Arrays.asList(4, 6), - Arrays.asList(7, 8), - Collections.singletonList(8)); - - List<List<Integer>> expectedImmediateReplaceesByFlavorId = Arrays.asList( - Collections.emptyList(), - Arrays.asList(0, 2), - Collections.emptyList(), - Collections.singletonList(1), - Arrays.asList(1, 6), - Collections.singletonList(3), - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList(7)); - - for (int i = 0; i < flavors.size(); i++) { - Flavor flavor = flavors.get(i); - FlavorSpareCount flavorSpareCount = flavorSpareCountByFlavor.get(flavor); - Set<FlavorSpareCount> expectedPossibleWantedFlavors = expectedPossibleWantedFlavorsByFlavorId.get(i) - .stream().map(flavors::get).map(flavorSpareCountByFlavor::get).collect(Collectors.toSet()); - Set<FlavorSpareCount> expectedImmediateReplacees = expectedImmediateReplaceesByFlavorId.get(i) - .stream().map(flavors::get).map(flavorSpareCountByFlavor::get).collect(Collectors.toSet()); - - assertEquals(expectedPossibleWantedFlavors, flavorSpareCount.getPossibleWantedFlavors()); - assertEquals(expectedImmediateReplacees, flavorSpareCount.getImmediateReplacees()); - } - } - - @Test - public void testSumOfReadyAmongReplacees() { - long[] numReadyPerFlavor = {3, 5, 2, 6, 2, 7, 4, 3, 4}; - for (int i = 0; i < numReadyPerFlavor.length; i++) { - flavorSpareCountByFlavor.get(flavors.get(i)) - .updateReadyAndActiveCounts(numReadyPerFlavor[i], (long) (100 * Math.random())); - } - - long[] expectedSumTrees = {3, 10, 2, 16, 16, 23, 4, 3, 7}; - for (int i = 0; i < expectedSumTrees.length; i++) { - assertEquals(expectedSumTrees[i], flavorSpareCountByFlavor.get(flavors.get(i)).getNumReadyAmongReplacees()); - } - } - - /** - * Takes in variable number of List of Integers: - * For each list a flavor is created - * For each element, n, in list, the new flavor replace n'th flavor - */ - @SafeVarargs - static List<Flavor> makeFlavors(List<Integer>... replaces) { - FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); - for (int i = 0; i < replaces.length; i++) { - FlavorsConfig.Flavor.Builder builder = flavorConfigBuilder - .addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); - - for (Integer replacesId : replaces[i]) { - flavorConfigBuilder.addReplaces("flavor-" + replacesId, builder); - } - } - return new NodeFlavors(flavorConfigBuilder.build()) - .getFlavors().stream() - .sorted(Comparator.comparing(Flavor::name)) - .collect(Collectors.toList()); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index f97460713a5..6d94e4ab992 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -4,11 +4,11 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.Iterators; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -26,7 +26,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -41,26 +41,29 @@ public class LoadBalancerProvisionerTest { @Test public void provision_load_balancer() { + Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); + Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList(); ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); - Set<RotationName> rotationsCluster1 = Set.of(RotationName.from("r1-1"), RotationName.from("r1-2")); - tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1, rotationsCluster1), - clusterRequest(ClusterSpec.Type.content, contentCluster))); - tester.activate(app2, prepare(app2, - clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); // Provision a load balancer for each application - Supplier<List<LoadBalancer>> loadBalancers = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); - assertEquals(1, loadBalancers.get().size()); - - assertEquals(app1, loadBalancers.get().get(0).id().application()); - assertEquals(containerCluster1, loadBalancers.get().get(0).id().cluster()); - assertEquals(Collections.singleton(4443), loadBalancers.get().get(0).instance().ports()); - assertEquals("127.0.0.1", get(loadBalancers.get().get(0).instance().reals(), 0).ipAddress()); - assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 0).port()); - assertEquals("127.0.0.2", get(loadBalancers.get().get(0).instance().reals(), 1).ipAddress()); - assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 1).port()); + var nodes = prepare(app1, + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.content, contentCluster)); + assertEquals(1, lbApp1.get().size()); + assertEquals("Prepare provisions load balancer with 0 reals", Set.of(), lbApp1.get().get(0).instance().reals()); + tester.activate(app1, nodes); + tester.activate(app2, prepare(app2, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + assertEquals(1, lbApp2.get().size()); + + // Reals are configured after activation + assertEquals(app1, lbApp1.get().get(0).id().application()); + assertEquals(containerCluster1, lbApp1.get().get(0).id().cluster()); + assertEquals(Collections.singleton(4443), lbApp1.get().get(0).instance().ports()); + assertEquals("127.0.0.1", get(lbApp1.get().get(0).instance().reals(), 0).ipAddress()); + assertEquals(4080, get(lbApp1.get().get(0).instance().reals(), 0).port()); + assertEquals("127.0.0.2", get(lbApp1.get().get(0).instance().reals(), 1).ipAddress()); + assertEquals(4080, get(lbApp1.get().get(0).instance().reals(), 1).port()); // A container is failed Supplier<List<Node>> containers = () -> tester.getNodes(app1).type(ClusterSpec.Type.container).asList(); @@ -79,17 +82,17 @@ public class LoadBalancerProvisionerTest { .noneMatch(hostname -> hostname.equals(toFail.hostname()))); assertEquals(containers.get().get(0).hostname(), get(loadBalancer.instance().reals(), 0).hostname().value()); assertEquals(containers.get().get(1).hostname(), get(loadBalancer.instance().reals(), 1).hostname().value()); + assertSame("State is unchanged", LoadBalancer.State.active, loadBalancer.state()); // Add another container cluster - Set<RotationName> rotationsCluster2 = Set.of(RotationName.from("r2-1"), RotationName.from("r2-2")); ClusterSpec.Id containerCluster2 = ClusterSpec.Id.from("qrs2"); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1, rotationsCluster1), - clusterRequest(ClusterSpec.Type.container, containerCluster2, rotationsCluster2), + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster2), clusterRequest(ClusterSpec.Type.content, contentCluster))); // Load balancer is provisioned for second container cluster - assertEquals(2, loadBalancers.get().size()); + assertEquals(2, lbApp1.get().size()); List<HostName> activeContainers = tester.getNodes(app1, Node.State.active) .type(ClusterSpec.Type.container).asList() .stream() @@ -97,7 +100,7 @@ public class LoadBalancerProvisionerTest { .map(HostName::from) .sorted() .collect(Collectors.toList()); - List<HostName> reals = loadBalancers.get().stream() + List<HostName> reals = lbApp1.get().stream() .map(LoadBalancer::instance) .map(LoadBalancerInstance::reals) .flatMap(Collection::stream) @@ -111,38 +114,35 @@ public class LoadBalancerProvisionerTest { tester.provisioner().remove(removeTransaction, app1); removeTransaction.commit(); - assertEquals(2, loadBalancers.get().size()); - assertTrue("Deactivated load balancers", loadBalancers.get().stream().allMatch(LoadBalancer::inactive)); + assertEquals(2, lbApp1.get().size()); + assertTrue("Deactivated load balancers", lbApp1.get().stream().allMatch(lb -> lb.state() == LoadBalancer.State.inactive)); + assertTrue("Load balancers for " + app2 + " remain active", lbApp2.get().stream().allMatch(lb -> lb.state() == LoadBalancer.State.active)); // Application is redeployed with one cluster and load balancer is re-activated tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); - assertFalse("Re-activated load balancer for " + containerCluster1, - loadBalancers.get().stream() + assertSame("Re-activated load balancer for " + containerCluster1, LoadBalancer.State.active, + lbApp1.get().stream() .filter(lb -> lb.id().cluster().equals(containerCluster1)) + .map(LoadBalancer::state) .findFirst() - .orElseThrow() - .inactive()); - } - - private ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) { - return clusterRequest(type, id, Collections.emptySet()); - } - - private ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Set<RotationName> rotations) { - return ClusterSpec.request(type, id, Version.fromString("6.42"), false, rotations); + .orElseThrow()); } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { tester.makeReadyNodes(specs.length * 2, "d-1-1-1"); Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { - allNodes.addAll(tester.prepare(application, spec, 2, 1, new NodeResources(1, 1, 1))); + allNodes.addAll(tester.prepare(application, spec, Capacity.fromCount(2, new NodeResources(1, 1, 1), false, true), 1, false)); } return allNodes; } + private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) { + return ClusterSpec.request(type, id, Version.fromString("6.42"), false); + } + private static <T> T get(Set<T> set, int position) { return Iterators.get(set.iterator(), position, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c8051c3bdee..294c153f86f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -139,16 +139,21 @@ public class ProvisioningTester { } public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) { + return prepare(application, cluster, capacity, groups, true); + } + + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups, boolean idempotentPrepare) { Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive)); - // prepare twice to ensure idempotence List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); - List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); - assertEquals(hosts1, hosts2); + if (idempotentPrepare) { // prepare twice to ensure idempotence + List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); + assertEquals(hosts1, hosts2); + } Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); newlyActivated.removeAll(reservedBefore); newlyActivated.removeAll(inactiveBefore); - return hosts2; + return hosts1; } public void activate(ApplicationId application, Collection<HostSpec> hosts) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 6524292f48c..bfb24d30284 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -831,7 +831,7 @@ public class RestApiTest { @Test public void test_load_balancers() throws Exception { assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); - assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers.json"); + assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json index e23f7129ae2..fd553a97ea4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json @@ -9,7 +9,6 @@ "canonicalFlavor": "default", "minDiskAvailableGb": 400.0, "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", "minCpuCores": 2.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json index 8e1accd65a2..aa818a9cf42 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json @@ -9,7 +9,6 @@ "canonicalFlavor": "default", "minDiskAvailableGb": 400.0, "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", "minCpuCores": 2.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json index a5d2a7a37dd..bfa34bc0517 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json @@ -9,7 +9,6 @@ "canonicalFlavor": "default", "minDiskAvailableGb": 400.0, "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", "minCpuCores": 2.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json index 5cbd372385a..fc91c883441 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth": 0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json index 5cbd372385a..fc91c883441 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth": 0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json index cf190ff36bc..f59af799f37 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json index 4ffedbf01d5..a01f4372fd8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json index 396a645ea3b..44a11c98da2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json index 146af5998bd..b3ec9aa0093 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json index 5dadfe68845..963d485ac70 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json index 637a7cc858d..efecd510266 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 1600.0, "minMainMemoryAvailableGb": 32.0, - "description": "Flavor-name-is-large", "minCpuCores": 4.0, "fastDisk": true, "bandwidth": 0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json new file mode 100644 index 00000000000..67d2c3bfa4b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json @@ -0,0 +1,36 @@ +{ + "loadBalancers": [ + { + "id": "tenant4:application4:instance4:id4", + "state": "active", + "changedAt": 123, + "application": "application4", + "tenant": "tenant4", + "instance": "instance4", + "cluster": "id4", + "hostname": "lb-tenant4.application4.instance4-id4", + "dnsZone": "zone-id-1", + "networks": [ + "10.2.3.0/24", + "10.4.5.0/24" + ], + "ports": [ + 4443 + ], + "reals": [ + { + "hostname": "host13.yahoo.com", + "ipAddress": "127.0.13.1", + "port": 4080 + }, + { + "hostname": "host14.yahoo.com", + "ipAddress": "127.0.14.1", + "port": 4080 + } + ], + "rotations": [], + "inactive": false + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json index d2c4d0ac857..36d4de598e2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json @@ -1,7 +1,30 @@ { "loadBalancers": [ { + "id": "tenant1:application1:instance1:id1", + "state": "reserved", + "changedAt": 123, + "application": "application1", + "tenant": "tenant1", + "instance": "instance1", + "cluster": "id1", + "hostname": "lb-tenant1.application1.instance1-id1", + "dnsZone": "zone-id-1", + "networks": [ + "10.2.3.0/24", + "10.4.5.0/24" + ], + "ports": [ + 4443 + ], + "reals": [], + "rotations": [], + "inactive": false + }, + { "id": "tenant4:application4:instance4:id4", + "state": "active", + "changedAt": 123, "application": "application4", "tenant": "tenant4", "instance": "instance4", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index 1432d2f4ea5..b72523963c0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -25,9 +25,6 @@ "name": "NodeRebooter" }, { - "name": "NodeRetirer" - }, - { "name": "OperatorChangeApplicationMaintainer" }, { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index c9983b3c996..f0c937d20f3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -10,7 +10,6 @@ "canonicalFlavor": "d-2-8-100", "minDiskAvailableGb": 100.0, "minMainMemoryAvailableGb": 8.0, - "description": "Flavor-name-is-d-2-8-100", "minCpuCores": 2.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index ab69b6b1d5a..e5a5c7a9520 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -9,7 +9,6 @@ "canonicalFlavor": "default", "minDiskAvailableGb": 400.0, "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", "minCpuCores": 2.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index 271ff3feef1..561cab22f85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 2000.0, "minMainMemoryAvailableGb": 128.0, - "description": "Flavor-name-is-large-variant", "minCpuCores": 64.0, "fastDisk": true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json index 9823ffcc14f..42cef1c3c83 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json @@ -9,7 +9,6 @@ "canonicalFlavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb]", "minDiskAvailableGb": 400.0, "minMainMemoryAvailableGb": 16.0, - "description": "Flavor-name-is-default", "minCpuCores": 2.0, "fastDisk":true, "bandwidth":0.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index ec319edb170..28bb960eb14 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -9,7 +9,6 @@ "canonicalFlavor": "large", "minDiskAvailableGb": 2000.0, "minMainMemoryAvailableGb": 128.0, - "description": "Flavor-name-is-large-variant", "minCpuCores": 64.0, "fastDisk":true, "bandwidth":0.0, diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index f60863ef0b0..721ee9978b0 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -12,7 +12,6 @@ #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/searchcore/proton/matching/error_constant_value.h> #include <vespa/searchcore/proton/matching/indexenvironment.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/fef.h> @@ -34,11 +33,11 @@ using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; using vespa::config::search::core::RankingConstantsConfig; using vespalib::eval::ConstantValue; -using vespalib::eval::ErrorValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; using vespalib::eval::SimpleConstantValue; +using vespalib::eval::BadConstantValue; class App : public FastOS_Application { @@ -61,13 +60,17 @@ struct DummyConstantValueRepo : IConstantValueRepo { DummyConstantValueRepo(const RankingConstantsConfig &cfg_in) : cfg(cfg_in) {} virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override { for (const auto &entry: cfg.constant) { - if (entry.name == name) { + if (entry.name == name) { const auto &engine = DefaultTensorEngine::ref(); - auto tensor = engine.from_spec(TensorSpec(entry.type)); - return std::make_unique<SimpleConstantValue>(std::move(tensor)); + try { + auto tensor = engine.from_spec(TensorSpec(entry.type)); + return std::make_unique<SimpleConstantValue>(std::move(tensor)); + } catch (std::exception &) { + return std::make_unique<BadConstantValue>(); + } } } - return std::make_unique<SimpleConstantValue>(std::make_unique<ErrorValue>()); + return vespalib::eval::ConstantValue::UP(nullptr); } }; diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index aba879e5a44..3154b420789 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -9,7 +9,6 @@ #include <vespa/searchcore/proton/docsummary/summarymanager.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> -#include <vespa/searchcore/proton/matching/error_constant_value.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> @@ -138,7 +137,7 @@ ViewSet::~ViewSet() {} struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory { virtual vespalib::eval::ConstantValue::UP create(const vespalib::string &, const vespalib::string &) const override { - return std::make_unique<ErrorConstantValue>(); + return vespalib::eval::ConstantValue::UP(nullptr); } }; diff --git a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp index acc16f41872..68959d8e20f 100644 --- a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp +++ b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp @@ -3,7 +3,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/constant_value_repo.h> -#include <vespa/searchcore/proton/matching/error_constant_value.h> #include <vespa/eval/eval/value_cache/constant_value.h> using namespace proton::matching; @@ -36,7 +35,7 @@ public: if (itr != _map.end()) { return std::make_unique<DoubleConstantValue>(itr->second); } - return std::make_unique<ErrorConstantValue>(); + return std::make_unique<BadConstantValue>(); } }; @@ -58,14 +57,14 @@ TEST_F("require that constant value can be retrieved from repo", Fixture) EXPECT_EQUAL(3, f.repo.getConstant("foo")->value().as_double()); } -TEST_F("require that non-existing constant value in repo returns error value", Fixture) +TEST_F("require that non-existing constant value in repo returns nullptr", Fixture) { - EXPECT_TRUE(f.repo.getConstant("none")->value().is_error()); + EXPECT_TRUE(f.repo.getConstant("none").get() == nullptr); } -TEST_F("require that non-existing constant value in factory returns error value", Fixture) +TEST_F("require that non-existing constant value in factory returns bad constant", Fixture) { - EXPECT_TRUE(f.repo.getConstant("bar")->value().is_error()); + EXPECT_TRUE(f.repo.getConstant("bar")->type().is_error()); } TEST_F("require that reconfigure replaces existing constant values in repo", Fixture) @@ -73,7 +72,7 @@ TEST_F("require that reconfigure replaces existing constant values in repo", Fix f.repo.reconfigure(RankingConstants({{"bar", "double", "path_3"}, {"baz", "double", "path_2"}})); f.factory.add("path_3", "double", 7); - EXPECT_TRUE(f.repo.getConstant("foo")->value().is_error()); + EXPECT_TRUE(f.repo.getConstant("foo").get() == nullptr); EXPECT_EQUAL(7, f.repo.getConstant("bar")->value().as_double()); EXPECT_EQUAL(5, f.repo.getConstant("baz")->value().as_double()); } diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 967d8bfd0aa..e46ed997d0f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -6,7 +6,6 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> -#include <vespa/searchcore/proton/matching/error_constant_value.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/i_constant_value_repo.h> #include <vespa/searchcore/proton/matching/isearchcontext.h> @@ -105,7 +104,7 @@ const uint32_t NUM_DOCS = 1000; struct EmptyConstantValueRepo : public proton::matching::IConstantValueRepo { virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &) const override { - return std::make_unique<proton::matching::ErrorConstantValue>(); + return vespalib::eval::ConstantValue::UP(nullptr); } }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp index 7e355da041c..bdfa4013c86 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "constant_value_repo.h" -#include "error_constant_value.h" using vespalib::eval::ConstantValue; @@ -27,7 +26,7 @@ ConstantValueRepo::getConstant(const vespalib::string &name) const if (constant != nullptr) { return _factory.create(constant->filePath, constant->type); } - return std::make_unique<ErrorConstantValue>(); + return ConstantValue::UP(nullptr); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/error_constant_value.h b/searchcore/src/vespa/searchcore/proton/matching/error_constant_value.h deleted file mode 100644 index 9b0b688085d..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/error_constant_value.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/value_cache/constant_value.h> - -namespace proton { -namespace matching { - -/** - * Class representing an error constant value. - * Typically used to indicate that a named constant value does not exists. - */ -class ErrorConstantValue : public vespalib::eval::ConstantValue { -private: - vespalib::eval::ErrorValue _value; - vespalib::eval::ValueType _type; -public: - ErrorConstantValue() : _value(), _type(vespalib::eval::ValueType::error_type()) {} - virtual const vespalib::eval::Value &value() const override { return _value; } - virtual const vespalib::eval::ValueType &type() const override { return _type; } -}; - -} -} diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h b/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h index faa368c734b..5ac4fd14802 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h +++ b/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h @@ -9,7 +9,7 @@ namespace matching { /** * Interface for retrieving a named constant rank value to be used by features in the rank framework. - * If the given value is not found an vespalib::eval::ErrorValue should be returned. + * If the given value is not found a nullptr should be returned. */ struct IConstantValueRepo { virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const = 0; diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp index df7f80e8601..36e9bde5c9f 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp @@ -18,10 +18,13 @@ using namespace search::memoryindex; using search::index::schema::DataType; using search::test::SearchIteratorVerifier; +using FieldIndexType = FieldIndex<false>; +using PostingIteratorType = PostingIterator<false>; + class Verifier : public SearchIteratorVerifier { private: mutable TermFieldMatchData _tfmd; - FieldIndex _field_index; + FieldIndexType _field_index; public: Verifier(const Schema& schema) @@ -41,8 +44,8 @@ public: (void) strict; TermFieldMatchDataArray match_data; match_data.add(&_tfmd); - return std::make_unique<PostingIterator>(_field_index.find("a"), - _field_index.getFeatureStore(), 0, match_data); + return std::make_unique<PostingIteratorType>(_field_index.find("a"), + _field_index.getFeatureStore(), 0, match_data); } }; diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 0f1c966ad5d..f2cc2580cd8 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -40,8 +40,10 @@ using vespalib::GenerationHandler; namespace memoryindex { using test::WrapInserter; -using PostingList = FieldIndex::PostingList; +using FieldIndexType = FieldIndex<false>; +using PostingList = FieldIndexType::PostingList; using PostingConstItr = PostingList::ConstIterator; +using PostingIteratorType = PostingIterator<false>; class MyBuilder : public IndexBuilder { private: @@ -197,6 +199,25 @@ assertPostingList(std::vector<uint32_t> &exp, PostingConstItr itr) return assertPostingList(ss.str(), itr); } +FieldIndexType::PostingList::Iterator +find_in_field_index(const vespalib::stringref word, + uint32_t fieldId, + const FieldIndexCollection& fic) +{ + auto* field_index = dynamic_cast<FieldIndexType*>(fic.getFieldIndex(fieldId)); + assert(field_index != nullptr); + return field_index->find(word); +} + +FieldIndexType::PostingList::ConstIterator +find_frozen_in_field_index(const vespalib::stringref word, + uint32_t fieldId, + const FieldIndexCollection& fic) +{ + auto* field_index = dynamic_cast<FieldIndexType*>(fic.getFieldIndex(fieldId)); + assert(field_index != nullptr); + return field_index->findFrozen(word); +} namespace { @@ -332,7 +353,7 @@ public: bool assertPosting(const vespalib::string &word, uint32_t fieldId) { std::vector<uint32_t> exp = _mock.find(word, fieldId); - PostingConstItr itr = _fieldIndexes.find(word, fieldId); + PostingConstItr itr = find_in_field_index(word, fieldId, _fieldIndexes); bool result = assertPostingList(exp, itr); EXPECT_TRUE(result); return result; @@ -390,7 +411,7 @@ public: { } - MyDrainRemoves(FieldIndex& field_index) + MyDrainRemoves(FieldIndexType& field_index) : _remover(field_index.getDocumentRemover()) { } @@ -468,7 +489,7 @@ make_single_field_schema() struct FieldIndexTest : public ::testing::Test { Schema schema; - FieldIndex idx; + FieldIndexType idx; FieldIndexTest() : schema(make_single_field_schema()), idx(schema, 0) @@ -487,6 +508,8 @@ make_multi_field_schema() return result; } + + struct FieldIndexCollectionTest : public ::testing::Test { Schema schema; FieldIndexCollection fic; @@ -496,6 +519,11 @@ struct FieldIndexCollectionTest : public ::testing::Test { { } ~FieldIndexCollectionTest() {} + + FieldIndexType::PostingList::Iterator find(const vespalib::stringref word, + uint32_t fieldId) const { + return find_in_field_index(word, fieldId, fic); + } }; TEST_F(FieldIndexTest, require_that_fresh_insert_works) @@ -529,12 +557,12 @@ TEST_F(FieldIndexCollectionTest, require_that_multiple_posting_lists_across_mult WrapInserter(fic, 0).word("a").add(10).word("b").add(11).add(15).flush(); WrapInserter(fic, 1).word("a").add(5).word("b").add(12).flush(); EXPECT_EQ(4u, fic.getNumUniqueWords()); - EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[5]", fic.find("a", 1))); - EXPECT_TRUE(assertPostingList("[11,15]", fic.find("b", 0))); - EXPECT_TRUE(assertPostingList("[12]", fic.find("b", 1))); - EXPECT_TRUE(assertPostingList("[]", fic.find("a", 2))); - EXPECT_TRUE(assertPostingList("[]", fic.find("c", 0))); + EXPECT_TRUE(assertPostingList("[10]", find("a", 0))); + EXPECT_TRUE(assertPostingList("[5]", find("a", 1))); + EXPECT_TRUE(assertPostingList("[11,15]", find("b", 0))); + EXPECT_TRUE(assertPostingList("[12]", find("b", 1))); + EXPECT_TRUE(assertPostingList("[]", find("a", 2))); + EXPECT_TRUE(assertPostingList("[]", find("c", 0))); } TEST_F(FieldIndexTest, require_that_remove_works) @@ -622,16 +650,16 @@ TEST_F(FieldIndexCollectionTest, require_that_features_are_in_posting_lists) { WrapInserter(fic, 0).word("a").add(1, getFeatures(4, 2)).flush(); EXPECT_TRUE(assertPostingList("[1{4:0,1}]", - fic.find("a", 0), + find("a", 0), featureStorePtr(fic, 0))); WrapInserter(fic, 0).word("b").add(2, getFeatures(5, 1)). add(3, getFeatures(6, 2)).flush(); EXPECT_TRUE(assertPostingList("[2{5:0},3{6:0,1}]", - fic.find("b", 0), + find("b", 0), featureStorePtr(fic, 0))); WrapInserter(fic, 1).word("c").add(4, getFeatures(7, 2)).flush(); EXPECT_TRUE(assertPostingList("[4{7:0,1}]", - fic.find("c", 1), + find("c", 1), featureStorePtr(fic, 1))); } @@ -645,16 +673,16 @@ TEST_F(FieldIndexTest, require_that_posting_iterator_is_working) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(idx.find("not"), - idx.getFeatureStore(), - 0, matchData); + PostingIteratorType itr(idx.find("not"), + idx.getFeatureStore(), + 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(idx.find("a"), - idx.getFeatureStore(), - 0, matchData); + PostingIteratorType itr(idx.find("a"), + idx.getFeatureStore(), + 0, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -764,6 +792,12 @@ public: _inv(_schema, _invertThreads, _pushThreads, _fic) { } + PostingList::Iterator find(const vespalib::stringref word, uint32_t fieldId) const { + return find_in_field_index(word, fieldId, _fic); + } + PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { + return find_frozen_in_field_index(word, fieldId, _fic); + } }; class BasicInverterTest : public InverterTest { @@ -922,12 +956,12 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(_fic.findFrozen("not", 0), featureStoreRef(_fic, 0), 0, matchData); + PostingIteratorType itr(findFrozen("not", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(_fic.findFrozen("a", 0), featureStoreRef(_fic, 0), 0, matchData); + PostingIteratorType itr(findFrozen("a", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -944,19 +978,19 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(_fic.findFrozen("x", 0), featureStoreRef(_fic, 0), 0, matchData); + PostingIteratorType itr(findFrozen("x", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(_fic.findFrozen("x", 1), featureStoreRef(_fic, 1), 1, matchData); + PostingIteratorType itr(findFrozen("x", 1), featureStoreRef(_fic, 1), 1, matchData); itr.initFullRange(); EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); EXPECT_EQ("{6:2[e=0,w=1,l=6]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(_fic.findFrozen("x", 2), featureStoreRef(_fic, 2), 2, matchData); + PostingIteratorType itr(findFrozen("x", 2), featureStoreRef(_fic, 2), 2, matchData); itr.initFullRange(); EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); @@ -964,7 +998,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) EXPECT_EQ("{2:1[e=0,w=1,l=2]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(_fic.findFrozen("x", 3), featureStoreRef(_fic, 3), 3, matchData); + PostingIteratorType itr(findFrozen("x", 3), featureStoreRef(_fic, 3), 3, matchData); itr.initFullRange(); EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); @@ -994,20 +1028,20 @@ TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remo myPushDocument(_inv); _pushThreads.sync(); - EXPECT_TRUE(assertPostingList("[1]", _fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[1,2]", _fic.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", _fic.find("c", 0))); - EXPECT_TRUE(assertPostingList("[1]", _fic.find("a", 1))); - EXPECT_TRUE(assertPostingList("[1]", _fic.find("c", 1))); + EXPECT_TRUE(assertPostingList("[1]", find("a", 0))); + EXPECT_TRUE(assertPostingList("[1,2]", find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", find("c", 0))); + EXPECT_TRUE(assertPostingList("[1]", find("a", 1))); + EXPECT_TRUE(assertPostingList("[1]", find("c", 1))); myremove(1, _inv, _invertThreads); _pushThreads.sync(); - EXPECT_TRUE(assertPostingList("[]", _fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[2]", _fic.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", _fic.find("c", 0))); - EXPECT_TRUE(assertPostingList("[]", _fic.find("a", 1))); - EXPECT_TRUE(assertPostingList("[]", _fic.find("c", 1))); + EXPECT_TRUE(assertPostingList("[]", find("a", 0))); + EXPECT_TRUE(assertPostingList("[2]", find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", find("c", 0))); + EXPECT_TRUE(assertPostingList("[]", find("a", 1))); + EXPECT_TRUE(assertPostingList("[]", find("c", 1))); } Schema @@ -1161,17 +1195,17 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working) matchData.add(&tfmd); { uint32_t fieldId = _schema.getIndexFieldId("iu"); - PostingIterator itr(_fic.findFrozen("not", fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("not", fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { uint32_t fieldId = _schema.getIndexFieldId("iu"); - PostingIterator itr(_fic.findFrozen("example", fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -1181,9 +1215,9 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working) } { uint32_t fieldId = _schema.getIndexFieldId("iau"); - PostingIterator itr(_fic.findFrozen("example", fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -1194,9 +1228,9 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working) } { uint32_t fieldId = _schema.getIndexFieldId("iwu"); - PostingIterator itr(_fic.findFrozen("example", fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -1247,18 +1281,18 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working) matchData.add(&tfmd); uint32_t fieldId = _schema.getIndexFieldId("f0"); { - PostingIterator itr(_fic.findFrozen("not", fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("not", fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(_fic.findFrozen("我就" - "是那个", - fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("我就" + "是那个", + fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -1267,11 +1301,11 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working) EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(_fic.findFrozen("大灰" - "狼", - fieldId), - featureStoreRef(_fic, fieldId), - fieldId, matchData); + PostingIteratorType itr(findFrozen("大灰" + "狼", + fieldId), + featureStoreRef(_fic, fieldId), + fieldId, matchData); itr.initFullRange(); EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); @@ -1315,9 +1349,9 @@ struct RemoverTest : public FieldIndexCollectionTest { void assertPostingLists(const vespalib::string &e1, const vespalib::string &e2, const vespalib::string &e3) { - EXPECT_TRUE(assertPostingList(e1, fic.find("a", 1))); - EXPECT_TRUE(assertPostingList(e2, fic.find("a", 2))); - EXPECT_TRUE(assertPostingList(e3, fic.find("b", 1))); + EXPECT_TRUE(assertPostingList(e1, find("a", 1))); + EXPECT_TRUE(assertPostingList(e2, find("a", 2))); + EXPECT_TRUE(assertPostingList(e3, find("b", 1))); } void remove(uint32_t docId) { DocumentInverter inv(schema, _invertThreads, _pushThreads, fic); diff --git a/searchlib/src/vespa/searchlib/expression/resultvector.h b/searchlib/src/vespa/searchlib/expression/resultvector.h index cd29178f24f..f1f863edf12 100644 --- a/searchlib/src/vespa/searchlib/expression/resultvector.h +++ b/searchlib/src/vespa/searchlib/expression/resultvector.h @@ -11,6 +11,7 @@ #include "stringbucketresultnode.h" #include "rawbucketresultnode.h" #include <vespa/vespalib/objects/visit.hpp> +#include <vespa/vespalib/stllike/identity.h> #include <algorithm> namespace search::expression { @@ -214,7 +215,7 @@ struct GetString { }; template <typename B> -class NumericResultNodeVectorT : public ResultNodeVectorT<B, cmpT<ResultNode>, std::_Identity<ResultNode> > +class NumericResultNodeVectorT : public ResultNodeVectorT<B, cmpT<ResultNode>, vespalib::Identity> { public: ResultNode & flattenMultiply(ResultNode & r) const override { @@ -366,7 +367,7 @@ public: const FloatBucketResultNode& getNullBucket() const override { return FloatBucketResultNode::getNull(); } }; -class StringResultNodeVector : public ResultNodeVectorT<StringResultNode, cmpT<ResultNode>, std::_Identity<ResultNode> > +class StringResultNodeVector : public ResultNodeVectorT<StringResultNode, cmpT<ResultNode>, vespalib::Identity> { public: StringResultNodeVector() { } @@ -375,7 +376,7 @@ public: const StringBucketResultNode& getNullBucket() const override { return StringBucketResultNode::getNull(); } }; -class RawResultNodeVector : public ResultNodeVectorT<RawResultNode, cmpT<ResultNode>, std::_Identity<ResultNode> > +class RawResultNodeVector : public ResultNodeVectorT<RawResultNode, cmpT<ResultNode>, vespalib::Identity> { public: RawResultNodeVector() { } diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 1560d043be2..e3f4cee4836 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -816,6 +816,16 @@ make_queryvector_key(const vespalib::string & base, const vespalib::string & sub return key; } +const vespalib::string & +make_queryvector_key_for_attribute(const IAttributeVector & attribute, const vespalib::string & key, vespalib::string & scratchPad) { + if (attribute.hasEnum() && (attribute.getCollectionType() == attribute::CollectionType::WSET)) { + scratchPad = key; + scratchPad.append(".").append(attribute.getName()); + return scratchPad; + } + return key; +} + vespalib::string make_attribute_key(const vespalib::string & base, const vespalib::string & subKey) { vespalib::string key(base); @@ -828,11 +838,12 @@ make_attribute_key(const vespalib::string & base, const vespalib::string & subKe const IAttributeVector * DotProductBlueprint::upgradeIfNecessary(const IAttributeVector * attribute, const IQueryEnvironment & env) const { - if ((attribute->getCollectionType() == attribute::CollectionType::WSET) && + if ((attribute != nullptr) && + (attribute->getCollectionType() == attribute::CollectionType::WSET) && attribute->hasEnum() && (attribute->isStringType() || attribute->isIntegerType())) { - attribute = env.getAttributeContext().getAttributeStableEnum(getAttribute(env)); + attribute = env.getAttributeContext().getAttributeStableEnum(attribute->getName()); } return attribute; } @@ -903,6 +914,7 @@ createQueryVector(const IQueryEnvironment & env, const IAttributeVector * attrib DotProductBlueprint::DotProductBlueprint() : Blueprint("dotProduct"), _defaultAttribute(), + _attributeOverride(), _queryVector(), _attrKey(), _queryVectorKey() @@ -910,10 +922,10 @@ DotProductBlueprint::DotProductBlueprint() : DotProductBlueprint::~DotProductBlueprint() = default; -vespalib::string +const vespalib::string & DotProductBlueprint::getAttribute(const IQueryEnvironment & env) const { - Property prop = env.getProperties().lookup(getBaseName(), _defaultAttribute + ".override.name"); + Property prop = env.getProperties().lookup(getBaseName(), _attributeOverride); if (prop.found() && !prop.get().empty()) { return prop.get(); } @@ -929,6 +941,7 @@ bool DotProductBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { _defaultAttribute = params[0].getValue(); + _attributeOverride = _defaultAttribute + ".override.name"; _queryVector = params[1].getValue(); _attrKey = make_attribute_key(getBaseName(), _defaultAttribute); _queryVectorKey = make_queryvector_key(getBaseName(), _queryVector); @@ -959,7 +972,8 @@ DotProductBlueprint::prepareSharedState(const IQueryEnvironment & env, IObjectSt if (queryVector == nullptr) { fef::Anything::UP arguments = createQueryVector(env, attribute, getBaseName(), _queryVector); if (arguments) { - store.add(_queryVectorKey, std::move(arguments)); + vespalib::string scratchPad; + store.add(make_queryvector_key_for_attribute(*attribute, _queryVectorKey, scratchPad), std::move(arguments)); } } @@ -971,16 +985,20 @@ DotProductBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta { // Doing it "manually" here to avoid looking up attribute override unless needed. const fef::Anything * attributeArg = env.getObjectStore().get(_attrKey); - const IAttributeVector * attribute = (attributeArg != nullptr) - ? static_cast<const fef::AnyWrapper<const IAttributeVector *> *>(attributeArg)->getValue() - : env.getAttributeContext().getAttribute(getAttribute(env)); + const IAttributeVector * attribute = nullptr; + if (attributeArg != nullptr) { + attribute = static_cast<const fef::AnyWrapper<const IAttributeVector *> *>(attributeArg)->getValue(); + } else { + attribute = env.getAttributeContext().getAttribute(getAttribute(env)); + attribute = upgradeIfNecessary(attribute, env); + } if (attribute == nullptr) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.", getAttribute(env).c_str()); return stash.create<SingleZeroValueExecutor>(); } - attribute = upgradeIfNecessary(attribute, env); - const fef::Anything * queryVectorArg = env.getObjectStore().get(_queryVectorKey); + vespalib::string scratchPad; + const fef::Anything * queryVectorArg = env.getObjectStore().get(make_queryvector_key_for_attribute(*attribute, _queryVectorKey, scratchPad)); if (queryVectorArg != nullptr) { return createFromObject(attribute, *queryVectorArg, stash); } else { diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index d315a24ecb3..8d2b3d9de72 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -309,11 +309,12 @@ class DotProductBlueprint : public fef::Blueprint { private: using IAttributeVector = attribute::IAttributeVector; vespalib::string _defaultAttribute; + vespalib::string _attributeOverride; vespalib::string _queryVector; vespalib::string _attrKey; vespalib::string _queryVectorKey; - vespalib::string getAttribute(const fef::IQueryEnvironment & env) const; + const vespalib::string & getAttribute(const fef::IQueryEnvironment & env) const; const IAttributeVector * upgradeIfNecessary(const IAttributeVector * attribute, const fef::IQueryEnvironment & env) const; public: diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp index 755245438f2..e998e4d18bd 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp @@ -7,14 +7,6 @@ namespace search::fef::test { using vespalib::eval::ValueType; -using vespalib::eval::ErrorValue; - -namespace { - -IndexEnvironment::Constant notFoundError(ValueType::error_type(), - std::make_unique<ErrorValue>()); - -} IndexEnvironment::IndexEnvironment() = default; @@ -46,7 +38,7 @@ IndexEnvironment::getConstantValue(const vespalib::string &name) const if (it != _constants.end()) { return std::make_unique<ConstantRef>(it->second); } else { - return std::make_unique<ConstantRef>(notFoundError); + return vespalib::eval::ConstantValue::UP(nullptr); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index 441fe12c383..c19596692cc 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchlib_memoryindex OBJECT document_inverter.cpp feature_store.cpp field_index.cpp + field_index_base.cpp field_index_collection.cpp field_index_remover.cpp field_inverter.cpp diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 13a7a860d03..a5dc921cfdf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -2,21 +2,33 @@ #include "field_index.h" #include "ordered_field_index_inserter.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/exceptions.h> +#include "posting_iterator.h" #include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/vespalib/btree/btree.hpp> +#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> -#include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/btree/btree.hpp> +#include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.memoryindex.field_index"); + +using search::fef::TermFieldMatchDataArray; using search::index::DocIdAndFeatures; -using search::index::WordDocElementFeatures; using search::index::Schema; +using search::index::WordDocElementFeatures; +using search::queryeval::BooleanMatchIteratorWrapper; +using search::queryeval::FieldSpecBase; +using search::queryeval::SearchIterator; +using search::queryeval::SimpleLeafBlueprint; +using vespalib::GenerationHandler; namespace search::memoryindex { @@ -36,33 +48,24 @@ void set_interleaved_features(DocIdAndFeatures &features) using datastore::EntryRef; -vespalib::asciistream & -operator<<(vespalib::asciistream & os, const FieldIndex::WordKey & rhs) -{ - os << "wr(" << rhs._wordRef.ref() << ")"; - return os; -} - -FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId) +template <bool interleaved_features> +FieldIndex<interleaved_features>::FieldIndex(const index::Schema& schema, uint32_t fieldId) : FieldIndex(schema, fieldId, index::FieldLengthInfo()) { } -FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info) - : _wordStore(), - _numUniqueWords(0), - _generationHandler(), - _dict(), - _postingListStore(), - _featureStore(schema), - _fieldId(fieldId), - _remover(_wordStore), - _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)), - _calculator(info) +template <bool interleaved_features> +FieldIndex<interleaved_features>::FieldIndex(const index::Schema& schema, uint32_t fieldId, + const index::FieldLengthInfo& info) + : FieldIndexBase(schema, fieldId, info), + _postingListStore() { + using InserterType = OrderedFieldIndexInserter<interleaved_features>; + _inserter = std::make_unique<InserterType>(*this); } -FieldIndex::~FieldIndex() +template <bool interleaved_features> +FieldIndex<interleaved_features>::~FieldIndex() { _postingListStore.disableFreeLists(); _postingListStore.disableElemHoldList(); @@ -89,28 +92,31 @@ FieldIndex::~FieldIndex() trimHoldLists(); } -FieldIndex::PostingList::Iterator -FieldIndex::find(const vespalib::stringref word) const +template <bool interleaved_features> +typename FieldIndex<interleaved_features>::PostingList::Iterator +FieldIndex<interleaved_features>::find(const vespalib::stringref word) const { DictionaryTree::Iterator itr = _dict.find(WordKey(EntryRef()), KeyComp(_wordStore, word)); if (itr.valid()) { return _postingListStore.begin(EntryRef(itr.getData())); } - return PostingList::Iterator(); + return typename PostingList::Iterator(); } -FieldIndex::PostingList::ConstIterator -FieldIndex::findFrozen(const vespalib::stringref word) const +template <bool interleaved_features> +typename FieldIndex<interleaved_features>::PostingList::ConstIterator +FieldIndex<interleaved_features>::findFrozen(const vespalib::stringref word) const { auto itr = _dict.getFrozenView().find(WordKey(EntryRef()), KeyComp(_wordStore, word)); if (itr.valid()) { return _postingListStore.beginFrozen(EntryRef(itr.getData())); } - return PostingList::Iterator(); + return typename PostingList::Iterator(); } +template <bool interleaved_features> void -FieldIndex::compactFeatures() +FieldIndex<interleaved_features>::compactFeatures() { std::vector<uint32_t> toHold; @@ -118,7 +124,7 @@ FieldIndex::compactFeatures() auto itr = _dict.begin(); uint32_t packedIndex = _fieldId; for (; itr.valid(); ++itr) { - PostingListStore::RefType pidx(EntryRef(itr.getData())); + typename PostingListStore::RefType pidx(EntryRef(itr.getData())); if (!pidx.valid()) { continue; } @@ -127,7 +133,7 @@ FieldIndex::compactFeatures() const PostingList *tree = _postingListStore.getTreeEntry(pidx); auto pitr = tree->begin(_postingListStore.getAllocator()); for (; pitr.valid(); ++pitr) { - const PostingListEntry &posting_entry(pitr.getData()); + const PostingListEntryType& posting_entry(pitr.getData()); // Filter on which buffers to move features from when // performing incremental compaction. @@ -144,7 +150,7 @@ FieldIndex::compactFeatures() const PostingListKeyDataType *shortArray = _postingListStore.getKeyDataEntry(pidx, clusterSize); const PostingListKeyDataType *ite = shortArray + clusterSize; for (const PostingListKeyDataType *it = shortArray; it < ite; ++it) { - const PostingListEntry &posting_entry(it->getData()); + const PostingListEntryType& posting_entry(it->getData()); // Filter on which buffers to move features from when // performing incremental compaction. @@ -165,8 +171,9 @@ FieldIndex::compactFeatures() _featureStore.transferHoldLists(generation); } +template <bool interleaved_features> void -FieldIndex::dump(search::index::IndexBuilder & indexBuilder) +FieldIndex<interleaved_features>::dump(search::index::IndexBuilder & indexBuilder) { vespalib::stringref word; FeatureStore::DecodeContextCooked decoder(nullptr); @@ -175,7 +182,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) _featureStore.setupForField(_fieldId, decoder); for (auto itr = _dict.begin(); itr.valid(); ++itr) { const WordKey & wk = itr.getKey(); - PostingListStore::RefType plist(EntryRef(itr.getData())); + typename PostingListStore::RefType plist(EntryRef(itr.getData())); word = _wordStore.getWord(wk._wordRef); if (!plist.valid()) { continue; @@ -213,8 +220,9 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) } } +template <bool interleaved_features> vespalib::MemoryUsage -FieldIndex::getMemoryUsage() const +FieldIndex<interleaved_features>::getMemoryUsage() const { vespalib::MemoryUsage usage; usage.merge(_wordStore.getMemoryUsage()); @@ -225,83 +233,147 @@ FieldIndex::getMemoryUsage() const return usage; } +namespace { + +template <bool interleaved_features> +class MemoryTermBlueprint : public SimpleLeafBlueprint { +private: + using FieldIndexType = FieldIndex<interleaved_features>; + using PostingListIteratorType = typename FieldIndexType::PostingList::ConstIterator; + GenerationHandler::Guard _guard; + PostingListIteratorType _posting_itr; + const FeatureStore& _feature_store; + const uint32_t _field_id; + const bool _use_bit_vector; + +public: + MemoryTermBlueprint(GenerationHandler::Guard&& guard, + PostingListIteratorType posting_itr, + const FeatureStore& feature_store, + const FieldSpecBase& field, + uint32_t field_id, + bool use_bit_vector) + : SimpleLeafBlueprint(field), + _guard(), + _posting_itr(posting_itr), + _feature_store(feature_store), + _field_id(field_id), + _use_bit_vector(use_bit_vector) + { + _guard = std::move(guard); + HitEstimate estimate(_posting_itr.size(), !_posting_itr.valid()); + setEstimate(estimate); + } + + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray& tfmda, bool) const override { + using PostingIteratorType = PostingIterator<interleaved_features>; + auto result = std::make_unique<PostingIteratorType>(_posting_itr, _feature_store, _field_id, tfmda); + if (_use_bit_vector) { + LOG(debug, "Return BooleanMatchIteratorWrapper: field_id(%u), doc_count(%zu)", + _field_id, _posting_itr.size()); + return std::make_unique<BooleanMatchIteratorWrapper>(std::move(result), tfmda); + } + LOG(debug, "Return PostingIterator: field_id(%u), doc_count(%zu)", + _field_id, _posting_itr.size()); + return result; + } +}; + +} + +template <bool interleaved_features> +std::unique_ptr<queryeval::SimpleLeafBlueprint> +FieldIndex<interleaved_features>::make_term_blueprint(const vespalib::string& term, + const queryeval::FieldSpecBase& field, + uint32_t field_id) +{ + auto guard = takeGenerationGuard(); + auto posting_itr = findFrozen(term); + bool use_bit_vector = field.isFilter(); + return std::make_unique<MemoryTermBlueprint<interleaved_features>> + (std::move(guard), posting_itr, getFeatureStore(), field, field_id, use_bit_vector); +} + +template +class FieldIndex<false>; + } namespace search::btree { template -class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; +class BTreeNodeDataWrap<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; +class BTreeNodeT<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; #if 0 template -class BTreeNodeT<memoryindex::FieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; #endif template -class BTreeNodeTT<memoryindex::FieldIndex::WordKey, +class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey, datastore::EntryRef, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; template -class BTreeNodeTT<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeInternalNode<memoryindex::FieldIndex::WordKey, +class BTreeInternalNode<memoryindex::FieldIndexBase::WordKey, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; template -class BTreeLeafNode<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeLeafNode<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeStore<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeStore<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeIterator<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeIterator<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, + const memoryindex::FieldIndexBase::KeyComp, BTreeDefaultTraits>; template -class BTree<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTree<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, + const memoryindex::FieldIndexBase::KeyComp, BTreeDefaultTraits>; template -class BTreeRoot<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeRoot<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, + const memoryindex::FieldIndexBase::KeyComp, BTreeDefaultTraits>; template -class BTreeRootBase<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeRootBase<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeAllocator<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index d5df2fa49c8..05665945800 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -2,26 +2,20 @@ #pragma once -#include "feature_store.h" -#include "field_index_remover.h" -#include "word_store.h" +#include "field_index_base.h" #include "posting_list_entry.h" -#include <vespa/searchlib/index/docidandfeatures.h> -#include <vespa/searchlib/index/field_length_calculator.h> #include <vespa/searchlib/index/indexbuilder.h> #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btreeroot.h> #include <vespa/vespalib/btree/btreestore.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/memoryusage.h> namespace search::memoryindex { -class OrderedFieldIndexInserter; +class IOrderedFieldIndexInserter; /** - * Memory index for a single field using lock-free B-Trees in underlying components. + * Implementation of memory index for a single field using lock-free B-Trees in underlying components. * * It consists of the following components: * - WordStore containing all unique words in this field (across all documents). @@ -32,94 +26,24 @@ class OrderedFieldIndexInserter; * This information is unpacked and used during ranking. * * Elements in the three stores are accessed using 32-bit references / handles. + * + * The template parameter specifies whether the underlying posting lists have interleaved features or not. */ -class FieldIndex { +template <bool interleaved_features> +class FieldIndex : public FieldIndexBase { public: // Mapping from docid -> feature ref - using PostingList = btree::BTreeRoot<uint32_t, PostingListEntry, search::btree::NoAggregated>; - using PostingListStore = btree::BTreeStore<uint32_t, PostingListEntry, + using PostingListEntryType = PostingListEntry<interleaved_features>; + using PostingList = btree::BTreeRoot<uint32_t, PostingListEntryType, search::btree::NoAggregated>; + using PostingListStore = btree::BTreeStore<uint32_t, PostingListEntryType, search::btree::NoAggregated, std::less<uint32_t>, btree::BTreeDefaultTraits>; - using PostingListKeyDataType = PostingListStore::KeyDataType; - - struct WordKey { - datastore::EntryRef _wordRef; - - explicit WordKey(datastore::EntryRef wordRef) : _wordRef(wordRef) { } - WordKey() : _wordRef() { } - - friend vespalib::asciistream & - operator<<(vespalib::asciistream & os, const WordKey & rhs); - }; - - class KeyComp { - private: - const WordStore &_wordStore; - const vespalib::stringref _word; - - const char *getWord(datastore::EntryRef wordRef) const { - if (wordRef.valid()) { - return _wordStore.getWord(wordRef); - } - return _word.data(); - } - - public: - KeyComp(const WordStore &wordStore, const vespalib::stringref word) - : _wordStore(wordStore), - _word(word) - { } - - bool operator()(const WordKey & lhs, const WordKey & rhs) const { - int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); - return cmpres < 0; - } - }; - - using PostingListPtr = uint32_t; - using DictionaryTree = btree::BTree<WordKey, PostingListPtr, - search::btree::NoAggregated, - const KeyComp>; -private: - using GenerationHandler = vespalib::GenerationHandler; - - WordStore _wordStore; - uint64_t _numUniqueWords; - GenerationHandler _generationHandler; - DictionaryTree _dict; - PostingListStore _postingListStore; - FeatureStore _featureStore; - uint32_t _fieldId; - FieldIndexRemover _remover; - std::unique_ptr<OrderedFieldIndexInserter> _inserter; - index::FieldLengthCalculator _calculator; - -public: - datastore::EntryRef addWord(const vespalib::stringref word) { - _numUniqueWords++; - return _wordStore.addWord(word); - } - - datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) { - return _featureStore.addFeatures(_fieldId, features).first; - } - - FieldIndex(const index::Schema& schema, uint32_t fieldId); - FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info); - ~FieldIndex(); - PostingList::Iterator find(const vespalib::stringref word) const; - - PostingList::ConstIterator - findFrozen(const vespalib::stringref word) const; - - uint64_t getNumUniqueWords() const { return _numUniqueWords; } - const FeatureStore & getFeatureStore() const { return _featureStore; } - const WordStore &getWordStore() const { return _wordStore; } - OrderedFieldIndexInserter &getInserter() const { return *_inserter; } - index::FieldLengthCalculator &get_calculator() { return _calculator; } + using PostingListKeyDataType = typename PostingListStore::KeyDataType; private: + PostingListStore _postingListStore; + void freeze() { _postingListStore.freeze(); _dict.getAllocator().freeze(); @@ -146,27 +70,31 @@ private: } public: - GenerationHandler::Guard takeGenerationGuard() { - return _generationHandler.takeGuard(); - } + FieldIndex(const index::Schema& schema, uint32_t fieldId); + FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info); + ~FieldIndex(); - void - compactFeatures(); + typename PostingList::Iterator find(const vespalib::stringref word) const; + typename PostingList::ConstIterator findFrozen(const vespalib::stringref word) const; - void dump(search::index::IndexBuilder & indexBuilder); + void compactFeatures() override; - vespalib::MemoryUsage getMemoryUsage() const; - DictionaryTree &getDictionaryTree() { return _dict; } + void dump(search::index::IndexBuilder & indexBuilder) override; + + vespalib::MemoryUsage getMemoryUsage() const override; PostingListStore &getPostingListStore() { return _postingListStore; } - FieldIndexRemover &getDocumentRemover() { return _remover; } - void commit() { + void commit() override { _remover.flush(); freeze(); transferHoldLists(); incGeneration(); trimHoldLists(); } + + std::unique_ptr<queryeval::SimpleLeafBlueprint> make_term_blueprint(const vespalib::string& term, + const queryeval::FieldSpecBase& field, + uint32_t field_id) override; }; } @@ -174,82 +102,82 @@ public: namespace search::btree { extern template -class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey, +class BTreeNodeDataWrap<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeT<memoryindex::FieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; #if 0 extern template -class BTreeNodeT<memoryindex::FieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; #endif extern template -class BTreeNodeTT<memoryindex::FieldIndex::WordKey, +class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey, datastore::EntryRef, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; extern template -class BTreeNodeTT<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeInternalNode<memoryindex::FieldIndex::WordKey, +class BTreeInternalNode<memoryindex::FieldIndexBase::WordKey, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; extern template -class BTreeLeafNode<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeLeafNode<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeStore<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeStore<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeIterator<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeIterator<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, + const memoryindex::FieldIndexBase::KeyComp, BTreeDefaultTraits>; extern template -class BTree<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTree<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, + const memoryindex::FieldIndexBase::KeyComp, BTreeDefaultTraits>; extern template -class BTreeRoot<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeRoot<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - const memoryindex::FieldIndex::KeyComp, - BTreeDefaultTraits>; + const memoryindex::FieldIndexBase::KeyComp, + BTreeDefaultTraits>; extern template -class BTreeRootBase<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeRootBase<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey, - memoryindex::FieldIndex::PostingListPtr, +class BTreeNodeAllocator<memoryindex::FieldIndexBase::WordKey, + memoryindex::FieldIndexBase::PostingListPtr, search::btree::NoAggregated, - BTreeDefaultTraits::INTERNAL_SLOTS, - BTreeDefaultTraits::LEAF_SLOTS>; + BTreeDefaultTraits::INTERNAL_SLOTS, + BTreeDefaultTraits::LEAF_SLOTS>; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp new file mode 100644 index 00000000000..ee1fee3d935 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp @@ -0,0 +1,36 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "field_index_base.h" +#include "i_ordered_field_index_inserter.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace search::memoryindex { + +vespalib::asciistream & +operator<<(vespalib::asciistream& os, const FieldIndexBase::WordKey& rhs) +{ + os << "wr(" << rhs._wordRef.ref() << ")"; + return os; +} + +FieldIndexBase::FieldIndexBase(const index::Schema& schema, uint32_t fieldId) + : FieldIndexBase(schema, fieldId, index::FieldLengthInfo()) +{ +} + +FieldIndexBase::FieldIndexBase(const index::Schema& schema, uint32_t fieldId, + const index::FieldLengthInfo& info) + : _wordStore(), + _numUniqueWords(0), + _generationHandler(), + _dict(), + _featureStore(schema), + _fieldId(fieldId), + _remover(_wordStore), + _inserter(), + _calculator(info) +{ +} + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h new file mode 100644 index 00000000000..7efec1f2ae8 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h @@ -0,0 +1,119 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "feature_store.h" +#include "field_index_remover.h" +#include "i_field_index.h" +#include "word_store.h" +#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/searchlib/index/field_length_calculator.h> +#include <vespa/vespalib/btree/btree.h> +#include <vespa/vespalib/btree/btreenodeallocator.h> +#include <vespa/vespalib/btree/btreeroot.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/memoryusage.h> + +namespace search::memoryindex { + +class IOrderedFieldIndexInserter; + +/** + * Abstract base class for implementation of memory index for a single field. + * + * Contains all components that are not dependent of the posting list format. + */ +class FieldIndexBase : public IFieldIndex { +public: + /** + * Class representing a word used as key in the dictionary. + */ + struct WordKey { + datastore::EntryRef _wordRef; + + explicit WordKey(datastore::EntryRef wordRef) : _wordRef(wordRef) { } + WordKey() : _wordRef() { } + + friend vespalib::asciistream& + operator<<(vespalib::asciistream& os, const WordKey& rhs); + }; + + /** + * Comparator class for words used in the dictionary. + */ + class KeyComp { + private: + const WordStore& _wordStore; + const vespalib::stringref _word; + + const char* getWord(datastore::EntryRef wordRef) const { + if (wordRef.valid()) { + return _wordStore.getWord(wordRef); + } + return _word.data(); + } + + public: + KeyComp(const WordStore& wordStore, const vespalib::stringref word) + : _wordStore(wordStore), + _word(word) + { } + + bool operator()(const WordKey& lhs, const WordKey& rhs) const { + int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); + return cmpres < 0; + } + }; + + using PostingListPtr = uint32_t; + using DictionaryTree = btree::BTree<WordKey, PostingListPtr, + search::btree::NoAggregated, + const KeyComp>; + +protected: + using GenerationHandler = vespalib::GenerationHandler; + + WordStore _wordStore; + uint64_t _numUniqueWords; + GenerationHandler _generationHandler; + DictionaryTree _dict; + FeatureStore _featureStore; + uint32_t _fieldId; + FieldIndexRemover _remover; + std::unique_ptr<IOrderedFieldIndexInserter> _inserter; + index::FieldLengthCalculator _calculator; + + void incGeneration() { + _generationHandler.incGeneration(); + } + +public: + datastore::EntryRef addWord(const vespalib::stringref word) { + _numUniqueWords++; + return _wordStore.addWord(word); + } + + datastore::EntryRef addFeatures(const index::DocIdAndFeatures& features) { + return _featureStore.addFeatures(_fieldId, features).first; + } + + FieldIndexBase(const index::Schema& schema, uint32_t fieldId); + FieldIndexBase(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info); + + uint64_t getNumUniqueWords() const override { return _numUniqueWords; } + const FeatureStore& getFeatureStore() const override { return _featureStore; } + const WordStore& getWordStore() const override { return _wordStore; } + IOrderedFieldIndexInserter& getInserter() override { return *_inserter; } + index::FieldLengthCalculator& get_calculator() override { return _calculator; } + + GenerationHandler::Guard takeGenerationGuard() override { + return _generationHandler.takeGuard(); + } + + DictionaryTree& getDictionaryTree() { return _dict; } + FieldIndexRemover& getDocumentRemover() override { return _remover; } + +}; + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index 40b1e8f360f..dc7d35a755d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -34,7 +34,7 @@ FieldIndexCollection::FieldIndexCollection(const Schema& schema, const IFieldLen { for (uint32_t fieldId = 0; fieldId < _numFields; ++fieldId) { const auto& field = schema.getIndexField(fieldId); - auto fieldIndex = std::make_unique<FieldIndex>(schema, fieldId, inspector.get_field_length_info(field.getName())); + auto fieldIndex = std::make_unique<FieldIndex<false>>(schema, fieldId, inspector.get_field_length_info(field.getName())); _fieldIndexes.push_back(std::move(fieldIndex)); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index 53f42658d0a..a737175d346 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -3,9 +3,14 @@ #pragma once #include "i_field_index_collection.h" -#include "field_index.h" +#include "i_field_index.h" +#include <memory> +#include <vector> -namespace search::index { class IFieldLengthInspector; } +namespace search::index { + class IFieldLengthInspector; + class Schema; +} namespace search::memoryindex { @@ -19,26 +24,15 @@ class FieldInverter; * for a given word in a given field. */ class FieldIndexCollection : public IFieldIndexCollection { -public: - using PostingList = FieldIndex::PostingList; - private: using GenerationHandler = vespalib::GenerationHandler; - std::vector<std::unique_ptr<FieldIndex>> _fieldIndexes; + std::vector<std::unique_ptr<IFieldIndex>> _fieldIndexes; uint32_t _numFields; public: FieldIndexCollection(const index::Schema& schema, const index::IFieldLengthInspector& inspector); ~FieldIndexCollection(); - PostingList::Iterator find(const vespalib::stringref word, - uint32_t fieldId) const { - return _fieldIndexes[fieldId]->find(word); - } - - PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { - return _fieldIndexes[fieldId]->findFrozen(word); - } uint64_t getNumUniqueWords() const { uint64_t numUniqueWords = 0; @@ -52,11 +46,11 @@ public: vespalib::MemoryUsage getMemoryUsage() const; - FieldIndex *getFieldIndex(uint32_t fieldId) const { + IFieldIndex *getFieldIndex(uint32_t fieldId) const { return _fieldIndexes[fieldId].get(); } - const std::vector<std::unique_ptr<FieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } + const std::vector<std::unique_ptr<IFieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } uint32_t getNumFields() const { return _numFields; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_field_index.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index.h new file mode 100644 index 00000000000..86082c08d36 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index.h @@ -0,0 +1,47 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/memoryusage.h> + +namespace search::index { +class FieldLengthCalculator; +class IndexBuilder; +} + +namespace search::memoryindex { + +class FeatureStore; +class FieldIndexRemover; +class IOrderedFieldIndexInserter; +class WordStore; + +/** + * Interface for a memory index for a single field as seen from the FieldIndexCollection. + */ +class IFieldIndex { +public: + virtual ~IFieldIndex() {} + + virtual uint64_t getNumUniqueWords() const = 0; + virtual vespalib::MemoryUsage getMemoryUsage() const = 0; + virtual const FeatureStore& getFeatureStore() const = 0; + virtual const WordStore& getWordStore() const = 0; + virtual IOrderedFieldIndexInserter& getInserter() = 0; + virtual FieldIndexRemover& getDocumentRemover() = 0; + virtual index::FieldLengthCalculator& get_calculator() = 0; + virtual void compactFeatures() = 0; + virtual void dump(search::index::IndexBuilder& indexBuilder) = 0; + + virtual std::unique_ptr<queryeval::SimpleLeafBlueprint> make_term_blueprint(const vespalib::string& term, + const queryeval::FieldSpecBase& field, + uint32_t field_id) = 0; + + // Should only be directly used by unit tests + virtual vespalib::GenerationHandler::Guard takeGenerationGuard() = 0; + virtual void commit() = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h index cf10db3c4d8..4da0844da58 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/datastore/entryref.h> #include <cstdint> namespace search::index { class DocIdAndFeatures; } @@ -30,6 +31,11 @@ public: virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; /** + * Returns the reference to the current word (only used by unit tests). + */ + virtual datastore::EntryRef getWordRef() const = 0; + + /** * Remove (word, docId) tuple. */ virtual void remove(uint32_t docId) = 0; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 6686745f8c2..d3d3004100c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -3,16 +3,15 @@ #include "document_inverter.h" #include "field_index_collection.h" #include "memory_index.h" -#include "posting_iterator.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/searchlib/index/field_length_calculator.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.memoryindex.memory_index"); @@ -20,19 +19,17 @@ LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; using vespalib::LockGuard; -using vespalib::GenerationHandler; namespace search { -using fef::TermFieldMatchDataArray; using index::FieldLengthInfo; using index::IFieldLengthInspector; using index::IndexBuilder; using index::Schema; using index::SchemaUtil; -using query::NumberTerm; using query::LocationTerm; using query::Node; +using query::NumberTerm; using query::PredicateQuery; using query::PrefixTerm; using query::RangeTerm; @@ -40,16 +37,12 @@ using query::RegExpTerm; using query::StringTerm; using query::SubstringTerm; using query::SuffixTerm; -using queryeval::SearchIterator; -using queryeval::Searchable; -using queryeval::CreateBlueprintVisitorHelper; using queryeval::Blueprint; -using queryeval::BooleanMatchIteratorWrapper; +using queryeval::CreateBlueprintVisitorHelper; using queryeval::EmptyBlueprint; -using queryeval::FieldSpecBase; -using queryeval::FieldSpecBaseList; using queryeval::FieldSpec; using queryeval::IRequestContext; +using queryeval::Searchable; } @@ -141,47 +134,6 @@ MemoryIndex::dump(IndexBuilder &indexBuilder) namespace { -class MemTermBlueprint : public queryeval::SimpleLeafBlueprint { -private: - GenerationHandler::Guard _genGuard; - FieldIndex::PostingList::ConstIterator _pitr; - const FeatureStore &_featureStore; - const uint32_t _fieldId; - const bool _useBitVector; - -public: - MemTermBlueprint(GenerationHandler::Guard &&genGuard, - FieldIndex::PostingList::ConstIterator pitr, - const FeatureStore &featureStore, - const FieldSpecBase &field, - uint32_t fieldId, - bool useBitVector) - : SimpleLeafBlueprint(field), - _genGuard(), - _pitr(pitr), - _featureStore(featureStore), - _fieldId(fieldId), - _useBitVector(useBitVector) - { - _genGuard = std::move(genGuard); - HitEstimate estimate(_pitr.size(), !_pitr.valid()); - setEstimate(estimate); - } - - SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { - auto search = std::make_unique<PostingIterator>(_pitr, _featureStore, _fieldId, tfmda); - if (_useBitVector) { - LOG(debug, "Return BooleanMatchIteratorWrapper: fieldId(%u), docCount(%zu)", - _fieldId, _pitr.size()); - return std::make_unique<BooleanMatchIteratorWrapper>(std::move(search), tfmda); - } - LOG(debug, "Return PostingIterator: fieldId(%u), docCount(%zu)", - _fieldId, _pitr.size()); - return search; - } - -}; - /** * Determines the correct Blueprint to use. **/ @@ -207,13 +159,8 @@ public: const vespalib::string termStr = queryeval::termAsString(n); LOG(debug, "searching for '%s' in '%s'", termStr.c_str(), _field.getName().c_str()); - FieldIndex *fieldIndex = _fieldIndexes.getFieldIndex(_fieldId); - GenerationHandler::Guard genGuard = fieldIndex->takeGenerationGuard(); - FieldIndex::PostingList::ConstIterator pitr = fieldIndex->findFrozen(termStr); - bool useBitVector = _field.isFilter(); - setResult(std::make_unique<MemTermBlueprint>(std::move(genGuard), pitr, - fieldIndex->getFeatureStore(), - _field, _fieldId, useBitVector)); + IFieldIndex* fieldIndex = _fieldIndexes.getFieldIndex(_fieldId); + setResult(fieldIndex->make_term_blueprint(termStr, _field, _fieldId)); } void visit(LocationTerm &n) override { visitTerm(n); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 1d38e88b747..c6524a2fc64 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -27,7 +27,8 @@ const vespalib::string emptyWord = ""; } -OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) +template <bool interleaved_features> +OrderedFieldIndexInserter<interleaved_features>::OrderedFieldIndexInserter(FieldIndexType& fieldIndex) : _word(), _prevDocId(noDocId), _prevAdd(false), @@ -39,10 +40,12 @@ OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) { } -OrderedFieldIndexInserter::~OrderedFieldIndexInserter() = default; +template <bool interleaved_features> +OrderedFieldIndexInserter<interleaved_features>::~OrderedFieldIndexInserter() = default; +template <bool interleaved_features> void -OrderedFieldIndexInserter::flushWord() +OrderedFieldIndexInserter<interleaved_features>::flushWord() { if (_removes.empty() && _adds.empty()) { return; @@ -64,21 +67,24 @@ OrderedFieldIndexInserter::flushWord() _adds.clear(); } +template <bool interleaved_features> void -OrderedFieldIndexInserter::flush() +OrderedFieldIndexInserter<interleaved_features>::flush() { flushWord(); _listener.flush(); } +template <bool interleaved_features> void -OrderedFieldIndexInserter::commit() +OrderedFieldIndexInserter<interleaved_features>::commit() { _fieldIndex.commit(); } +template <bool interleaved_features> void -OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) +OrderedFieldIndexInserter<interleaved_features>::setNextWord(const vespalib::stringref word) { // TODO: Adjust here if zero length words should be legal. assert(_word < word); @@ -103,22 +109,24 @@ OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) assert(_word == wordStore.getWord(_dItr.getKey()._wordRef)); } +template <bool interleaved_features> void -OrderedFieldIndexInserter::add(uint32_t docId, - const index::DocIdAndFeatures &features) +OrderedFieldIndexInserter<interleaved_features>::add(uint32_t docId, + const index::DocIdAndFeatures &features) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId || (_prevDocId == docId && !_prevAdd)); datastore::EntryRef featureRef = _fieldIndex.addFeatures(features); - _adds.push_back(PostingListKeyDataType(docId, featureRef)); + _adds.push_back(PostingListKeyDataType(docId, PostingListEntryType(featureRef))); _listener.insert(_dItr.getKey()._wordRef, docId); _prevDocId = docId; _prevAdd = true; } +template <bool interleaved_features> void -OrderedFieldIndexInserter::remove(uint32_t docId) +OrderedFieldIndexInserter<interleaved_features>::remove(uint32_t docId) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId); @@ -127,8 +135,9 @@ OrderedFieldIndexInserter::remove(uint32_t docId) _prevAdd = false; } +template <bool interleaved_features> void -OrderedFieldIndexInserter::rewind() +OrderedFieldIndexInserter<interleaved_features>::rewind() { assert(_removes.empty() && _adds.empty()); _word = ""; @@ -137,10 +146,14 @@ OrderedFieldIndexInserter::rewind() _dItr.begin(); } +template <bool interleaved_features> datastore::EntryRef -OrderedFieldIndexInserter::getWordRef() const +OrderedFieldIndexInserter<interleaved_features>::getWordRef() const { return _dItr.getKey()._wordRef; } +template +class OrderedFieldIndexInserter<false>; + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h index 18765f9bae3..0e04b126f32 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h @@ -18,26 +18,30 @@ class IFieldIndexInsertListener; * and for each word updating the posting list with docId adds / removes. * * Insert order must be properly sorted, first by word, then by docId. + * + * The template parameter specifies whether the posting lists of the field index have interleaved features or not. */ +template <bool interleaved_features> class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { private: vespalib::stringref _word; uint32_t _prevDocId; bool _prevAdd; - using DictionaryTree = FieldIndex::DictionaryTree; - using PostingListStore = FieldIndex::PostingListStore; - using KeyComp = FieldIndex::KeyComp; - using WordKey = FieldIndex::WordKey; - using PostingListKeyDataType = FieldIndex::PostingListKeyDataType; - FieldIndex &_fieldIndex; - DictionaryTree::Iterator _dItr; + using FieldIndexType = FieldIndex<interleaved_features>; + using DictionaryTree = typename FieldIndexType::DictionaryTree; + using PostingListStore = typename FieldIndexType::PostingListStore; + using KeyComp = typename FieldIndexType::KeyComp; + using WordKey = typename FieldIndexType::WordKey; + using PostingListEntryType = typename FieldIndexType::PostingListEntryType; + using PostingListKeyDataType = typename FieldIndexType::PostingListKeyDataType; + FieldIndexType& _fieldIndex; + typename DictionaryTree::Iterator _dItr; IFieldIndexInsertListener &_listener; // Pending changes to posting list for (_word) std::vector<uint32_t> _removes; std::vector<PostingListKeyDataType> _adds; - static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max(); @@ -49,7 +53,7 @@ private: void flushWord(); public: - OrderedFieldIndexInserter(FieldIndex &fieldIndex); + OrderedFieldIndexInserter(FieldIndexType& fieldIndex); ~OrderedFieldIndexInserter() override; void setNextWord(const vespalib::stringref word) override; void add(uint32_t docId, const index::DocIdAndFeatures &features) override; @@ -71,8 +75,7 @@ public: */ void rewind() override; - // Used by unit test - datastore::EntryRef getWordRef() const; + datastore::EntryRef getWordRef() const override; }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index 290aa16dfe4..0e84c2b7968 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -13,10 +13,11 @@ LOG_SETUP(".searchlib.memoryindex.posting_iterator"); namespace search::memoryindex { -PostingIterator::PostingIterator(FieldIndex::PostingList::ConstIterator itr, - const FeatureStore & featureStore, - uint32_t packedIndex, - const fef::TermFieldMatchDataArray & matchData) : +template <bool interleaved_features> +PostingIterator<interleaved_features>::PostingIterator(PostingListIteratorType itr, + const FeatureStore& featureStore, + uint32_t packedIndex, + const fef::TermFieldMatchDataArray& matchData) : queryeval::RankedSearchIteratorBase(matchData), _itr(itr), _featureStore(featureStore), @@ -25,10 +26,12 @@ PostingIterator::PostingIterator(FieldIndex::PostingList::ConstIterator itr, _featureStore.setupForField(packedIndex, _featureDecoder); } -PostingIterator::~PostingIterator() {} +template <bool interleaved_features> +PostingIterator<interleaved_features>::~PostingIterator() = default; +template <bool interleaved_features> void -PostingIterator::initRange(uint32_t begin, uint32_t end) +PostingIterator<interleaved_features>::initRange(uint32_t begin, uint32_t end) { SearchIterator::initRange(begin, end); _itr.lower_bound(begin); @@ -40,8 +43,9 @@ PostingIterator::initRange(uint32_t begin, uint32_t end) clearUnpacked(); } +template <bool interleaved_features> void -PostingIterator::doSeek(uint32_t docId) +PostingIterator<interleaved_features>::doSeek(uint32_t docId) { if (getUnpacked()) { clearUnpacked(); @@ -54,8 +58,9 @@ PostingIterator::doSeek(uint32_t docId) } } +template <bool interleaved_features> void -PostingIterator::doUnpack(uint32_t docId) +PostingIterator<interleaved_features>::doUnpack(uint32_t docId) { if (!_matchData.valid() || getUnpacked()) { return; @@ -69,59 +74,62 @@ PostingIterator::doUnpack(uint32_t docId) setUnpacked(); } +template +class PostingIterator<false>; + } namespace search::btree { template class BTreeNodeTT<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; template class BTreeLeafNode<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; template class BTreeNodeStore<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template class BTreeIteratorBase<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS, BTreeDefaultTraits::PATH_SIZE>; template class BTreeIterator<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>; template class BTree<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>; template class BTreeRoot<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>; template class BTreeRootBase<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template class BTreeNodeAllocator<uint32_t, - search::memoryindex::PostingListEntry, + search::memoryindex::PostingListEntry<false>, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index de337ef49f3..f029c837cf7 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -9,10 +9,15 @@ namespace search::memoryindex { /** * Search iterator for memory field index posting list. + * + * The template parameter specifies whether the wrapped posting list has interleaved features or not. */ +template <bool interleaved_features> class PostingIterator : public queryeval::RankedSearchIteratorBase { private: - FieldIndex::PostingList::ConstIterator _itr; + using FieldIndexType = FieldIndex<interleaved_features>; + using PostingListIteratorType = typename FieldIndexType::PostingList::ConstIterator; + PostingListIteratorType _itr; const FeatureStore &_featureStore; FeatureStore::DecodeContextCooked _featureDecoder; @@ -25,7 +30,7 @@ public: * @param packedIndex the field or field collection owning features. * @param matchData the match data to unpack features into. **/ - PostingIterator(FieldIndex::PostingList::ConstIterator itr, + PostingIterator(PostingListIteratorType itr, const FeatureStore &featureStore, uint32_t packedIndex, const fef::TermFieldMatchDataArray &matchData); diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h index b28cd87736c..373de21e836 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h @@ -9,11 +9,12 @@ namespace search::memoryindex { /** * Entry per document in memory index posting list. */ +template <bool interleaved_features> class PostingListEntry { mutable datastore::EntryRef _features; // reference to compressed features public: - PostingListEntry(datastore::EntryRef features) + explicit PostingListEntry(datastore::EntryRef features) : _features(features) { } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index f5300430bea..a4996c931e2 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -129,10 +129,10 @@ search::queryeval::SearchIterator * FakeMemTreeOcc:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { - return new search::memoryindex::PostingIterator(_tree.begin(_allocator), - _mgr._featureStore, - _packedIndex, - matchData); + return new search::memoryindex::PostingIterator<false>(_tree.begin(_allocator), + _mgr._featureStore, + _packedIndex, + matchData); } @@ -267,7 +267,7 @@ FakeMemTreeOccMgr::flush() } } else { if (!itr.valid() || docId < itr.getKey()) { - tree.insert(itr, docId, i->getFeatureRef()); + tree.insert(itr, docId, PostingListEntryType(i->getFeatureRef())); } } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index f0363500559..69114611fe6 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -9,18 +9,18 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search { -namespace fakedata { +namespace search::fakedata { -class FakeMemTreeOccMgr : public FakeWord::RandomizedWriter -{ +class FakeMemTreeOccMgr : public FakeWord::RandomizedWriter { public: - typedef memoryindex::FieldIndex::PostingList Tree; - typedef Tree::NodeAllocatorType NodeAllocator; - typedef memoryindex::FeatureStore FeatureStore; - typedef datastore::EntryRef EntryRef; - typedef index::Schema Schema; - typedef bitcompression::PosOccFieldsParams PosOccFieldsParams; + // TODO: Create implementation for "interleaved features" posting list as well. + using Tree = memoryindex::FieldIndex<false>::PostingList; + using PostingListEntryType = memoryindex::FieldIndex<false>::PostingListEntryType; + using NodeAllocator = Tree::NodeAllocatorType; + using FeatureStore = memoryindex::FeatureStore; + using EntryRef = datastore::EntryRef; + using Schema = index::Schema; + using PosOccFieldsParams = bitcompression::PosOccFieldsParams; vespalib::GenerationHandler _generationHandler; NodeAllocator _allocator; @@ -179,6 +179,4 @@ public: queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index 9680da7af11..c0ea7be0ce1 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -14,15 +14,14 @@ class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { bool _show_interleaved_features; uint32_t _fieldId; - void - addComma() - { + void addComma() { if (!_first) { _ss << ","; } else { _first = false; } } + public: OrderedFieldIndexInserter() : _ss(), @@ -33,23 +32,17 @@ public: { } - virtual void - setNextWord(const vespalib::stringref word) override - { + virtual void setNextWord(const vespalib::stringref word) override { addComma(); _ss << "w=" << word; } - void - setFieldId(uint32_t fieldId) - { + void setFieldId(uint32_t fieldId) { _fieldId = fieldId; } - virtual void - add(uint32_t docId, - const index::DocIdAndFeatures &features) override - { + virtual void add(uint32_t docId, + const index::DocIdAndFeatures &features) override { (void) features; addComma(); _ss << "a=" << docId; @@ -85,9 +78,9 @@ public: } } - virtual void - remove(uint32_t docId) override - { + virtual datastore::EntryRef getWordRef() const override { return datastore::EntryRef(); } + + virtual void remove(uint32_t docId) override { addComma(); _ss << "r=" << docId; } @@ -99,15 +92,11 @@ public: _ss << "f=" << _fieldId; } - std::string - toStr() const - { + std::string toStr() const { return _ss.str(); } - void - reset() - { + void reset() { _ss.str(""); _first = true; _verbose = false; diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h index eeb09898aa2..268bf834d21 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h @@ -12,7 +12,7 @@ namespace search::memoryindex::test { */ class WrapInserter { private: - OrderedFieldIndexInserter& _inserter; + IOrderedFieldIndexInserter& _inserter; public: WrapInserter(FieldIndexCollection& field_indexes, uint32_t field_id) @@ -20,7 +20,7 @@ public: { } - WrapInserter(FieldIndex& field_index) + WrapInserter(IFieldIndex& field_index) : _inserter(field_index.getInserter()) { } diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index 2783d0e3ebf..bf41b77ab05 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -80,7 +80,7 @@ struct SpecList bool compare(MirrorAPI &api, const char *pattern, SpecList expect) { - for (int i = 0; i < 250; ++i) { + for (int i = 0; i < 600; ++i) { SpecList actual(api.lookup(pattern)); if (actual == expect) { return true; diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h index 128a7dc9424..07137263cf6 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/hashtable.h> #include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/select.h> #include <vector> namespace vespalib { @@ -29,7 +30,7 @@ struct LruParam { typedef LinkedValue<V> LV; typedef std::pair< K, LV > value_type; - typedef std::_Select1st< value_type > select_key; + typedef vespalib::Select1st< value_type > select_key; typedef K Key; typedef V Value; typedef H Hash; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java index ee2ee0add4c..40377da30ef 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -2,9 +2,21 @@ package ai.vespa.hosted.cd; /** - * @deprecated Use {@link UpgradeTest}. + * Tests that assert continuity of behaviour for Vespa application deployments, through upgrades. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical upgrade test is to do some operations against a test deployment prior to upgrade, like feed and + * search for some documents, perhaps recording some metrics from the deployment, and then to upgrade it, + * repeat the exercise, and compare the results from pre and post upgrade. + * + * TODO Split in platform upgrades and application upgrades? + * + * @author jonmv */ -@Deprecated -public class StagingTest { - +public interface StagingTest { + // Want to verify documents are not damaged by upgrade. + // May want to verify metrics during upgrade. } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 6a8d1b4cbe4..c67d86fc8de 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -2,9 +2,29 @@ package ai.vespa.hosted.cd; /** - * @deprecated use {@link FunctionalTest}. + * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical system test is to feed some documents, optionally verifying that the documents have been processed + * as expected, and then to see that queries give the expected results. Another common use is to verify integration + * with external services. + * + * @author jonmv */ -@Deprecated -public class SystemTest { - +public interface SystemTest { + // Want to feed some documents. + // Want to verify document processing and routing is as expected. + // Want to check recall on those documents. + // Want to verify queries give expected documents. + // Want to verify searchers. + // Want to verify updates. + // Want to verify deletion. + // May want to verify reprocessing. + // Must likely delete documents between tests. + // Must be able to feed documents, setting route. + // Must be able to search. + // Must be able to visit. } diff --git a/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java index a988a3f6fa2..e2da62b6098 100644 --- a/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java +++ b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java @@ -24,6 +24,12 @@ public class MutableInteger { return value; } + /** Increments the value by 1 and returns the value of this *before* incrementing */ + public int next() { + value++; + return value - 1; + } + /** Adds the increment to the current value and returns the resulting value */ public int subtract(int increment) { value -= increment; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index aca2bfc1b0f..02f54b5790a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -210,7 +210,36 @@ public abstract class IndexedTensor implements Tensor { } @Override - public String toString() { return Tensor.toStandardString(this); } + public String toString() { + if (type.rank() == 0) return Tensor.toStandardString(this); + if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) return Tensor.toStandardString(this); + + Indexes indexes = Indexes.of(dimensionSizes); + + StringBuilder b = new StringBuilder(type.toString()).append(":"); + for (int index = 0; index < size(); index++) { + indexes.next(); + + // start brackets + for (int i = 0; i < indexes.rightDimensionsWhichAreAtStart(); i++) + b.append("["); + + // value + if (type.valueType() == TensorType.Value.DOUBLE) + b.append(get(index)); + else if (type.valueType() == TensorType.Value.FLOAT) + b.append(get(index)); // TODO: Use getFloat + else + throw new IllegalStateException("Unexpected value type " + type.valueType()); + + // end bracket and comma + for (int i = 0; i < indexes.rightDimensionsWhichAreAtEnd(); i++) + b.append("]"); + if (index < size() - 1) + b.append(", "); + } + return b.toString(); + } @Override public boolean equals(Object other) { @@ -382,8 +411,10 @@ public abstract class IndexedTensor implements Tensor { DimensionSizes sizes() { return sizes; } + /** Sets a value by its right-adjacent traversal position */ public abstract void cellByDirectIndex(long index, double value); + /** Sets a value by its right-adjacent traversal position */ public abstract void cellByDirectIndex(long index, float value); } @@ -827,6 +858,27 @@ public abstract class IndexedTensor implements Tensor { public abstract void next(); + /** Returns the number of dimensions from the right which are currently at the start position (0) */ + int rightDimensionsWhichAreAtStart() { + int dimension = indexes.length - 1; + int atStartCount = 0; + while (dimension >= 0 && indexes[dimension] == 0) { + atStartCount++; + dimension--; + } + return atStartCount; + } + + /** Returns the number of dimensions from the right which are currently at the end position */ + int rightDimensionsWhichAreAtEnd() { + int dimension = indexes.length - 1; + int atEndCount = 0; + while (dimension >= 0 && indexes[dimension] == dimensionSizes().size(dimension) - 1) { + atEndCount++; + dimension--; + } + return atEndCount; + } } private final static class EmptyIndexes extends Indexes { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 1a210a614cc..c73ff03a0eb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; +import com.yahoo.lang.MutableInteger; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -8,6 +9,7 @@ import com.yahoo.slime.JsonDecoder; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.Type; +import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; @@ -17,7 +19,9 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: * A JSON map containing a 'cells' array. - * See http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor + * See a http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor + * + * @author bratseth */ public class JsonFormat { @@ -54,14 +58,24 @@ public class JsonFormat { } /** Deserializes the given tensor from JSON format */ + // NOTE: This must be kept in sync with com.yahoo.document.json.readers.TensorReader in the document module public static Tensor decode(TensorType type, byte[] jsonTensorValue) { - Tensor.Builder tensorBuilder = Tensor.Builder.of(type); + Tensor.Builder builder = Tensor.Builder.of(type); Inspector root = new JsonDecoder().decode(new Slime(), jsonTensorValue).get(); - Inspector cells = root.field("cells"); + + if (root.field("cells").valid()) + decodeCells(root.field("cells"), builder); + else if (root.field("values").valid()) + decodeValues(root.field("values"), builder); + else if (builder.type().dimensions().stream().anyMatch(d -> d.isIndexed())) // sparse can be empty + throw new IllegalArgumentException("Expected a tensor value to contain either 'cells' or 'values'"); + return builder.build(); + } + + private static void decodeCells(Inspector cells, Tensor.Builder builder) { if ( cells.type() != Type.ARRAY) - throw new IllegalArgumentException("Excepted an array item named 'cells' at the top level"); - cells.traverse((ArrayTraverser) (__, cell) -> decodeCell(cell, tensorBuilder.cell())); - return tensorBuilder.build(); + throw new IllegalArgumentException("Excepted 'cells' to contain an array, not " + cells.type()); + cells.traverse((ArrayTraverser) (__, cell) -> decodeCell(cell, builder.cell())); } private static void decodeCell(Inspector cell, Tensor.Builder.CellBuilder cellBuilder) { @@ -76,4 +90,20 @@ public class JsonFormat { cellBuilder.value(value.asDouble()); } + private static void decodeValues(Inspector values, Tensor.Builder builder) { + if ( ! (builder instanceof IndexedTensor.BoundBuilder)) + throw new IllegalArgumentException("The 'values' field can only be used with dense tensors. " + + "Use 'cells' instead"); + if ( values.type() != Type.ARRAY) + throw new IllegalArgumentException("Excepted 'values' to contain an array, not " + values.type()); + + IndexedTensor.BoundBuilder indexedBuilder = (IndexedTensor.BoundBuilder)builder; + MutableInteger index = new MutableInteger(0); + values.traverse((ArrayTraverser) (__, value) -> { + if (value.type() != Type.LONG && value.type() != Type.DOUBLE) + throw new IllegalArgumentException("Excepted the values array to contain numbers, not " + value.type()); + indexedBuilder.cellByDirectIndex(index.next(), value.asDouble()); + }); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 63fe40565bd..1928971820c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -23,37 +23,42 @@ public class TensorParserTestCase { @Test public void testDenseParsing() { - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor()")).build(), - Tensor.from("tensor():[]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1])")).cell(1.0, 0).build(), - Tensor.from("tensor(x[1]):[1.0]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2])")).cell(1.0, 0).cell(2.0, 1).build(), - Tensor.from("tensor(x[2]):[1.0, 2.0]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[3])")) + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor()")).build(), + "tensor():{0.0}"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor()")).cell(1.3).build(), + "tensor():{1.3}"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[])")).cell(1.0, 0).build(), + "tensor(x[]):{{x:0}:1.0}"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1])")).cell(1.0, 0).build(), + "tensor(x[1]):[1.0]"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2])")).cell(1.0, 0).cell(2.0, 1).build(), + "tensor(x[2]):[1.0, 2.0]"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[3])")) .cell(1.0, 0, 0) .cell(2.0, 0, 1) .cell(3.0, 0, 2) .cell(4.0, 1, 0) .cell(5.0, 1, 1) .cell(6.0, 1, 2).build(), - Tensor.from("tensor(x[2],y[3]):[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1],y[2],z[3])")) + "tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1],y[2],z[3])")) .cell(1.0, 0, 0, 0) .cell(2.0, 0, 0, 1) .cell(3.0, 0, 0, 2) .cell(4.0, 0, 1, 0) .cell(5.0, 0, 1, 1) .cell(6.0, 0, 1, 2).build(), - Tensor.from("tensor(x[1],y[2],z[3]):[[[1.0], [2.0]], [[3.0], [4.0]], [[5.0], [6.0]]]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) + "tensor(x[1],y[2],z[3]):[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]]"); + assertDense(Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) .cell(1.0, 0, 0, 0) .cell(2.0, 0, 1, 0) .cell(3.0, 1, 0, 0) .cell(4.0, 1, 1, 0) .cell(5.0, 2, 0, 0) .cell(6.0, 2, 1, 0).build(), - Tensor.from("tensor(x[3],y[2],z[1]):[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]]")); - assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) + "tensor(x[3],y[2],z[1]):[[[1.0], [2.0]], [[3.0], [4.0]], [[5.0], [6.0]]]"); + assertEquals("Messy input", + Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) .cell( 1.0, 0, 0, 0) .cell( 2.0, 0, 1, 0) .cell( 3.0, 1, 0, 0) @@ -61,6 +66,20 @@ public class TensorParserTestCase { .cell( 5.0, 2, 0, 0) .cell(-6.0, 2, 1, 0).build(), Tensor.from("tensor( x[3],y[2],z[1]) : [ [ [1.0, 2.0, 3.0] , [4.0, 5,-6.0] ] ]")); + assertEquals("Skipping syntactic sugar", + Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) + .cell( 1.0, 0, 0, 0) + .cell( 2.0, 0, 1, 0) + .cell( 3.0, 1, 0, 0) + .cell( 4.0, 1, 1, 0) + .cell( 5.0, 2, 0, 0) + .cell(-6.0, 2, 1, 0).build(), + Tensor.from("tensor( x[3],y[2],z[1]) : [1.0, 2.0, 3.0 , 4.0, 5, -6.0]")); + } + + private void assertDense(Tensor expectedTensor, String denseFormat) { + assertEquals(denseFormat, expectedTensor, Tensor.from(denseFormat)); + assertEquals(denseFormat, expectedTensor.toString()); } @Test diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index c53db160806..3d5d8d1f5ae 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -89,9 +89,9 @@ public class TensorTestCase { @Test public void testCombineInDimensionIndexed() { - Tensor input = Tensor.from("tensor(input[]):{{input:0}:3, {input:1}:7}"); + Tensor input = Tensor.from("tensor(input[2]):{{input:0}:3, {input:1}:7}"); Tensor result = input.concat(11, "input"); - assertEquals("tensor(input[]):{{input:0}:3.0,{input:1}:7.0,{input:2}:11.0}", result.toString()); + assertEquals("tensor(input[3]):[3.0, 7.0, 11.0]", result.toString()); } /** All functions are more throughly tested in searchlib EvaluationTestCase */ diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java index b466307d3b9..4c44cbbf5c7 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java @@ -33,7 +33,7 @@ public class JsonFormatTestCase { @Test public void testDenseTensor() { - Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})")); + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[2])")); builder.cell().label("x", 0).label("y", 0).value(2.0); builder.cell().label("x", 0).label("y", 1).value(3.0); builder.cell().label("x", 1).label("y", 0).value(5.0); @@ -52,6 +52,21 @@ public class JsonFormatTestCase { } @Test + public void testDenseTensorInDenseForm() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[3])")); + builder.cell().label("x", 0).label("y", 0).value(2.0); + builder.cell().label("x", 0).label("y", 1).value(3.0); + builder.cell().label("x", 0).label("y", 2).value(4.0); + builder.cell().label("x", 1).label("y", 0).value(5.0); + builder.cell().label("x", 1).label("y", 1).value(6.0); + builder.cell().label("x", 1).label("y", 2).value(7.0); + Tensor expected = builder.build(); + String denseJson = "{\"values\":[2.0, 3.0, 4.0, 5.0, 6.0, 7.0]}"; + Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test public void testTooManyCells() { TensorType x2 = TensorType.fromSpec("tensor(x[2])"); String json = "{\"cells\":[" + diff --git a/vespalib/src/tests/stllike/asciistream_test.cpp b/vespalib/src/tests/stllike/asciistream_test.cpp index b1ba70e6ae2..fd362d9c49a 100644 --- a/vespalib/src/tests/stllike/asciistream_test.cpp +++ b/vespalib/src/tests/stllike/asciistream_test.cpp @@ -40,10 +40,21 @@ AsciistreamTest::verifyBothWays(T value, const char * expected) os << value; EXPECT_EQUAL(os.str(), string(expected)); EXPECT_EQUAL(os.size(), strlen(expected)); - T v; - os >> v; - EXPECT_EQUAL(value, v); - EXPECT_TRUE(os.empty()); + { + T v; + os >> v; + EXPECT_EQUAL(value, v); + EXPECT_TRUE(os.empty()); + } + + { + os << " " << expected; + T v; + os >> v; + EXPECT_EQUAL(value, v); + EXPECT_TRUE(os.empty()); + EXPECT_EQUAL(0u, os.size()); + } } template <typename T> @@ -72,16 +83,16 @@ AsciistreamTest::testIllegalNumbers() { asciistream is("777777777777"); uint16_t s(0); - EXPECT_EXCEPTION(is >> s, IllegalArgumentException, "An unsigned short can not represent '777777777777'"); + EXPECT_EXCEPTION(is >> s, IllegalArgumentException, "strToInt value '777777777777' is outside of range"); EXPECT_EQUAL(12u, is.size()); uint32_t i(0); - EXPECT_EXCEPTION(is >> i, IllegalArgumentException, "An unsigned int can not represent '777777777777'"); + EXPECT_EXCEPTION(is >> i, IllegalArgumentException, "strToInt value '777777777777' is outside of range"); EXPECT_EQUAL(12u, is.size()); int16_t si(0); - EXPECT_EXCEPTION(is >> si, IllegalArgumentException, "A short can not represent '777777777777'"); + EXPECT_EXCEPTION(is >> si, IllegalArgumentException, "strToInt value '777777777777' is outside of range"); EXPECT_EQUAL(12u, is.size()); int32_t ii(0); - EXPECT_EXCEPTION(is >> ii, IllegalArgumentException, "An int can not represent '777777777777'"); + EXPECT_EXCEPTION(is >> ii, IllegalArgumentException, "strToInt value '777777777777' is outside of range"); EXPECT_EQUAL(12u, is.size()); is << "777777777777"; EXPECT_EQUAL(24u, is.size()); @@ -95,10 +106,10 @@ AsciistreamTest::testIllegalNumbers() { asciistream is("-77"); uint16_t s(0); - EXPECT_EXCEPTION(is >> s, IllegalArgumentException, "An unsigned short can not represent '-77'"); + EXPECT_EXCEPTION(is >> s, IllegalArgumentException, "Illegal strToInt value '-77'"); EXPECT_EQUAL(3u, is.size()); uint32_t i(0); - EXPECT_EXCEPTION(is >> i, IllegalArgumentException, "An unsigned int can not represent '-77'"); + EXPECT_EXCEPTION(is >> i, IllegalArgumentException, "Illegal strToInt value '-77'"); EXPECT_EQUAL(3u, is.size()); } { @@ -131,12 +142,12 @@ AsciistreamTest::testIllegalNumbers() EXPECT_TRUE(is.empty()); { uint32_t l(0); - EXPECT_EXCEPTION(is >> l, IllegalArgumentException, "Failed decoding a unsigned long long from ''."); + EXPECT_EXCEPTION(is >> l, IllegalArgumentException, "buffer underflow at pos 0."); EXPECT_TRUE(is.empty()); } { int32_t l(0); - EXPECT_EXCEPTION(is >> l, IllegalArgumentException, "Failed decoding a long long from ''."); + EXPECT_EXCEPTION(is >> l, IllegalArgumentException, "buffer underflow at pos 0"); EXPECT_TRUE(is.empty()); } { diff --git a/vespalib/src/tests/stllike/hashtable_test.cpp b/vespalib/src/tests/stllike/hashtable_test.cpp index 4948faf450f..877a5dddcb5 100644 --- a/vespalib/src/tests/stllike/hashtable_test.cpp +++ b/vespalib/src/tests/stllike/hashtable_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/hashtable.hpp> #include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/identity.h> #include <vespa/vespalib/testkit/testapp.h> #include <memory> #include <vector> @@ -63,7 +64,7 @@ TEST("require that hashtable can store pairs of <key, unique_ptr to value>") { } template<typename K> using set_hashtable = - hashtable<K, K, vespalib::hash<K>, std::equal_to<K>, std::_Identity<K>>; + hashtable<K, K, vespalib::hash<K>, std::equal_to<K>, Identity>; TEST("require that hashtable<int> can be copied") { set_hashtable<int> table(100); diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp index 7d585cf1cf6..8114923a9fc 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp @@ -10,7 +10,8 @@ #include <limits> #include <stdexcept> #include <cassert> -#include <math.h> +#include <cmath> +#include <charconv> #include <vespa/log/log.h> LOG_SETUP(".vespalib.stllike.asciistream"); @@ -77,9 +78,7 @@ asciistream::asciistream(stringref buf) : } } -asciistream::~asciistream() -{ -} +asciistream::~asciistream() = default; asciistream::asciistream(const asciistream & rhs) : _rPos(0), @@ -145,10 +144,11 @@ namespace { int getValue(double & val, const char *buf) __attribute__((noinline)); int getValue(float & val, const char *buf) __attribute__((noinline)); -int getValue(unsigned long long & val, const char *buf) __attribute__((noinline)); -int getValue(long long & val, const char *buf) __attribute__((noinline)); void throwInputError(int e, const char * t, const char * buf) __attribute__((noinline)); +void throwInputError(std::errc e, const char * t, const char * buf) __attribute__((noinline)); void throwUnderflow(size_t pos) __attribute__((noinline)); +template <typename T> +T strToInt(T & v, const char *begin, const char *end) __attribute__((noinline)); void throwInputError(int e, const char * t, const char * buf) { @@ -163,6 +163,16 @@ void throwInputError(int e, const char * t, const char * buf) } } +void throwInputError(std::errc e, const char * t, const char * buf) { + if (e == std::errc::invalid_argument) { + throw IllegalArgumentException("Illegal " + string(t) + " value '" + string(buf) + "'.", VESPA_STRLOC); + } else if (e == std::errc::result_out_of_range) { + throw IllegalArgumentException(string(t) + " value '" + string(buf) + "' is outside of range.", VESPA_STRLOC); + } else { + throw IllegalArgumentException("Unknown error decoding an " + string(t) + " from '" + string(buf) + "'.", VESPA_STRLOC); + } +} + void throwUnderflow(size_t pos) { throw IllegalArgumentException(make_string("buffer underflow at pos %ld.", pos), VESPA_STRLOC); @@ -190,26 +200,28 @@ int getValue(float & val, const char *buf) return ebuf - buf; } -int getValue(unsigned long long & val, const char *buf) +template <typename T> +T strToInt(T & v, const char *begin, const char *end) { - char *ebuf; - errno = 0; - val = strtoull(buf, &ebuf, 0); - if ((errno != 0) || (buf == ebuf)) { - throwInputError(errno, "unsigned long long", buf); - } - return ebuf - buf; -} + const char * curr = begin; + for (;(curr < end) && std::isspace(*curr); curr++); -int getValue(long long & val, const char *buf) -{ - char *ebuf; - errno = 0; - val = strtoll(buf, &ebuf, 0); - if ((errno != 0) || (buf == ebuf)) { - throwInputError(errno, "long long", buf); + std::from_chars_result err; + if (((end - curr) > 2) && (curr[0] == '0') && ((curr[1] | 0x20) == 'x')) { + err = std::from_chars(curr+2, end, v, 16); + } else { + err = std::from_chars(curr, end, v, 10); } - return ebuf - buf; + if (err.ec == std::errc::invalid_argument) { + if (err.ptr >= end) { + throwUnderflow(err.ptr - begin); + } + throwInputError(err.ec, "strToInt", begin); + } else if (err.ec == std::errc::result_out_of_range) { + throwInputError(err.ec, "strToInt", begin); + } + + return err.ptr - begin; } } @@ -260,81 +272,49 @@ asciistream & asciistream::operator >> (unsigned char & v) asciistream & asciistream::operator >> (unsigned short & v) { - unsigned long long l(0); - size_t r = getValue(l, &_rbuf[_rPos]); - if (l > std::numeric_limits<unsigned short>::max()) { - throw IllegalArgumentException(make_string("An unsigned short can not represent '%lld'.", l), VESPA_STRLOC); - } - _rPos += r; - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (unsigned int & v) { - unsigned long long l(0); - size_t r = getValue(l, &_rbuf[_rPos]); - if (l > std::numeric_limits<unsigned int>::max()) { - throw IllegalArgumentException(make_string("An unsigned int can not represent '%lld'.", l), VESPA_STRLOC); - } - _rPos += r; - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (unsigned long & v) { - unsigned long long l(0); - _rPos += getValue(l, &_rbuf[_rPos]); - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (unsigned long long & v) { - unsigned long long l(0); - _rPos += getValue(l, &_rbuf[_rPos]); - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (short & v) { - long long l(0); - size_t r = getValue(l, &_rbuf[_rPos]); - if ((l < std::numeric_limits<short>::min()) || (l > std::numeric_limits<short>::max())) { - throw IllegalArgumentException(make_string("A short can not represent '%lld'.", l), VESPA_STRLOC); - } - _rPos += r; - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (int & v) { - long long l(0); - size_t r = getValue(l, &_rbuf[_rPos]); - if ((l < std::numeric_limits<int>::min()) || (l > std::numeric_limits<int>::max())) { - throw IllegalArgumentException(make_string("An int can not represent '%lld'.", l), VESPA_STRLOC); - } - _rPos += r; - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (long & v) { - long long l(0); - _rPos += getValue(l, &_rbuf[_rPos]); - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } asciistream & asciistream::operator >> (long long & v) { - long long l(0); - _rPos += getValue(l, &_rbuf[_rPos]); - v = l; + _rPos += strToInt(v, &_rbuf[_rPos], &_rbuf[length()]); return *this; } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 0de03cb97ee..5eae4cea55e 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -3,6 +3,7 @@ #include "hashtable.h" #include "hash_fun.h" +#include "select.h" namespace vespalib { @@ -13,7 +14,7 @@ public: typedef std::pair<K, V> value_type; typedef K key_type; typedef V mapped_type; - using HashTable = hashtable< K, value_type, H, EQ, std::_Select1st< value_type >, M >; + using HashTable = hashtable< K, value_type, H, EQ, Select1st<value_type>, M >; private: HashTable _ht; public: diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp index 2ca6b97748f..311a256be76 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -3,6 +3,7 @@ #include "hash_map_insert.hpp" #include "hashtable.hpp" +#include "select.h" namespace vespalib { @@ -68,11 +69,11 @@ hash_map<K, V, H, EQ, M>::getMemoryUsed() const #define VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, M) \ template class vespalib::hash_map<K, V, H, E, M>; \ - template class vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>; \ - template vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert_result \ - vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \ - template vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert_result \ - vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); \ + template class vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>; \ + template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \ + vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \ + template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \ + vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); \ template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; #define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \ diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h index 7a2db4735aa..919e5e8c47f 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h @@ -3,6 +3,7 @@ #include "hashtable.h" #include "hash_fun.h" +#include "identity.h" #include <initializer_list> namespace vespalib { @@ -11,7 +12,7 @@ template< typename K, typename H = vespalib::hash<K>, typename EQ = std::equal_t class hash_set { private: - using HashTable = hashtable< K, K, H, EQ, std::_Identity<K>, M>; + using HashTable = hashtable< K, K, H, EQ, Identity, M>; HashTable _ht; public: typedef typename HashTable::iterator iterator; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp index 814c96a3c85..19114798806 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp @@ -3,6 +3,7 @@ #include "hash_set_insert.hpp" #include "hashtable.hpp" +#include "identity.h" namespace vespalib { @@ -84,11 +85,11 @@ hash_set<K, H, EQ, M>::insert(K &&value) { #define VESPALIB_HASH_SET_INSTANTIATE(K) \ template class vespalib::hash_set<K>; \ - template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, std::_Identity<K>>; \ + template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, vespalib::Identity>; \ template class vespalib::Array<vespalib::hash_node<K>>; #define VESPALIB_HASH_SET_INSTANTIATE_H(K, H) \ template class vespalib::hash_set<K, H>; \ - template class vespalib::hashtable<K, K, H, std::equal_to<>, std::_Identity<K>>; \ + template class vespalib::hashtable<K, K, H, std::equal_to<>, vespalib::Identity>; \ template class vespalib::Array<vespalib::hash_node<K>>; diff --git a/vespalib/src/vespa/vespalib/stllike/identity.h b/vespalib/src/vespa/vespalib/stllike/identity.h new file mode 100644 index 00000000000..06019fb4690 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/identity.h @@ -0,0 +1,18 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <utility> + +namespace vespalib { + +// Functor which returns its argument unchanged. +// Functionally identical to C++20's std::identity +// TODO remove and replace with std::identity once it is available. +struct Identity { + template <typename T> + constexpr T&& operator()(T&& v) const noexcept { + return std::forward<T>(v); + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/stllike/select.h b/vespalib/src/vespa/vespalib/stllike/select.h new file mode 100644 index 00000000000..28bcd6a01fc --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/select.h @@ -0,0 +1,17 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +// Convenience functor for extracting the first element of a std::pair (or compatible type) +template <typename Pair> +struct Select1st { + constexpr typename Pair::first_type& operator()(Pair& p) const noexcept { + return p.first; + } + constexpr const typename Pair::first_type& operator()(const Pair& p) const noexcept { + return p.first; + } +}; + +} |