diff options
32 files changed, 261 insertions, 200 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 1c73000cc24..e98c09f8128 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -97,6 +98,11 @@ public class HostResource implements Comparable<HostResource> { .collect(Collectors.toSet())); } + /** Returns the flavor of this resource. Empty for self-hosted Vespa. */ + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public Optional<Flavor> getFlavor() { return spec.flavor(); } + /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ public NodeResources realResources() { return spec.realResources(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index 45465fa91c4..779b6854e21 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -110,6 +110,11 @@ public class ClusterMembership { @Override public String toString() { return stringValue(); } + // TODO: Remove when 7.208 is the latest model in use + public static ClusterMembership from(String stringValue, Version vespaVersion) { + return new ClusterMembership(stringValue, vespaVersion, Optional.empty()); + } + public static ClusterMembership from(String stringValue, Version vespaVersion, Optional<DockerImage> dockerImageRepo) { return new ClusterMembership(stringValue, vespaVersion, dockerImageRepo); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 63a163135db..5b76163e97f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -20,7 +20,7 @@ public final class ClusterSpec { /** The group id of these hosts, or empty if this is represents a request for hosts */ private final Optional<Group> groupId; private final Version vespaVersion; - private final boolean exclusive; + private boolean exclusive; private final Optional<Id> combinedId; private final Optional<DockerImage> dockerImageRepo; @@ -109,7 +109,7 @@ public final class ClusterSpec { public ClusterSpec build() { if (specification) { - if (groupId.isEmpty()) throw new IllegalArgumentException("groupId is required to be set when creating a ClusterSpec with specification()"); + if (groupId.isEmpty()) throw new IllegalArgumentException("groupIs is required to be set when creating a ClusterSpec with specification()"); if (vespaVersion == null) throw new IllegalArgumentException("vespaVersion is required to be set when creating a ClusterSpec with specification()"); } else if (groupId.isPresent()) throw new IllegalArgumentException("groupId is not allowed to be set when creating a ClusterSpec with request()"); @@ -141,6 +141,13 @@ public final class ClusterSpec { return this; } + @Deprecated + // TODO: Remove after 7.208 is oldest version in use + public Builder dockerImageRepo(Optional<String> dockerImageRepo) { + this.dockerImageRepo = dockerImageRepo.map(DockerImage::fromString); + return this; + } + public Builder dockerImageRepository(Optional<DockerImage> dockerImageRepo) { this.dockerImageRepo = dockerImageRepo; return this; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 6a7158c1c21..50156d1baa1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -109,6 +109,27 @@ public class Flavor { public Optional<FlavorOverrides> flavorOverrides() { return flavorOverrides; } + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public double getMinMainMemoryAvailableGb() { return resources.memoryGb(); } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public double getMinDiskAvailableGb() { return resources.diskGb(); } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public boolean hasFastDisk() { return resources.diskSpeed() == NodeResources.DiskSpeed.fast; } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public double getBandwidthGbps() { return resources.bandwidthGbps(); } + + /** Returns the number of cores available in this flavor, not scaled for speed. */ + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public double getMinCpuCores() { return minCpuCores; } + public Type getType() { return type; } /** Convenience, returns getType() == Type.DOCKER_CONTAINER */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 81c01cd2e7f..6bc3809bd54 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -3,6 +3,7 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -82,6 +83,109 @@ public class HostSpec implements Comparable<HostSpec> { this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, Optional<ClusterMembership> membership) { + this(hostname, new ArrayList<>(), + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + membership, + Optional.empty(), Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) { + this(hostname, new ArrayList<>(), + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.of(membership), version, Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases) { + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, Flavor flavor) { + this(hostname, aliases, + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, ClusterMembership membership) { + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.of(membership), + Optional.empty(), Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, Optional.empty(), Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<Version> version) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, + Optional.empty(), Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<Version> version, + Optional<NetworkPorts> networkPorts) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, networkPorts, + Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, + Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<Version> version, + Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, Optional.empty()); + } + + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<Version> version, + Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources, + Optional<DockerImage> dockerImageRepo) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, dockerImageRepo); + } + /** Returns the name identifying this host */ public String hostname() { return hostname; } @@ -95,7 +199,7 @@ public class HostSpec implements Comparable<HostSpec> { public NodeResources advertisedResources() { return advertisedResources; } /** A flavor contained the advertised resources of this host */ - // TODO: Remove when models older than 7.285 are gone + // TODO: Remove when models older than 7.226 are gone public Optional<Flavor> flavor() { return advertisedResources.asOptional().map(resources -> new Flavor(resources)); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index 1d4265172a3..fc267bbac29 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -45,6 +45,12 @@ public enum NodeType { this.description = description; } + // TODO: Remove when no version older than 7.247 is live + @Deprecated + public boolean isDockerHost() { + return !childNodeTypes.isEmpty(); + } + public boolean isHost() { return !childNodeTypes.isEmpty(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java index 708b02d6f30..f616174de1a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java @@ -67,6 +67,10 @@ public class Zone { } /** Returns the current cloud */ + // TODO: For compatibility with older config models. Remove after June 2020 + public CloudName cloud() { return cloud.name(); } + + /** Returns the current cloud */ public Cloud getCloud() { return cloud; } /** Returns the current system */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index a071daa7427..e4913643e8d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -4,6 +4,7 @@ package com.yahoo.config.provision.serialization; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.ArrayTraverser; @@ -90,6 +91,7 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line when 7.272 has been released toSlime(host.realResources(), object.setObject(realResourcesKey)); toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); @@ -104,6 +106,13 @@ public class AllocatedHostsSerializer { aliases.addString(alias); } + private static void toSlime(Flavor flavor, Cursor object) { + if (flavor.isConfigured()) + object.setString(flavorKey, flavor.name()); + else + toSlime(flavor.resources(), object.setObject(resourcesKey)); + } + private static void toSlime(NodeResources resources, Cursor resourcesObject) { resourcesObject.setDouble(vcpuKey, resources.vcpu()); resourcesObject.setDouble(memoryKey, resources.memoryGb()); @@ -185,6 +194,7 @@ public class AllocatedHostsSerializer { } private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { + if ( ! storageType.valid()) return NodeResources.StorageType.getDefault(); // TODO: Remove this line after December 2019 switch (storageType.asString()) { case "remote" : return NodeResources.StorageType.remote; case "local" : return NodeResources.StorageType.local; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index fb07bf626f3..eb13baf3e6b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -285,11 +285,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, Instant now) { + public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, + boolean ignoreSessionStaleFailure, Instant now) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); PrepareResult prepareResult; try { - prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, now); + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, ignoreSessionStaleFailure, now); } finally { cleanupTempDirectory(tempDir); } @@ -297,15 +298,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { - return deploy(applicationPackage, prepareParams, Instant.now()); + return deploy(applicationPackage, prepareParams, false, Instant.now()); } - public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, Instant now) { + public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, + boolean ignoreSessionStaleFailure, Instant now) { ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); PrepareResult result = prepare(tenant, sessionId, prepareParams, now); - activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); + activate(tenant, sessionId, prepareParams.getTimeoutBudget(), ignoreSessionStaleFailure); return result; } @@ -384,22 +386,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ApplicationId activate(Tenant tenant, long sessionId, TimeoutBudget timeoutBudget, - boolean force) { + boolean ignoreSessionStaleFailure) { LocalSession localSession = getLocalSession(tenant, sessionId); - Deployment deployment = deployment(localSession, tenant, timeoutBudget.timeLeft(), force); + Deployment deployment = deployFromPreparedSession(localSession, tenant, timeoutBudget.timeLeft()); + deployment.setIgnoreSessionStaleFailure(ignoreSessionStaleFailure); deployment.activate(); return localSession.getApplicationId(); } - private Deployment deployment(LocalSession session, Tenant tenant, Duration timeout, boolean force) { - return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false, force); + private Deployment deployFromPreparedSession(LocalSession session, Tenant tenant, Duration timeout) { + return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false); } - public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean force) { + public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean ignoreStaleSessionFailure) { Tenant tenant = tenantRepository.getTenant(prepared.getTenantName()); Transaction transaction = tenant.getSessionRepository().createActivateTransaction(prepared); if (active != null) { - checkIfActiveHasChanged(prepared, active, force); + checkIfActiveHasChanged(prepared, active, ignoreStaleSessionFailure); checkIfActiveIsNewerThanSessionToBeActivated(prepared.getSessionId(), active.getSessionId()); transaction.add(active.createDeactivateTransaction().operations()); } @@ -739,10 +742,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye - public CompletionWaiter activate(LocalSession session, Session previousActiveSession, ApplicationId applicationId, boolean force) { + public CompletionWaiter activate(LocalSession session, Session previousActiveSession, ApplicationId applicationId, boolean ignoreSessionStaleFailure) { CompletionWaiter waiter = session.getSessionZooKeeperClient().createActiveWaiter(); NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, force)); + transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure)); hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); transaction.commit(); return waiter; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 6ce17f859b5..11ce659625d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -67,12 +67,11 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Whether this model should be validated (only takes effect if prepared=false) */ private final boolean validate; - /** Whether activation of this model should be forced */ - private final boolean force; + private boolean ignoreSessionStaleFailure = false; private Deployment(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, - Clock clock, boolean prepared, boolean validate, boolean isBootstrap, boolean force) { + Clock clock, boolean prepared, boolean validate, boolean isBootstrap) { this.session = session; this.applicationRepository = applicationRepository; this.hostProvisioner = hostProvisioner; @@ -85,21 +84,24 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.version = session.getVespaVersion(); this.isBootstrap = isBootstrap; this.athenzDomain = session.getAthenzDomain(); - this.force = force; } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, - validate, isBootstrap, false); + validate, isBootstrap); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean isBootstrap, boolean force) { + Duration timeout, Clock clock, boolean isBootstrap) { return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, isBootstrap, force); + timeout, clock, true, true, isBootstrap); + } + + public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { + this.ignoreSessionStaleFailure = ignoreSessionStaleFailure; } /** Prepares this. This does nothing if this is already prepared */ @@ -140,7 +142,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { CompletionWaiter waiter; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { previousActiveSession = applicationRepository.getActiveSession(applicationId); - waiter = applicationRepository.activate(session, previousActiveSession, applicationId, force); + waiter = applicationRepository.activate(session, previousActiveSession, applicationId, ignoreSessionStaleFailure); } catch (RuntimeException e) { throw e; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index d6badb8a9a2..dedd96da6f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -53,10 +53,13 @@ public class ApplicationApiHandler extends SessionHandler { @Override protected HttpResponse handlePOST(HttpRequest request) { validateDataAndHeader(request); - TenantName tenantName = validateTenant(request); - PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); + Tenant tenant = validateTenant(request); + PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenant.getName(), zookeeperBarrierTimeout); CompressedApplicationInputStream compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); - PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.deploy(compressedStream, + prepareParams, + shouldIgnoreSessionStaleFailure(request), + Instant.now()); return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); } @@ -65,10 +68,10 @@ public class ApplicationApiHandler extends SessionHandler { return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); } - private TenantName validateTenant(HttpRequest request) { + private Tenant validateTenant(HttpRequest request) { TenantName tenantName = getTenantNameFromRequest(request); checkThatTenantExists(tenantRepository, tenantName); - return tenantName; + return tenantRepository.getTenant(tenantName); } public static TenantName getTenantNameFromRequest(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index d962218b63a..1fea966503b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -44,7 +44,6 @@ public final class PrepareParams { static final String APPLICATION_HOST_ROLE = "applicationHostRole"; static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; static final String QUOTA_PARAM_NAME = "quota"; - static final String FORCE_PARAM_NAME = "force"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -52,7 +51,6 @@ public final class PrepareParams { private final boolean dryRun; private final boolean verbose; private final boolean isBootstrap; - private final boolean force; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; @@ -67,7 +65,7 @@ public final class PrepareParams { List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, - Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { + Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -82,7 +80,6 @@ public final class PrepareParams { this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; this.quota = quota; - this.force = force; } public static class Builder { @@ -91,7 +88,6 @@ public final class PrepareParams { private boolean dryRun = false; private boolean verbose = false; private boolean isBootstrap = false; - private boolean force = false; private ApplicationId applicationId = null; private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); @@ -203,16 +199,11 @@ public final class PrepareParams { return this; } - public Builder force(boolean force) { - this.force = force; - return this; - } - public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, force); + applicationRoles, quota); } } @@ -230,7 +221,6 @@ public final class PrepareParams { .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .quota(request.getProperty(QUOTA_PARAM_NAME)) - .force(request.getBooleanProperty(FORCE_PARAM_NAME)) .build(); } @@ -280,8 +270,6 @@ public final class PrepareParams { public boolean isBootstrap() { return isBootstrap; } - public boolean force() { return force; } - public TimeoutBudget getTimeoutBudget() { return timeoutBudget; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 4770b0797eb..f879f6c2a2a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -704,7 +704,7 @@ public class ApplicationRepositoryTest { } private PrepareResult prepareAndActivate(File application) { - return applicationRepository.deploy(application, prepareParams(), Instant.now()); + return applicationRepository.deploy(application, prepareParams(), false, Instant.now()); } private PrepareResult deployApp(File applicationPackage) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index d92245bf5c1..e69fe21dd92 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -215,7 +215,7 @@ public class DeployTester { paramsBuilder.applicationId(applicationId) .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); - return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), now); + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, now); } public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 78d69b75d59..7999f9280c0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -67,7 +67,10 @@ class MaintainerTester { } void deployApp(File applicationPath, PrepareParams.Builder prepareParams) { - applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build(), clock.instant()); + applicationRepository.deploy(applicationPath, + prepareParams.ignoreValidationErrors(true).build(), + false, + clock.instant()); } Curator curator() { return curator; } diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 5e5e2394e49..feeac3d9da0 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -88,12 +88,8 @@ public class DocumentSelectorTestCase { manager.registerDocumentType(new DocumentType("andornot")); manager.registerDocumentType(new DocumentType("idid")); manager.registerDocumentType(new DocumentType("usergroup")); - var userType = new DocumentType("user"); - userType.addField("id", DataType.INT); - manager.registerDocumentType(userType); - var groupType = new DocumentType("group"); - groupType.addField("iD", DataType.INT); // For checking case preservation - manager.registerDocumentType(groupType); + manager.registerDocumentType(new DocumentType("user")); + manager.registerDocumentType(new DocumentType("group")); } @Test @@ -161,8 +157,6 @@ public class DocumentSelectorTestCase { assertParse(null, "true or or_t or ortype"); assertParse(null, "user or group"); assertParse(null, "user.foo or group.bar"); - assertParse("user.id == id.user", "user.id == id.user"); - assertParse("group.iD == id.user", "group.iD == id.user"); // Casing is preserved } @Test diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 30b2bfbb1b4..6fd9ab80faa 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -94,11 +94,9 @@ void DocumentSelectParserTest::SetUp() Struct("usergroup.header"), Struct("usergroup.body")); builder.document(875463456, "user", - Struct("user.header").addField("id", DataType::T_INT), - Struct("user.body")); + Struct("user.header"), Struct("user.body")); builder.document(567463442, "group", - Struct("group.header").addField("iD", DataType::T_INT), - Struct("group.body")); + Struct("group.header"), Struct("group.body")); _repo = std::make_unique<DocumentTypeRepo>(builder.config()); _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); @@ -1457,9 +1455,6 @@ TEST_F(DocumentSelectParserTest, special_tokens_are_allowed_as_freestanding_iden EXPECT_EQ("(== (ID id.user) (FIELD user user))", parse_to_tree("id.user == user.user")); EXPECT_EQ("(NOT (DOCTYPE group))", parse_to_tree("not group")); EXPECT_EQ("(== (ID id.group) (FIELD group group))", parse_to_tree("id.group == group.group")); - EXPECT_EQ("(== (FIELD user id) (ID id.user))", parse_to_tree("user.id == id.user")); - // Case is preserved for special ID field - EXPECT_EQ("(== (FIELD group iD) (ID id.user))", parse_to_tree("group.iD == id.user")); } TEST_F(DocumentSelectParserTest, test_can_build_field_value_from_field_expr_node) diff --git a/document/src/vespa/document/select/grammar/lexer.ll b/document/src/vespa/document/select/grammar/lexer.ll index d52cf0db7a9..1222aac02a2 100644 --- a/document/src/vespa/document/select/grammar/lexer.ll +++ b/document/src/vespa/document/select/grammar/lexer.ll @@ -119,6 +119,7 @@ SQ_STRING \'(\\([\\tnfr']|x{HEXDIGIT}{2})|[^'\\])*\' \[{WS}*(${IDCHARS}|{DECIMAL}){WS}*\] STRING_TOKEN(FP_ARRAY_LOOKUP) /* Primary tokens are case insensitive */ +(?i:"id") NAMED_TOKEN(ID) (?i:"null") NAMED_TOKEN(NULL) (?i:"true") NAMED_TOKEN(TRUE) (?i:"false") NAMED_TOKEN(FALSE) @@ -127,7 +128,6 @@ SQ_STRING \'(\\([\\tnfr']|x{HEXDIGIT}{2})|[^'\\])*\' (?i:"not") NAMED_TOKEN(NOT) /* We expose the verbatim input as the token value, as these may also be used for identifiers... */ -(?i:"id") STRING_TOKEN(ID) (?i:"user") STRING_TOKEN(USER) (?i:"group") STRING_TOKEN(GROUP) (?i:"scheme") STRING_TOKEN(SCHEME) diff --git a/document/src/vespa/document/select/grammar/parser.yy b/document/src/vespa/document/select/grammar/parser.yy index 9e4a1d1a222..9d5b5825330 100644 --- a/document/src/vespa/document/select/grammar/parser.yy +++ b/document/src/vespa/document/select/grammar/parser.yy @@ -50,6 +50,7 @@ %token LE "<=" %token GT ">" %token LT "<" +%token ID %token NOW_FUNC /* @@ -72,7 +73,7 @@ %token <string_val> FP_MAP_LOOKUP FP_ARRAY_LOOKUP %token <double_val> FLOAT %token <i64_val> INTEGER -%token <string_val> ID USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE +%token <string_val> USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE %type <string_val> ident mangled_ident %type <abstract_node> bool_ @@ -83,7 +84,7 @@ %type <field_expr_node> field_spec %destructor { delete $$; } IDENTIFIER STRING FP_MAP_LOOKUP FP_ARRAY_LOOKUP -%destructor { delete $$; } ID USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE +%destructor { delete $$; } USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE %destructor { delete $$; } null_ bool_ number string doc_type ident id_arg id_spec %destructor { delete $$; } variable mangled_ident field_spec value arith_expr %destructor { delete $$; } comparison leaf logical_expr expression @@ -243,18 +244,9 @@ id_arg ; id_spec - : ID %prec NON_DOT { - (void)steal<string>($1); // Explicitly discard. - $$ = new IdValueNode(bucket_id_factory, "id", ""); // Prefer shifting instead of reducing. - } - | ID "." id_arg { - (void)steal<string>($1); // Explicitly discard. - $$ = new IdValueNode(bucket_id_factory, "id", *steal<string>($3)); - } - | ID "." IDENTIFIER "(" ")" { - (void)steal<string>($1); // Explicitly discard. - $$ = new FunctionValueNode(*steal<string>($3), std::make_unique<IdValueNode>(bucket_id_factory, "id", "")); - } + : ID %prec NON_DOT { $$ = new IdValueNode(bucket_id_factory, "id", ""); } /* Prefer shifting instead of reducing */ + | ID "." id_arg { $$ = new IdValueNode(bucket_id_factory, "id", *steal<string>($3)); } + | ID "." IDENTIFIER "(" ")" { $$ = new FunctionValueNode(*steal<string>($3), std::make_unique<IdValueNode>(bucket_id_factory, "id", "")); } ; variable @@ -262,17 +254,12 @@ variable ; /* FIXME this is a horrible leftover of post-parsed fieldpath processing */ - /* At least we verify structural integrity at initial parse-time now... */ - /* Post-parsing should be replaced with an actual parse-time built AST! */ - /* This rule is only used after matching an initial valid identifier, so */ - /* we add some special casing of lexer keywords that today are allowed as */ - /* regular field names (but not as document type names). Not pretty, but */ - /* it avoids parser ambiguities. */ + /* At least we verify structural integrity at initial parse-time now... */ + /* Post-parsing should be replaced with an actual parse-time built AST! */ mangled_ident : ident { $$ = $1; } | mangled_ident FP_MAP_LOOKUP { $1->append(*steal<string>($2)); $$ = $1; } | mangled_ident FP_ARRAY_LOOKUP { $1->append(*steal<string>($2)); $$ = $1; } - | ID { $$ = $1; } ; field_spec diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0da68f576b6..3e326871f7c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -264,6 +264,12 @@ public class Flags { "Whether to provision and use endpoint certs for apps in shared routing zones", "Takes effect on next deployment of the application", APPLICATION_ID); + public static final UnboundBooleanFlag PHRASE_SEGMENTING = defineFeatureFlag( + "phrase-segmenting", false, + "Should 'implicit phrases' in queries we parsed to a phrase or and?", + "Takes effect on redeploy", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag NLB_PROXY_PROTOCOL = defineFeatureFlag( "nlb-proxy-protocol", false, "Configure NLB to use proxy protocol", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index b08dc6bbaf2..e146583ae04 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -204,11 +204,6 @@ public final class Node { return with(requireAllocation("Cannot unretire").unretire()); } - /** Returns a copy of this with removable set to the given value */ - public Node removable(boolean removable) { - return with(requireAllocation("Cannot set removable").removable(removable)); - } - /** Returns a copy of this with the restart generation set to generation */ public Node withRestart(Generation generation) { Allocation allocation = requireAllocation("Cannot set restart generation"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 4ec7ddd04c4..983ba5165e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -511,7 +511,7 @@ public class NodeRepository extends AbstractComponent { public void setRemovable(ApplicationId application, List<Node> nodes) { try (Mutex lock = lock(application)) { List<Node> removableNodes = - nodes.stream().map(node -> node.with(node.allocation().get().removable(true))) + nodes.stream().map(node -> node.with(node.allocation().get().removable())) .collect(Collectors.toList()); write(removableNodes, lock); } @@ -641,7 +641,7 @@ public class NodeRepository extends AbstractComponent { } private Node move(Node node, State toState, Agent agent, Optional<String> reason) { - if (toState == Node.State.active && node.allocation().isEmpty()) + if (toState == Node.State.active && ! node.allocation().isPresent()) illegal("Could not set " + node + " active. It has no allocation."); try (Mutex lock = lock(node)) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java index b476a2bdefc..30ef84c6927 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java @@ -85,9 +85,9 @@ public class Allocation { return new Allocation(owner, clusterMembership, requestedResources, generation, removable, networkPorts); } - /** Returns a copy of this allocation where removable is set to the given value */ - public Allocation removable(boolean removable) { - return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, removable, networkPorts); + /** Returns a copy of this allocation where removable is set to true */ + public Allocation removable() { + return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, true, networkPorts); } public Allocation with(ClusterMembership newMembership) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 5a3584b6ff4..37842115949 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -241,7 +241,7 @@ public class NodeSerializer { } private Optional<Allocation> allocationFromSlime(NodeResources assignedResources, Inspector object) { - if ( ! object.valid()) return Optional.empty(); + if ( ! object.valid()) return Optional.empty(); // TODO: Remove this line (and to the simplifications that follows) after November 2019 return Optional.of(new Allocation(applicationIdFromSlime(object), clusterMembershipFromSlime(object), NodeResourcesSerializer.optionalResourcesFromSlime(object.field(requestedResourcesKey)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a37da10f5f0..1aa0f69dd9b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -104,23 +104,27 @@ class NodeAllocation { Node offered = node.node; if (offered.allocation().isPresent()) { - Allocation allocation = offered.allocation().get(); - ClusterMembership membership = allocation.membership(); - if ( ! allocation.owner().equals(application)) continue; // wrong application + ClusterMembership membership = offered.allocation().get().membership(); + if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type if ((! node.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it - if ( offered.state() == Node.State.active && allocation.isRemovable()) continue; // don't accept; causes removal + if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) - boolean resizeable = false; - boolean acceptToRetire = false; if (requestedNodes.considerRetiring()) { - resizeable = node.isResizable; - acceptToRetire = acceptToRetire(node); + boolean wantToRetireNode = false; + if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) wantToRetireNode = true; + if (violatesParentHostPolicy(this.nodes, offered)) wantToRetireNode = true; + if ( ! hasCompatibleFlavor(node)) wantToRetireNode = true; + if (offered.status().wantToRetire()) wantToRetireNode = true; + if (requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), application.application(), offered.parentHostname())) + wantToRetireNode = true; + if ((! saturated() && hasCompatibleFlavor(node)) || acceptToRetire(node)) + accepted.add(acceptNode(node, wantToRetireNode, node.isResizable)); + } + else { + accepted.add(acceptNode(node, false, false)); } - - if ((! saturated() && hasCompatibleFlavor(node) && requestedNodes.acceptable(offered)) || acceptToRetire) - accepted.add(acceptNode(node, shouldRetire(node), resizeable)); } else if (! saturated() && hasCompatibleFlavor(node)) { if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) { @@ -135,7 +139,7 @@ class NodeAllocation { ++rejectedDueToExclusivity; continue; } - if ( requestedNodes.isExclusive() && ! hostsOnly(application, offered.parentHostname())) { + if ( requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), application.application(), offered.parentHostname())) { ++rejectedDueToExclusivity; continue; } @@ -153,15 +157,6 @@ class NodeAllocation { return accepted; } - private boolean shouldRetire(PrioritizableNode node) { - if ( ! requestedNodes.considerRetiring()) return false; - if ( ! nodeResourceLimits.isWithinRealLimits(node.node, cluster)) return true; - if (violatesParentHostPolicy(this.nodes, node.node)) return true; - if ( ! hasCompatibleFlavor(node)) return true; - if (node.node.status().wantToRetire()) return true; - if (requestedNodes.isExclusive() && ! hostsOnly(application, node.node.parentHostname())) return true; - return false; - } private boolean violatesParentHostPolicy(Collection<PrioritizableNode> accepted, Node offered) { return checkForClashingParentHost() && offeredNodeHasParentHostnameAlreadyAccepted(accepted, offered); @@ -198,13 +193,13 @@ class NodeAllocation { return true; } - /** Returns true if this host only hosts the given application (in any instance) */ - private boolean hostsOnly(ApplicationId application, Optional<String> parentHostname) { + /** Returns true if this host only hosts the given applicaton (in any instance) */ + private boolean hostsOnly(TenantName tenant, ApplicationName application, Optional<String> parentHostname) { if (parentHostname.isEmpty()) return true; // yes, as host is exclusive for (Node nodeOnHost : allNodes.childrenOf(parentHostname.get())) { if (nodeOnHost.allocation().isEmpty()) continue; - if ( ! allocatedTo(application.tenant(), application.application(), nodeOnHost)) return false; + if ( ! allocatedTo(tenant, application, nodeOnHost)) return false; } return true; } @@ -261,8 +256,8 @@ class NodeAllocation { if (resizeable && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) node = resize(node); - if (node.state() != Node.State.active) // reactivated node - wipe state that deactivated it - node = node.unretire().removable(false); + if (node.state() != Node.State.active) // reactivated node - make sure its not retired + node = node.unretire(); } else { ++wasRetiredJustNow; node = node.retire(nodeRepository.clock().instant()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index f50c988edfd..9971aae1714 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -60,9 +60,6 @@ public interface NodeSpec { /** Returns whether the given node must be resized to match this spec */ boolean needsResize(Node node); - /** Returns true if there exist some circumstance where we may accept to have this node allocated */ - boolean acceptable(Node node); - /** * Returns true if a node with given current resources and current spare host resources can be resized * in-place to resources in this spec. @@ -160,9 +157,6 @@ public interface NodeSpec { } @Override - public boolean acceptable(Node node) { return true; } - - @Override public String toString() { return "request for " + count + " nodes with " + requestedNodeResources; } } @@ -217,12 +211,6 @@ public interface NodeSpec { public boolean needsResize(Node node) { return false; } @Override - public boolean acceptable(Node node) { - // Since we consume all offered nodes we should not accept previously deactivated nodes - return node.state() != Node.State.inactive; - } - - @Override public String toString() { return "request for all nodes of type '" + type + "'"; } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index cb39e8fecce..1137ae5ce2c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -105,7 +105,7 @@ class AutoscalingTester { try (Mutex lock = nodeRepository().lock(application)){ for (Node node : nodeRepository().getNodes(application, Node.State.active)) { if (node.allocation().get().membership().retired()) - nodeRepository().write(node.with(node.allocation().get().removable(true)), lock); + nodeRepository().write(node.with(node.allocation().get().removable()), lock); } } deploy(application, cluster, resources); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index dbbad0b8982..5e4bfc2a7bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -179,7 +179,7 @@ public class NodeSerializerTest { (copy.history().event(History.Event.Type.retired).get()).agent()); assertTrue(copy.allocation().get().membership().retired()); - Node removable = copy.with(node.allocation().get().removable(true)); + Node removable = copy.with(node.allocation().get().removable()); Node removableCopy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(removable)); assertTrue(removableCopy.allocation().get().isRemovable()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index e566172b524..cddc1fcb253 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -20,7 +20,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; import org.junit.Test; import java.util.HashSet; @@ -381,63 +380,6 @@ public class DockerProvisioningTest { } } - @Test - public void test_startup_redeployment_with_inactive_nodes() { - NodeResources r = new NodeResources(20, 40, 100, 4); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) - .flavors(List.of(new Flavor(r))) - .build(); - tester.makeReadyHosts(5, r).deployZoneApp(); - - ApplicationId app1 = tester.makeApplicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); - - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(5, 1, r))); - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); - - assertEquals(2, tester.getNodes(app1, Node.State.active).size()); - assertEquals(3, tester.getNodes(app1, Node.State.inactive).size()); - - // Startup deployment: Not failable - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r), false, false)); - // ... causes no change - assertEquals(2, tester.getNodes(app1, Node.State.active).size()); - assertEquals(3, tester.getNodes(app1, Node.State.inactive).size()); - } - - @Test - public void inactive_container_nodes_are_reused() { - assertInactiveReuse(ClusterSpec.Type.container); - } - - @Test - public void inactive_content_nodes_are_reused() { - assertInactiveReuse(ClusterSpec.Type.content); - } - - private void assertInactiveReuse(ClusterSpec.Type clusterType) { - NodeResources r = new NodeResources(20, 40, 100, 4); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) - .flavors(List.of(new Flavor(r))) - .build(); - tester.makeReadyHosts(4, r).deployZoneApp(); - - ApplicationId app1 = tester.makeApplicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(clusterType, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); - - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(4, 1, r))); - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); - - // Deactivate any retired nodes - usually done by the RetiredExpirer - tester.nodeRepository().setRemovable(app1, tester.getNodes(app1).retired().asList()); - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); - - assertEquals(2, tester.getNodes(app1, Node.State.inactive).size()); - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(4, 1, r))); - assertEquals(0, tester.getNodes(app1, Node.State.inactive).size()); - } - - private Set<String> hostsOf(NodeList nodes) { return nodes.asList().stream().map(Node::parentHostname).map(Optional::get).collect(Collectors.toSet()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index ff2f0ffca96..5c730912c49 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -888,7 +888,7 @@ public class ProvisioningTest { // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); var initialNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.from(new ClusterResources(2, 1, defaultResources)))); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); // Application is redeployed with cluster type combined cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")) @@ -896,7 +896,7 @@ public class ProvisioningTest { .combinedId(Optional.of(ClusterSpec.Id.from("qrs"))) .build(); var newNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.from(new ClusterResources(2, 1, defaultResources)))); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", @@ -906,7 +906,7 @@ public class ProvisioningTest { // Application is redeployed with cluster type content again cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); newNodes = tester.activate(application, tester.prepare(application, cluster, - Capacity.from(new ClusterResources(2, 1, defaultResources)))); + Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", Set.of(ClusterSpec.Type.content), diff --git a/searchcore/src/tests/proton/docsummary/summarymap.cfg b/searchcore/src/tests/proton/docsummary/summarymap.cfg index cb03ce76fd1..f2d429b1412 100644 --- a/searchcore/src/tests/proton/docsummary/summarymap.cfg +++ b/searchcore/src/tests/proton/docsummary/summarymap.cfg @@ -1,4 +1,4 @@ -override[17] +override[16] override[0].field "aa" override[0].command "copy" override[0].arguments "ab" @@ -46,6 +46,3 @@ override[14].arguments "wp2" override[15].field "bj" override[15].command "attribute" override[15].arguments "bj" -override[16].field "missing" -override[16].command "attribute" -override[16].arguments "missing" diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index b9d0c27bd16..1a269e556d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -90,7 +90,7 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } else if (overrideName == "attribute") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { fieldWriter = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), argument); - rc = true; // Allow missing attribute vector + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { |