diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2019-02-11 13:23:31 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-02-11 13:23:31 +0100 |
commit | ee45d854e0a540d8ed954cb4dd7749644f93de12 (patch) | |
tree | ec55c0d0bc43bb36fb6474642c5f18f8bbf14a7c | |
parent | a73d44d194df6296976d8f53116c835aa992015e (diff) | |
parent | 1f3d5bb43009736f3a1478cebecdc6e933ee7ea2 (diff) |
Merge branch 'master' into jvenstad/deploy-without-application-lock
97 files changed, 474 insertions, 299 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index c932958b58b..0b36ea3bee2 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -303,6 +303,8 @@ "public static java.lang.String toMessageString(java.lang.Throwable)", "public java.util.Optional athenzDomain()", "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", "public static void main(java.lang.String[])" ], "fields": [ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index a6cecefe940..d15d76fd11b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -267,6 +267,27 @@ public class DeploymentSpec { return Optional.ofNullable(athenzService); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeploymentSpec that = (DeploymentSpec) o; + return globalServiceId.equals(that.globalServiceId) && + upgradePolicy == that.upgradePolicy && + majorVersion.equals(that.majorVersion) && + changeBlockers.equals(that.changeBlockers) && + steps.equals(that.steps) && + xmlForm.equals(that.xmlForm) && + athenzDomain.equals(that.athenzDomain) && + athenzService.equals(that.athenzService) && + notifications.equals(that.notifications); + } + + @Override + public int hashCode() { + return Objects.hash(globalServiceId, upgradePolicy, majorVersion, changeBlockers, steps, xmlForm, athenzDomain, athenzService, notifications); + } + /** This may be invoked by a continuous build */ public static void main(String[] args) { if (args.length != 2 && args.length != 3) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java index f9469b1ae65..3c24cb58cff 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -44,7 +44,7 @@ public class UrlDownloadRpcServer { private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; private final File downloadBaseDir; - private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("Rpc download executor")); UrlDownloadRpcServer(Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index f240129eda1..991cd99968d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -212,6 +212,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { + log.log(LogLevel.INFO, "Redeployment of " + applicationsNotRedeployed + + " failed, will retry in " + sleepTimeWhenRedeployingFails); Thread.sleep(sleepTimeWhenRedeployingFails.toMillis()); } } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 16aaef048b5..3034c7dfd53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -85,8 +85,8 @@ public class FileServer { private FileServer(ConnectionPool connectionPool, File rootDir) { this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.pushExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - this.pullExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); + this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); } boolean hasFile(String fileReference) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 1ce90fad465..e8abecc3236 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -126,7 +126,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { this.metrics = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.hostLivenessTracker = hostLivenessTracker; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); - int numberOfRpcThreads = (config.numRpcThreads() == 0) ? Runtime.getRuntime().availableProcessors() : config.numRpcThreads(); + int numberOfRpcThreads = (config.numRpcThreads() == 0) + ? Math.max(8, Runtime.getRuntime().availableProcessors()) + : config.numRpcThreads(); executorService = new ThreadPoolExecutor(numberOfRpcThreads, numberOfRpcThreads, 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index 612941ece7a..5700e316493 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; /** @@ -69,7 +70,7 @@ public class FS4InvokerFactory { * @return Optional containing the SearchInvoker or <i>empty</i> if some node in the * list is invalid and the remaining coverage is not sufficient */ - public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { + public Optional<SearchInvoker> getSearchInvoker(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 1d39cffa9d2..146b132be22 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; /** @@ -106,7 +107,7 @@ public class Dispatcher extends AbstractComponent { @FunctionalInterface private interface SearchInvokerSupplier { - Optional<SearchInvoker> supply(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage); + Optional<SearchInvoker> supply(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage); } // build invoker based on searchpath @@ -121,7 +122,7 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } else { query.trace(false, 2, "Dispatching internally with search path ", searchPath); - return invokerFactory.supply(query, -1, nodes, true); + return invokerFactory.supply(query, OptionalInt.empty(), nodes, true); } } catch (InvalidSearchPathException e) { return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage()))); @@ -133,7 +134,7 @@ public class Dispatcher extends AbstractComponent { if (directNode.isPresent()) { Node node = directNode.get(); query.trace(false, 2, "Dispatching directly to ", node); - return invokerFactory.supply(query, -1, Arrays.asList(node), true); + return invokerFactory.supply(query, OptionalInt.empty(), Arrays.asList(node), true); } int covered = searchCluster.groupsWithSufficientCoverage(); @@ -148,7 +149,8 @@ public class Dispatcher extends AbstractComponent { } Group group = groupInCluster.get(); boolean acceptIncompleteCoverage = (i == max - 1); - Optional<SearchInvoker> invoker = invokerFactory.supply(query, group.id(), group.nodes(), acceptIncompleteCoverage); + Optional<SearchInvoker> invoker = invokerFactory.supply(query, OptionalInt.of(group.id()), group.nodes(), + acceptIncompleteCoverage); if (invoker.isPresent()) { query.trace(false, 2, "Dispatching internally to search group ", group.id()); query.getModel().setSearchPath("/" + group.id()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index e497ef6751b..a1658684b87 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -18,6 +18,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; @@ -62,11 +63,11 @@ public class SearchCluster implements NodeManager<Node> { public SearchCluster(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; - this.size = dispatchConfig.node().size(); this.fs4ResourcePool = fs4ResourcePool; this.vipStatus = vipStatus; List<Node> nodes = toNodes(dispatchConfig); + this.size = nodes.size(); // Create groups ImmutableMap.Builder<Integer, Group> groupsBuilder = new ImmutableMap.Builder<>(); @@ -331,9 +332,10 @@ public class SearchCluster implements NodeManager<Node> { } } - private void logIfInsufficientCoverage(boolean sufficient, int groupId, int nodes) { + private void logIfInsufficientCoverage(boolean sufficient, OptionalInt groupId, int nodes) { if (!sufficient) { - log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", groupId, nodes, groupSize(), + String group = groupId.isPresent()? Integer.toString(groupId.getAsInt()) : "(unspecified)"; + log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", group, nodes, groupSize(), groupSize() - dispatchConfig.maxNodesDownPerGroup())); } } @@ -341,14 +343,22 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - public boolean isPartialGroupCoverageSufficient(int groupId, List<Node> nodes) { + public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { if (orderedGroups.size() == 1) { boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); - logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + logIfInsufficientCoverage(sufficient, knownGroupId, nodes.size()); return sufficient; } - int nodesInGroup = groups.get(groupId).nodes().size(); + if (knownGroupId.isEmpty()) { + return false; + } + int groupId = knownGroupId.getAsInt(); + Group group = groups.get(groupId); + if (group == null) { + return false; + } + int nodesInGroup = group.nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; for (Group g : orderedGroups) { @@ -363,7 +373,7 @@ public class SearchCluster implements NodeManager<Node> { } long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; boolean sufficient = isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); - logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + logIfInsufficientCoverage(sufficient, knownGroupId, nodes.size()); return sufficient; } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index fae659b0d70..708caafa3f5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.hamcrest.Matchers.is; @@ -112,7 +113,7 @@ public class DispatcherTest { } @Override - public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { + public Optional<SearchInvoker> getSearchInvoker(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { if (step >= events.length) { throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e5fe8a68444..66ba44ec129 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -276,17 +276,17 @@ public class ApplicationController { .map(app -> new LockedApplication(app, lock)) .orElseGet(() -> new LockedApplication(createApplication(applicationId, Optional.empty()), lock)); - boolean canDeployDirectly = options.deployDirectly || zone.environment().isManuallyDeployed(); + boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed(); boolean preferOldestVersion = options.deployCurrentVersion; // Determine versions to use. - if (canDeployDirectly) { - platformVersion = options.vespaVersion.map(Version::new).orElse(application.get().deploymentSpec().majorVersion() - .flatMap(this::lastCompatibleVersion) - .orElse(controller.systemVersion())); + if (manuallyDeployed) { applicationVersion = applicationVersionFromDeployer.orElse(ApplicationVersion.unknown); applicationPackage = applicationPackageFromDeployer.orElseThrow( () -> new IllegalArgumentException("Application package must be given when deploying to " + zone)); + platformVersion = options.vespaVersion.map(Version::new).orElse(applicationPackage.deploymentSpec().majorVersion() + .flatMap(this::lastCompatibleVersion) + .orElse(controller.systemVersion())); } else { JobType jobType = JobType.from(controller.system(), zone) @@ -320,8 +320,10 @@ public class ApplicationController { }); // Update application with information from application package - if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally()) - // TODO jvenstad: Store only on submissions (not on deployments to dev!!) + if ( ! preferOldestVersion + && ! application.get().deploymentJobs().deployedInternally() + && ! zone.environment().isManuallyDeployed()) + // TODO jvenstad: Store only on submissions storeWithUpdatedConfig(application, applicationPackage); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 19ff6df3ccb..1cac93a9971 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,7 +49,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -70,7 +69,6 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; -import com.yahoo.vespa.hosted.controller.maintenance.InfrastructureUpgrader; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; @@ -95,13 +93,11 @@ import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Scanner; -import java.util.function.Function; import java.util.logging.Level; import static java.util.stream.Collectors.joining; @@ -178,6 +174,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.getUri().getQuery()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); @@ -682,6 +680,18 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse deploying(String tenant, String application, HttpRequest request) { + Application app = controller.applications().require(ApplicationId.from(tenant, application, "default")); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + if (!app.change().isEmpty()) { + app.change().platform().ifPresent(version -> root.setString("platform", version.toString())); + app.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); + root.setBool("pinned", app.change().isPinned()); + } + return new SlimeJsonResponse(slime); + } + private HttpResponse suspended(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); @@ -882,12 +892,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // To avoid second guessing the orchestrated upgrades of system applications // we don't allow to deploy these during an system upgrade (i.e when new vespa is being rolled out) - Version version = wantedSystemVersion(zone, SystemApplication.zone); - if (!controller.systemVersion().equals(version)) { - throw new RuntimeException("Deployment of system applications during a system upgrade is not allowed"); + if (controller.versionStatus().isUpgrading()) { + throw new IllegalArgumentException("Deployment of system applications during a system upgrade is not allowed"); + } + Optional<VespaVersion> systemVersion = controller.versionStatus().systemVersion(); + if (systemVersion.isEmpty()) { + throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); } ActivateResult result = controller.applications() - .deploySystemApplicationPackage(SystemApplication.zone, zone, version); + .deploySystemApplicationPackage(SystemApplication.zone, zone, systemVersion.get().versionNumber()); return new SlimeJsonResponse(toSlime(result)); } @@ -956,23 +969,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(toSlime(result)); } - /** Find the minimum value of a version field in a zone */ - private Version wantedSystemVersion(ZoneId zone, SystemApplication application) { - try { - return controller.configServer() - .nodeRepository() - .list(zone, application.id()) - .stream() - .filter(node -> node.state().equals(Node.State.active)) - .map(Node::wantedVersion) - .min(Comparator.naturalOrder()).orElseThrow( - () -> new RuntimeException("System version not found in node repo")); - } catch (Exception e) { - throw new RuntimeException(String.format("Failed to get version for %s in %s: %s", - application.id(), zone, Exceptions.toMessageString(e))); - } - } - private HttpResponse deleteTenant(String tenantName, HttpRequest request) { Optional<Tenant> tenant = controller.tenants().tenant(tenantName); if ( ! tenant.isPresent()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); // NOTE: The Jersey implementation would silently ignore this diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index e367de35d46..26410280566 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -72,6 +72,12 @@ public class VersionStatus { return versions().stream().filter(VespaVersion::isSystemVersion).findFirst(); } + /** Returns whether the system is currently upgrading */ + public boolean isUpgrading() { + return systemVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion) + .isBefore(controllerVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion)); + } + /** * Lists all currently active Vespa versions, with deployment statistics, * sorted from lowest to highest version number. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 06417da8157..32068b006f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; @@ -426,7 +427,7 @@ public class ControllerTest { } @Test - public void testDeployDirectly() { + public void testIntegrationTestDeployment() { DeploymentTester tester = new DeploymentTester(); Version six = Version.fromString("6.1"); tester.upgradeSystem(six); @@ -461,6 +462,28 @@ public class ControllerTest { } @Test + public void testDevDeployment() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.dev) + .majorVersion(6) + .region("us-east-1") + .build(); + + // Create application + Application app = tester.createApplication("app1", "tenant1", 1, 2L); + ZoneId zone = ZoneId.from("dev", "us-east-1"); + + // Deploy + tester.controller().applications().deploy(app.id(), zone, Optional.of(applicationPackage), DeployOptions.none()); + assertTrue("Application deployed and activated", + tester.controllerTester().configServer().application(app.id()).get().activated()); + assertTrue("No job status added", + tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); + assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, tester.applications().require(app.id()).deploymentSpec()); + } + + @Test public void testSuspension() { DeploymentTester tester = new DeploymentTester(); Application app = tester.createApplication("app1", "tenant1", 1, 11L); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7697cf00b86..552bcd34fe4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -392,27 +392,39 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID) .data("6.1.0"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", GET) + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false}"); // POST pinning again tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the version, but leave the pin tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/platform", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .userIdentity(USER_ID), "{\"pinned\":true}"); // DELETE also the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .userIdentity(USER_ID), "{}"); // POST a pause to a production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) @@ -672,6 +684,12 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) a system application with an application package HttpEntity noAppEntity = createApplicationDeployData(Optional.empty(), true); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) + .data(noAppEntity) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Deployment of system applications during a system upgrade is not allowed\"}", + 400); + tester.upgradeSystem(tester.controller().versionStatus().controllerVersion().get().versionNumber()); + tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) .data(noAppEntity) .userIdentity(USER_ID), new File("deploy-result.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 443a2c23b6a..3744e44152a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -66,7 +66,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastCompleted": { @@ -80,7 +80,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastSuccess": { @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" } }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 64c92f9cde4..822bc447d8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -66,7 +66,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastCompleted": { @@ -80,7 +80,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastSuccess": { @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" } }, diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 1053c5ff44d..418837ca2a0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -152,8 +152,11 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeWeightedSet(JsonGenerator generator, FieldBase field, WeightedSet<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); - wrapIOException(() -> { generator.writeStartObject(); @@ -168,8 +171,11 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeCollectionField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, CollectionFieldValue<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); - wrapIOException(() -> { generator.writeStartArray(); Iterator<T> i = value.iterator(); @@ -184,6 +190,10 @@ public class JsonSerializationHelper { public static <K extends FieldValue, V extends FieldValue> void serializeMapField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, MapFieldValue<K, V> map) { + // Hide empty fields + if (map.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); wrapIOException(() -> { generator.writeStartObject(); @@ -203,6 +213,10 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeArrayField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Array<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } wrapIOException(() -> { fieldNameIfNotNull(generator, field); generator.writeStartArray(); diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index c7660e5d527..32d23349dc0 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -178,6 +178,12 @@ std::unique_ptr<Tensor> createExpectedUpdatedTensorWith2Cells() { {{{"x", "9"}, {"y", "9"}}, 11} }, {"x", "y"}); } +std::unique_ptr<Tensor> createExpectedAddUpdatedTensorWith3Cells() { + return createTensor({ {{{"x", "8"}, {"y", "8"}}, 2}, + {{{"x", "8"}, {"y", "9"}}, 2}, + {{{"x", "9"}, {"y", "9"}}, 11} }, {"x", "y"}); +} + FieldValue::UP createTensorFieldValueWith2Cells() { auto fv(std::make_unique<TensorFieldValue>()); *fv = createTensorWith2Cells(); @@ -973,14 +979,21 @@ DocumentUpdateTest::testTensorAddUpdate() updated.setValue(updated.getField("tensor"), *oldTensor); CPPUNIT_ASSERT(*doc != updated); testValueUpdate(*createTensorAddUpdate(), *DataType::TENSOR); + std::string expTensorAddUpdateString("TensorAddUpdate(" + "{TensorFieldValue: " + "{\"dimensions\":[\"x\",\"y\"]," + "\"cells\":[" + "{\"address\":{\"x\":\"8\",\"y\":\"9\"},\"value\":2}," + "{\"address\":{\"x\":\"8\",\"y\":\"8\"},\"value\":2}" + "]}})"); + CPPUNIT_ASSERT_EQUAL(expTensorAddUpdateString, createTensorAddUpdate()->toString()); DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(*createTensorAddUpdate())); upd.applyTo(updated); FieldValue::UP fval(updated.getValue("tensor")); CPPUNIT_ASSERT(fval); auto &tensor = asTensor(*fval); - // Note: Placeholder value for now - auto expectedUpdatedTensor = createTensorWith2Cells(); + auto expectedUpdatedTensor = createExpectedAddUpdatedTensorWith3Cells(); CPPUNIT_ASSERT(tensor.equals(*expectedUpdatedTensor)); } @@ -994,6 +1007,13 @@ DocumentUpdateTest::testTensorModifyUpdate() updated.setValue(updated.getField("tensor"), *oldTensor); CPPUNIT_ASSERT(*doc != updated); testValueUpdate(*createTensorModifyUpdate(), *DataType::TENSOR); + std::string expTensorModifyUpdateString("TensorModifyUpdate(replace," + "{TensorFieldValue: " + "{\"dimensions\":[\"x\",\"y\"]," + "\"cells\":[" + "{\"address\":{\"x\":\"8\",\"y\":\"9\"},\"value\":2}" + "]}})"); + CPPUNIT_ASSERT_EQUAL(expTensorModifyUpdateString, createTensorModifyUpdate()->toString()); DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(*createTensorModifyUpdate())); upd.applyTo(updated); diff --git a/document/src/vespa/document/update/tensoraddupdate.cpp b/document/src/vespa/document/update/tensoraddupdate.cpp index eb708d9f651..714eee28ff3 100644 --- a/document/src/vespa/document/update/tensoraddupdate.cpp +++ b/document/src/vespa/document/update/tensoraddupdate.cpp @@ -81,7 +81,11 @@ TensorAddUpdate::checkCompatibility(const Field& field) const std::unique_ptr<Tensor> TensorAddUpdate::applyTo(const Tensor &tensor) const { - return tensor.clone(); + auto &addTensor = _tensor->getAsTensorPtr(); + if (addTensor) { + return tensor.add(*addTensor); + } + return std::unique_ptr<Tensor>(); } bool @@ -112,9 +116,11 @@ TensorAddUpdate::printXml(XmlOutputStream& xos) const void TensorAddUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { - (void) verbose; - (void) indent; - out << "{TensorAddUpdate::print not yet implemented}"; + out << indent << "TensorAddUpdate("; + if (_tensor) { + _tensor->print(out, verbose, indent); + } + out << ")"; } void diff --git a/document/src/vespa/document/update/tensormodifyupdate.cpp b/document/src/vespa/document/update/tensormodifyupdate.cpp index a02379e4991..8cee367cae0 100644 --- a/document/src/vespa/document/update/tensormodifyupdate.cpp +++ b/document/src/vespa/document/update/tensormodifyupdate.cpp @@ -50,6 +50,23 @@ getJoinFunction(TensorModifyUpdate::Operation operation) } } +vespalib::string +getJoinFunctionName(TensorModifyUpdate::Operation operation) +{ + using Operation = TensorModifyUpdate::Operation; + + switch (operation) { + case Operation::REPLACE: + return "replace"; + case Operation::ADD: + return "add"; + case Operation::MUL: + return "multiply"; + default: + throw IllegalArgumentException("Bad operation", VESPA_STRLOC); + } +} + } IMPLEMENT_IDENTIFIABLE(TensorModifyUpdate, ValueUpdate); @@ -156,9 +173,11 @@ TensorModifyUpdate::printXml(XmlOutputStream& xos) const void TensorModifyUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { - (void) verbose; - (void) indent; - out << "{TensorModifyUpdate::print not yet implemented}"; + out << indent << "TensorModifyUpdate(" << getJoinFunctionName(_operation) << ","; + if (_tensor) { + _tensor->print(out, verbose, indent); + } + out << ")"; } void diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp index e3b4c8dee42..79a8b994480 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp @@ -21,13 +21,13 @@ MutableDenseTensorView::MutableValueType::MutableValueType(ValueType type_in) MutableDenseTensorView::MutableValueType::~MutableValueType() = default; MutableDenseTensorView::MutableDenseTensorView(ValueType type_in) - : DenseTensorView(_concreteType.fast_type(), CellsRef()), + : DenseTensorView(_concreteType._type, CellsRef()), _concreteType(type_in) { } MutableDenseTensorView::MutableDenseTensorView(ValueType type_in, CellsRef cells_in) - : DenseTensorView(_concreteType.fast_type(), cells_in), + : DenseTensorView(_concreteType._type, cells_in), _concreteType(type_in) { } diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h index b68a1594905..2132f861896 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h @@ -15,8 +15,8 @@ class MutableDenseTensorView : public DenseTensorView private: struct MutableValueType { - private: eval::ValueType _type; + private: std::vector<eval::ValueType::Dimension::size_type *> _unboundDimSizes; public: diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp index 90b25621cf8..25bc4e23d96 100644 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ b/metrics/src/vespa/metrics/xmlwriter.cpp @@ -11,8 +11,8 @@ namespace metrics { XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, - uint32_t period, int verbosity) - : _period(period), _xos(xos), _verbosity(verbosity) {} + [[maybe_unused]] uint32_t period, int verbosity) + : _xos(xos), _verbosity(verbosity) {} bool XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h index fbaf59d2ecd..31feb18bae8 100644 --- a/metrics/src/vespa/metrics/xmlwriter.h +++ b/metrics/src/vespa/metrics/xmlwriter.h @@ -8,7 +8,6 @@ namespace metrics { class XmlWriter : public MetricVisitor { - uint32_t _period; vespalib::xml::XmlOutputStream& _xos; int _verbosity; diff --git a/node-admin/pom.xml b/node-admin/pom.xml index fda4acebfa0..d550ceb7c9d 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -55,6 +55,16 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <scope>provided</scope> + </dependency> <!-- Compile --> <dependency> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 1bb0cf3fb10..0e6000c651b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import org.apache.http.HttpHeaders; @@ -40,8 +39,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static java.util.Collections.singleton; - /** * Retries request on config server a few times before giving up. Assumes that all requests should be sent with * content-type application/json @@ -58,11 +55,6 @@ public class ConfigServerApiImpl implements ConfigServerApi { private final CloseableHttpClient client; - // TODO: Remove after 2018-12-01 - public static ConfigServerApiImpl create(ConfigServerInfo info, SiaIdentityProvider provider) { - return create(info, provider, new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity()))); - } - public static ConfigServerApiImpl create(ConfigServerInfo info, SiaIdentityProvider provider, HostnameVerifier hostnameVerifier) { return new ConfigServerApiImpl( info.getConfigServerUris(), @@ -70,13 +62,6 @@ public class ConfigServerApiImpl implements ConfigServerApi { provider); } - // TODO: Remove after 2018-12-01 - public static ConfigServerApiImpl createFor(ConfigServerInfo info, - SiaIdentityProvider provider, - HostName configServerHostname) { - return createFor(info, provider, new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity())), configServerHostname); - } - public static ConfigServerApiImpl createFor(ConfigServerInfo info, SiaIdentityProvider provider, HostnameVerifier hostnameVerifier, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 28e812e6ca1..a5635507ab7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -145,6 +145,10 @@ public class NodeAttributes { currentFirmwareCheck, hardwareDivergence, hardwareFailureDescription, wantToDeprovision, reports); } + public boolean isEmpty() { + return equals(new NodeAttributes()); + } + @Override public boolean equals(final Object o) { if (!(o instanceof NodeAttributes)) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java index 26c1b90218f..42804b69b17 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Objects; +import java.util.Optional; import static com.yahoo.yolean.Exceptions.uncheck; @@ -21,8 +22,8 @@ import static com.yahoo.yolean.Exceptions.uncheck; * <p><strong>Subclass requirements</strong> * * <ol> - * <li>A subclass must maintain the property that {@link ObjectMapper} can map an instance to {@link JsonNode}, - * see {@link #toJsonNode()}.</li> + * <li>A subclass mush be a Jackson class that can be mapped to {@link JsonNode} with {@link #toJsonNode()}, + * and from {@link JsonNode} with {@link #fromJsonNode(JsonNode, Class)}.</li> * <li>A subclass must override {@link #updates(BaseReport)} and make sure to return false if * {@code !super.updates(current)}.</li> * </ol> @@ -52,12 +53,12 @@ public class BaseReport { } @JsonGetter(CREATED_FIELD) - public Long getCreatedMillisOrNull() { + public final Long getCreatedMillisOrNull() { return createdMillis; } @JsonGetter(DESCRIPTION_FIELD) - public String getDescriptionOrNull() { + public final String getDescriptionOrNull() { return description; } @@ -73,8 +74,36 @@ public class BaseReport { return !Objects.equals(description, current.description); } + /** A variant of {@link #updates(BaseReport)} handling possibly absent reports, whether new or old. */ + public static <TNEW extends BaseReport, TOLD extends BaseReport> + boolean updates2(Optional<TNEW> newReport, Optional<TOLD> oldReport) { + if (newReport.isPresent() ^ oldReport.isPresent()) return true; + return newReport.map(r -> r.updates(oldReport.get())).orElse(false); + } + + public static BaseReport fromJsonNode(JsonNode jsonNode) { + return fromJsonNode(jsonNode, BaseReport.class); + } + + public static <R extends BaseReport> R fromJsonNode(JsonNode jsonNode, Class<R> jacksonClass) { + return uncheck(() -> mapper.treeToValue(jsonNode, jacksonClass)); + } + + public static BaseReport fromJson(String json) { + return fromJson(json, BaseReport.class); + } + + public static <R extends BaseReport> R fromJson(String json, Class<R> jacksonClass) { + return uncheck(() -> mapper.readValue(json, jacksonClass)); + } + /** Returns {@code this} as a {@link JsonNode}. */ public JsonNode toJsonNode() { return uncheck(() -> mapper.valueToTree(this)); } + + /** Returns {@code this} as a compact JSON string. */ + public String toJson() { + return uncheck(() -> mapper.writeValueAsString(this)); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a3e8d077513..577b4c5eef0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -253,8 +253,7 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container) { - final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(context, container); - coredumpHandler.converge(context, nodeAttributes); + coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container)); } private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) { @@ -264,7 +263,7 @@ public class StorageMaintainer { attributes.put("environment", context.zoneId().environment().value()); attributes.put("flavor", context.node().getFlavor()); attributes.put("kernel_version", System.getProperty("os.version")); - attributes.put("cpu_microcode_version", getMicrocodeVersion(context)); + attributes.put("cpu_microcode_version", getMicrocodeVersion()); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); context.node().getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); @@ -292,14 +291,11 @@ public class StorageMaintainer { new UnixPath(context.pathOnHostFromPathInNode("/")).deleteRecursively(); } - private String getMicrocodeVersion(NodeAgentContext context) { - String output = terminal.newCommandLine(context) - .add("grep", "microcode", "/proc/cpuinfo") - .setTimeout(Duration.ofSeconds(60)) - .executeSilently() - .getOutputLinesStream() + private String getMicrocodeVersion() { + String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() + .filter(line -> line.startsWith("microcode")) .findFirst() - .orElseThrow(() -> new RuntimeException("No microcode information found in /proc/cpuinfo")); + .orElseThrow(() -> new RuntimeException("No microcode information found in /proc/cpuinfo"))); String[] results = output.split(":"); if (results.length != 2) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java index 93d00cb2cc5..a9954200f8a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java @@ -11,6 +11,8 @@ import java.util.List; /** * An editor that assumes all rules in the filter table are exactly as the the wanted rules + * + * @author smorgrav */ class FilterTableLineEditor implements LineEditor { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java index 419432b54f7..e1dc4a661ff 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java @@ -10,6 +10,8 @@ import java.util.List; /** * An editor that only cares about the REDIRECT statement + * + * @author smorgrav */ class NatTableLineEditor implements LineEditor { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 19dce1bf75b..95964ec8e7f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -68,7 +68,7 @@ public class CoredumpHandler { } - public void converge(NodeAgentContext context, Map<String, Object> nodeAttributes) { + public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier) { Path containerCrashPathOnHost = context.pathOnHostFromPathInNode(crashPatchInContainer); Path containerProcessingPathOnHost = containerCrashPathOnHost.resolve(PROCESSING_DIRECTORY_NAME); @@ -80,7 +80,7 @@ public class CoredumpHandler { // Check if we have already started to process a core dump or we can enqueue a new core one getCoredumpToProcess(containerCrashPathOnHost, containerProcessingPathOnHost) - .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributes)); + .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributesSupplier)); } /** @return path to directory inside processing directory that contains a core dump file to process */ @@ -88,8 +88,7 @@ public class CoredumpHandler { return FileFinder.directories(containerProcessingPathOnHost).stream() .map(FileFinder.FileAttributes::path) .findAny() - .map(Optional::of) - .orElseGet(() -> enqueueCoredump(containerCrashPathOnHost, containerProcessingPathOnHost)); + .or(() -> enqueueCoredump(containerCrashPathOnHost, containerProcessingPathOnHost)); } /** @@ -115,9 +114,9 @@ public class CoredumpHandler { }); } - void processAndReportSingleCoredump(NodeAgentContext context, Path coredumpDirectory, Map<String, Object> nodeAttributes) { + void processAndReportSingleCoredump(NodeAgentContext context, Path coredumpDirectory, Supplier<Map<String, Object>> nodeAttributesSupplier) { try { - String metadata = getMetadata(context, coredumpDirectory, nodeAttributes); + String metadata = getMetadata(context, coredumpDirectory, nodeAttributesSupplier); String coredumpId = coredumpDirectory.getFileName().toString(); coredumpReporter.reportCoredump(coredumpId, metadata); finishProcessing(context, coredumpDirectory); @@ -131,13 +130,13 @@ public class CoredumpHandler { * @return coredump metadata from metadata.json if present, otherwise attempts to get metadata using * {@link CoreCollector} and stores it to metadata.json */ - String getMetadata(NodeAgentContext context, Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException { + String getMetadata(NodeAgentContext context, Path coredumpDirectory, Supplier<Map<String, Object>> nodeAttributesSupplier) throws IOException { UnixPath metadataPath = new UnixPath(coredumpDirectory.resolve(METADATA_FILE_NAME)); if (!Files.exists(metadataPath.toPath())) { Path coredumpFilePathOnHost = findCoredumpFileInProcessingDirectory(coredumpDirectory); Path coredumpFilePathInContainer = context.pathInNodeFromPathOnHost(coredumpFilePathOnHost); Map<String, Object> metadata = coreCollector.collect(context, coredumpFilePathInContainer); - metadata.putAll(nodeAttributes); + metadata.putAll(nodeAttributesSupplier.get()); String metadataFields = objectMapper.writeValueAsString(ImmutableMap.of("fields", metadata)); metadataPath.writeUtf8File(metadataFields); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index 37d79d97e74..456391c65c2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -18,6 +18,9 @@ public interface NodeAdmin { */ void refreshContainersToRun(final List<NodeSpec> containersToRun); + /** Gather node agent and its docker container metrics and forward them to the {@code MetricReceiverWrapper} */ + void updateNodeAgentMetrics(); + /** * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that * they will not pick up any changes from NodeRepository. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 2303f78217c..2b37dcdf69c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -38,17 +38,18 @@ import java.util.stream.Collectors; public class NodeAdminImpl implements NodeAdmin { private static final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(NodeAdmin.class); private static final Duration NODE_AGENT_FREEZE_TIMEOUT = Duration.ofSeconds(5); + private static final Duration NODE_AGENT_SPREAD = Duration.ofSeconds(3); private final ScheduledExecutorService aclScheduler = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("aclscheduler")); - private final ScheduledExecutorService metricsScheduler = - Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory; private final NodeAgentContextFactory nodeAgentContextFactory; private final Optional<AclMaintainer> aclMaintainer; private final Clock clock; + private final Duration freezeTimeout; + private final Duration spread; private boolean previousWantFrozen; private boolean isFrozen; private Instant startOfFreezeConvergence; @@ -64,19 +65,25 @@ public class NodeAdminImpl implements NodeAdmin { MetricReceiverWrapper metricReceiver, Clock clock) { this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - nodeAgentContextFactory, aclMaintainer, metricReceiver, clock); + nodeAgentContextFactory, aclMaintainer, metricReceiver, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); + } + + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, NodeAgentContextFactory nodeAgentContextFactory, + Optional<AclMaintainer> aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock, Duration freezeTimeout, Duration spread) { + this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), + nodeAgentContextFactory, aclMaintainer, metricReceiver, clock, freezeTimeout, spread); } NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, - NodeAgentContextFactory nodeAgentContextFactory, - Optional<AclMaintainer> aclMaintainer, - MetricReceiverWrapper metricReceiver, - Clock clock) { + NodeAgentContextFactory nodeAgentContextFactory, Optional<AclMaintainer> aclMaintainer, MetricReceiverWrapper metricReceiver, + Clock clock, Duration freezeTimeout, Duration spread) { this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; this.nodeAgentContextFactory = nodeAgentContextFactory; this.aclMaintainer = aclMaintainer; this.clock = clock; + this.freezeTimeout = freezeTimeout; + this.spread = spread; this.previousWantFrozen = true; this.isFrozen = true; this.startOfFreezeConvergence = clock.instant(); @@ -102,19 +109,25 @@ public class NodeAdminImpl implements NodeAdmin { nodeAgentWithSchedulerByHostname.put(hostname, naws); }); + Duration timeBetweenNodeAgents = spread.dividedBy(Math.max(nodeAgentContextsByHostname.size() - 1, 1)); + Instant nextAgentStart = clock.instant(); // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys - nodeAgentContextsByHostname.forEach((hostname, context) -> - nodeAgentWithSchedulerByHostname.get(hostname).scheduleTickWith(context) - ); + for (String hostname : nodeAgentContextsByHostname.keySet()) { + NodeAgentContext context = nodeAgentContextsByHostname.get(hostname); + nodeAgentWithSchedulerByHostname.get(hostname).scheduleTickWith(context, nextAgentStart); + nextAgentStart = nextAgentStart.plus(timeBetweenNodeAgents); + } } - private void updateNodeAgentMetrics() { + @Override + public void updateNodeAgentMetrics() { int numberContainersWaitingImage = 0; int numberOfNewUnhandledExceptions = 0; for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) { if (nodeAgentWithScheduler.isDownloadingImage()) numberContainersWaitingImage++; numberOfNewUnhandledExceptions += nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions(); + nodeAgentWithScheduler.updateContainerNodeMetrics(); } numberOfContainersInLoadImageState.sample(numberContainersWaitingImage); @@ -135,7 +148,7 @@ public class NodeAdminImpl implements NodeAdmin { // Use filter with count instead of allMatch() because allMatch() will short circuit on first non-match boolean allNodeAgentsConverged = nodeAgentWithSchedulerByHostname.values().parallelStream() - .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, NODE_AGENT_FREEZE_TIMEOUT)) + .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, freezeTimeout)) .count() == 0; if (wantFrozen) { @@ -173,15 +186,6 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void start() { - metricsScheduler.scheduleAtFixedRate(() -> { - try { - updateNodeAgentMetrics(); - nodeAgentWithSchedulerByHostname.values().forEach(NodeAgent::updateContainerNodeMetrics); - } catch (Throwable e) { - logger.warning("Metric fetcher scheduler failed", e); - } - }, 10, 55, TimeUnit.SECONDS); - aclMaintainer.ifPresent(maintainer -> { int delay = 120; // WARNING: Reducing this will increase the load on config servers. aclScheduler.scheduleWithFixedDelay(() -> { @@ -192,7 +196,6 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void stop() { - metricsScheduler.shutdown(); aclScheduler.shutdown(); // Stop all node-agents in parallel, will block until the last NodeAgent is stopped @@ -200,12 +203,11 @@ public class NodeAdminImpl implements NodeAdmin { do { try { - metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); aclScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { logger.info("Was interrupted while waiting for metricsScheduler and aclScheduler to shutdown"); } - } while (!metricsScheduler.isTerminated() || !aclScheduler.isTerminated()); + } while (!aclScheduler.isTerminated()); } // Set-difference. Returns minuend minus subtrahend. @@ -232,7 +234,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public boolean isDownloadingImage() { return nodeAgent.isDownloadingImage(); } @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } - @Override public void scheduleTickWith(NodeAgentContext context) { nodeAgentScheduler.scheduleTickWith(context); } + @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); } @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 13d3f3307d2..18c3a836e41 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; @@ -10,11 +11,17 @@ import com.yahoo.vespa.hosted.provision.Node; import java.time.Duration; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.TRANSITIONING; @@ -27,6 +34,9 @@ public class NodeAdminStateUpdater { private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); private static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); + private final ScheduledExecutorService metricsScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); + private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final NodeAdmin nodeAdmin; @@ -34,7 +44,7 @@ public class NodeAdminStateUpdater { public enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED } - private State currentState = SUSPENDED_NODE_ADMIN; + private volatile State currentState = SUSPENDED_NODE_ADMIN; public NodeAdminStateUpdater( NodeRepository nodeRepository, @@ -49,6 +59,31 @@ public class NodeAdminStateUpdater { public void start() { nodeAdmin.start(); + + EnumSet<State> suspendedStates = EnumSet.of(SUSPENDED_NODE_ADMIN, SUSPENDED); + metricsScheduler.scheduleAtFixedRate(() -> { + try { + if (suspendedStates.contains(currentState)) return; + nodeAdmin.updateNodeAgentMetrics(); + } catch (Throwable e) { + log.log(Level.WARNING, "Metric fetcher scheduler failed", e); + } + }, 10, 55, TimeUnit.SECONDS); + } + + public void stop() { + metricsScheduler.shutdown(); + + // Stop all node-agents in parallel, will block until the last NodeAgent is stopped + nodeAdmin.stop(); + + do { + try { + metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + log.info("Was interrupted while waiting for metricsScheduler and shutdown"); + } + } while (!metricsScheduler.isTerminated()); } /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java index 54f357d5f29..237b7d0daf7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.Objects; /** @@ -17,6 +18,7 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg private NodeAgentContext currentContext; private NodeAgentContext nextContext; + private Instant nextContextAt; private boolean wantFrozen = false; private boolean isFrozen = true; private boolean pendingInterrupt = false; @@ -27,9 +29,10 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg } @Override - public void scheduleTickWith(NodeAgentContext context) { + public void scheduleTickWith(NodeAgentContext context, Instant at) { synchronized (monitor) { nextContext = Objects.requireNonNull(context); + nextContextAt = Objects.requireNonNull(at); monitor.notifyAll(); // Notify of new context } } @@ -58,14 +61,17 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg @Override public NodeAgentContext nextContext() throws InterruptedException { synchronized (monitor) { - while (setAndGetIsFrozen(wantFrozen) || nextContext == null) { + Duration untilNextContext = Duration.ZERO; + while (setAndGetIsFrozen(wantFrozen) || + nextContext == null || + (untilNextContext = Duration.between(Instant.now(), nextContextAt)).toMillis() > 0) { if (pendingInterrupt) { pendingInterrupt = false; throw new InterruptedException("interrupt() was called before next context was scheduled"); } try { - monitor.wait(); // Wait until scheduler provides a new context + monitor.wait(Math.max(untilNextContext.toMillis(), 0L)); // Wait until scheduler provides a new context } catch (InterruptedException ignored) { } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java index 540601ffa4f..a5daab8dcfd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java @@ -2,14 +2,15 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import java.time.Duration; +import java.time.Instant; /** * @author freva */ public interface NodeAgentScheduler { - /** Schedule a tick for NodeAgent to run with the given NodeAgentContext */ - void scheduleTickWith(NodeAgentContext context); + /** Schedule a tick for NodeAgent to run with the given NodeAgentContext, at no earlier than given instant */ + void scheduleTickWith(NodeAgentContext context, Instant at); /** * Will eventually freeze/unfreeze the node agent diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index 9a35188a372..4588a9edcf4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -12,8 +12,6 @@ import static com.yahoo.yolean.Exceptions.uncheck; /** * Utils related to IOException. * - * todo: replace much of the below with com.yahoo.yolean.Exceptions::uncheck - * * @author hakonhall */ public class IOExceptionUtil { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java index eec17f019bd..561733d7d76 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; import com.yahoo.test.json.JsonTestHelper; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -11,11 +12,11 @@ import static org.junit.Assert.assertTrue; * @author hakonhall */ public class BaseReportTest { + private static final String JSON_1 = "{\"createdMillis\": 1, \"description\": \"desc\"}"; @Test public void testSerialization() { - BaseReport report = new BaseReport(1L, "desc"); JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc").toJsonNode(), - "{\"createdMillis\": 1, \"description\": \"desc\"}"); + JSON_1); JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc").toJsonNode(), "{\"description\": \"desc\"}"); JsonTestHelper.assertJsonEquals(new BaseReport(1L, null).toJsonNode(), @@ -35,4 +36,12 @@ public class BaseReportTest { assertTrue(new BaseReport(1L, "desc 2").updates(report)); assertTrue(new BaseReport(1L, null).updates(report)); } + + @Test + public void testJsonSerialization() { + BaseReport report = BaseReport.fromJson(JSON_1); + assertEquals(1L, (long) report.getCreatedMillisOrNull()); + assertEquals("desc", report.getDescriptionOrNull()); + JsonTestHelper.assertJsonEquals(report.toJson(), JSON_1); + } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 88cc833f1f8..0254f58e7eb 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -98,7 +98,7 @@ public class DockerTester implements AutoCloseable { Optional.empty(), Optional.empty(), Optional.empty()); NodeAgentContextFactory nodeAgentContextFactory = nodeSpec -> new NodeAgentContextImpl.Builder(nodeSpec).fileSystem(fileSystem).build(); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, nodeAgentContextFactory, Optional.empty(), mr, Clock.systemUTC()); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, nodeAgentContextFactory, Optional.empty(), mr, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, nodeAdmin, HOST_HOSTNAME); @@ -139,7 +139,7 @@ public class DockerTester implements AutoCloseable { @Override public void close() { // First, stop NodeAdmin and all the NodeAgents - nodeAdmin.stop(); + nodeAdminStateUpdater.stop(); terminated = true; do { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 7779a74ae03..6d3a4dbb553 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; -import com.google.common.collect.ImmutableMap; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -20,10 +19,8 @@ import java.nio.file.Paths; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -124,9 +121,9 @@ public class CoredumpHandlerTest { public void get_metadata_test() throws IOException { Map<String, Object> metadata = new HashMap<>(); metadata.put("bin_path", "/bin/bash"); - metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something")); + metadata.put("backtrace", List.of("call 1", "function 2", "something something")); - Map<String, Object> attributes = ImmutableMap.of( + Map<String, Object> attributes = Map.of( "hostname", "host123.yahoo.com", "vespa_version", "6.48.4", "kernel_version", "3.10.0-862.9.1.el7.x86_64", @@ -149,17 +146,17 @@ public class CoredumpHandlerTest { when(coreCollector.collect(eq(context), eq(coredumpDirectoryInContainer.resolve("dump_core.456")))) .thenReturn(metadata); - assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, attributes)); + assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, () -> attributes)); verify(coreCollector, times(1)).collect(any(), any()); // Calling it again will simply read the previously generated metadata from disk - assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, attributes)); + assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, () -> attributes)); verify(coreCollector, times(1)).collect(any(), any()); } @Test(expected = IllegalStateException.class) public void cant_get_metadata_if_no_core_file() throws IOException { - coredumpHandler.getMetadata(context, fileSystem.getPath("/fake/path"), Collections.emptyMap()); + coredumpHandler.getMetadata(context, fileSystem.getPath("/fake/path"), Map::of); } @Test(expected = IllegalStateException.class) @@ -184,7 +181,7 @@ public class CoredumpHandlerTest { uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3)))); return new TestChildProcess2(0, ""); }); - coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Collections.emptyMap()); + coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Map::of); verify(coreCollector, never()).collect(any(), any()); verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata")); assertFalse(Files.exists(coredumpDirectory)); @@ -203,7 +200,7 @@ public class CoredumpHandlerTest { } private static void assertFolderContents(Path pathToFolder, String... filenames) { - Set<String> expectedContentsOfFolder = new HashSet<>(Arrays.asList(filenames)); + Set<String> expectedContentsOfFolder = Set.of(filenames); Set<String> actualContentsOfFolder = new UnixPath(pathToFolder) .listContentsOfDirectory().stream() .map(unixPath -> unixPath.toPath().getFileName().toString()) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 47e220a968b..f8e87ccef53 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -45,7 +45,7 @@ public class NodeAdminImplTest { private final ManualClock clock = new ManualClock(); private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, nodeAgentContextFactory, - Optional.empty(), new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); + Optional.empty(), new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock, Duration.ZERO, Duration.ZERO); @Test public void nodeAgentsAreProperlyLifeCycleManaged() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index f32e3d91e34..5aeccb4ab7d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -26,7 +27,7 @@ public class NodeAgentContextManagerTest { @Test(timeout = TIMEOUT) public void returns_immediately_if_next_context_is_ready() throws InterruptedException { NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1); + manager.scheduleTickWith(context1, clock.instant()); assertSame(initialContext, manager.currentContext()); assertSame(context1, manager.nextContext()); @@ -34,6 +35,19 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) + public void returns_no_earlier_than_at_given_time() throws InterruptedException { + NodeAgentContext context1 = generateContext(); + Instant returnAt = clock.instant().plusMillis(500); + manager.scheduleTickWith(context1, returnAt); + + assertSame(initialContext, manager.currentContext()); + assertSame(context1, manager.nextContext()); + assertSame(context1, manager.currentContext()); + // Is accurate to a millisecond + assertFalse(clock.instant().plusMillis(1).isBefore(returnAt)); + } + + @Test(timeout = TIMEOUT) public void blocks_in_nextContext_until_one_is_scheduled() throws InterruptedException { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); assertFalse(async.response.isPresent()); @@ -41,7 +55,7 @@ public class NodeAgentContextManagerTest { assertFalse(async.response.isPresent()); NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1); + manager.scheduleTickWith(context1, clock.instant()); async.awaitResult(); assertEquals(Optional.of(context1), async.response); @@ -68,7 +82,7 @@ public class NodeAgentContextManagerTest { // Generate new context and get it from the supplier, this completes the unfreeze NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1); + manager.scheduleTickWith(context1, clock.instant()); assertSame(context1, manager.nextContext()); assertTrue(manager.setFrozen(false, Duration.ZERO)); @@ -91,7 +105,7 @@ public class NodeAgentContextManagerTest { assertFalse(async.response.isPresent()); NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1); + manager.scheduleTickWith(context1, clock.instant()); assertSame(context1, manager.nextContext()); async.awaitResult(); diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index e6b7b34903b..ede3f8256af 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -350,7 +350,6 @@ RandTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t) class ModTextFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; std::vector<uint32_t> _mods; public: @@ -363,10 +362,9 @@ public: ModTextFieldGenerator::ModTextFieldGenerator(const string &name, - search::Rand48 &rnd, + [[maybe_unused]] search::Rand48 &rnd, const std::vector<uint32_t> &mods) : FieldGenerator(name), - _rnd(rnd), _mods(mods) { } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 40c2733d230..fb8674b5255 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -778,12 +778,7 @@ Test::requireThatAttributesAreUsed() "bj", *rep, 0, rclass)); // empty doc - EXPECT_TRUE(assertSlime("{bd:[]," - "be:[]," - "bf:[]," - "bg:[]," - "bh:[]," - "bi:[]}", *rep, 1, false)); + EXPECT_TRUE(assertSlime("{}", *rep, 1, false)); TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass)); proton::IAttributeManager::SP attributeManager = dc._ddb->getReadySubDB()->getAttributeManager(); @@ -807,9 +802,7 @@ Test::requireThatAttributesAreUsed() req3.hits.push_back(DocsumRequest::Hit(gid3)); DocsumReply::UP rep3 = dc._ddb->getDocsums(req3); - EXPECT_TRUE(assertSlime("{bd:[],be:[],bf:[],bg:[]," - "bh:[],bi:[]," - "bj:'0x01020178017901016101624010000000000000'}", + EXPECT_TRUE(assertSlime("{bj:'0x01020178017901016101624010000000000000'}", *rep3, 0, true)); } diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp index c1626e94809..8c1ad7bf551 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp @@ -22,7 +22,7 @@ getGateVector(size_t size) { GateVector retval; for (size_t i = 0; i < size; ++i) { - retval.push_back(std::move(GateUP(new Gate()))); + retval.push_back(GateUP(new Gate())); } return retval; } diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 4a3ced6891c..c3974368e54 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -1003,7 +1003,6 @@ TEST_F("require that bucket move controller is active", f._builder.createDocs(3, 1, 3); // 2 docs f._builder.createDocs(4, 3, 6); // 3 docs test::UserDocuments notReadyDocs(f._builder.getDocs()); - BucketId bucketId3(notReadyDocs.getBucket(3)); BucketId bucketId4(notReadyDocs.getBucket(4)); f.insertDocs(notReadyDocs, f._notReady); f._builder.clearDocs(); @@ -1053,15 +1052,12 @@ TEST_F("require that document pruner is active", f._builder.createDocs(1, 1, 4); // 3 docs f._builder.createDocs(2, 4, 6); // 2 docs test::UserDocuments keepDocs(f._builder.getDocs()); - BucketId bucketId1(keepDocs.getBucket(1)); - BucketId bucketId2(keepDocs.getBucket(2)); f.removeDocs(keepDocs, keepTime); f._builder.clearDocs(); f._builder.createDocs(3, 6, 8); // 2 docs f._builder.createDocs(4, 8, 11); // 3 docs test::UserDocuments removeDocs(f._builder.getDocs()); BucketId bucketId3(removeDocs.getBucket(3)); - BucketId bucketId4(removeDocs.getBucket(4)); f.removeDocs(removeDocs, remTime); f.notifyClusterStateChanged(); EXPECT_TRUE(f._executor.isIdle()); diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index bbe6db7733e..d436c63ae2e 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -709,10 +709,10 @@ TEST("requireThatWeCanPutAndRemoveBeforeFreeListConstruct") EXPECT_EQUAL(2u, dms.getNumUsedLids()); EXPECT_EQUAL(5u, dms.getNumDocs()); // gid1 already there with lid 1 - EXPECT_EXCEPTION(!dms.put(gid1, bucketId1, time1, docSize1, 2).ok(), + EXPECT_EXCEPTION(dms.put(gid1, bucketId1, time1, docSize1, 2).ok(), vespalib::IllegalStateException, "gid found, but using another lid"); - EXPECT_EXCEPTION(!dms.put(gid5, bucketId5, time5, docSize5, 1).ok(), + EXPECT_EXCEPTION(dms.put(gid5, bucketId5, time5, docSize5, 1).ok(), vespalib::IllegalStateException, "gid not found, but lid is used by another gid"); EXPECT_TRUE(assertLid(1, gid1, dms)); diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index e851e30a31d..18d4fbdb6d7 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -91,7 +91,6 @@ const string word1 = "foo"; const string word2 = "bar"; const DocumentIdT doc_id1 = 1; const DocumentIdT doc_id2 = 2; -const uint32_t field_id = 1; Schema getSchema() { Schema schema; diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index f668072b9fd..96092625979 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -108,7 +108,7 @@ public: return std::make_unique<WrappedFlushTask>(std::move(task), _handler); } - return std::move(task); + return task; } }; @@ -160,7 +160,7 @@ public: wrappedTargets.push_back(std::make_shared<WrappedFlushTarget> (target, *this)); } - return std::move(wrappedTargets); + return wrappedTargets; } // Called once by flush engine slave thread for each task done diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 7c6779fdc63..2a29847a634 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -185,10 +185,9 @@ struct MyWorld { for (uint32_t i = 0; i < NUM_DOCS; ++i) { document::DocumentId docId(vespalib::make_string("doc::%u", i)); const document::GlobalId &gid = docId.getGlobalId(); - typedef DocumentMetaStore::Result PutRes; document::BucketId bucketId(BucketFactory::getBucketId(docId)); uint32_t docSize = 1; - PutRes putRes(metaStore.put(gid, bucketId, Timestamp(0u), docSize, i)); + metaStore.put(gid, bucketId, Timestamp(0u), docSize, i); metaStore.setBucketState(bucketId, true); } } diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 82aab72068d..8ecf6fd4a43 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -802,7 +802,8 @@ void Test::requireThatWeakAndBlueprintsAreCreatedCorrectly() { wand.append(Node::UP(new ProtonStringTerm("foo", field, 0, Weight(3)))); wand.append(Node::UP(new ProtonStringTerm("bar", field, 0, Weight(7)))); - ResolveViewVisitor resolve_visitor(ViewResolver(), plain_index_env); + ViewResolver viewResolver; + ResolveViewVisitor resolve_visitor(viewResolver, plain_index_env); wand.accept(resolve_visitor); FakeRequestContext requestContext; @@ -834,7 +835,8 @@ void Test::requireThatParallelWandBlueprintsAreCreatedCorrectly() { wand.append(Node::UP(new ProtonStringTerm("foo", field, 0, Weight(3)))); wand.append(Node::UP(new ProtonStringTerm("bar", field, 0, Weight(7)))); - ResolveViewVisitor resolve_visitor(ViewResolver(), attribute_index_env); + ViewResolver viewResolver; + ResolveViewVisitor resolve_visitor(viewResolver, attribute_index_env); wand.accept(resolve_visitor); FakeRequestContext requestContext; diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index b3a7f5f525e..716b369d928 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -89,13 +89,13 @@ TEST_F("require that strategy is updated with conservative max tls size value if TEST_F("require that strategy is updated with conservative max memory value if memory limit is reached", Fixture) { f.notifyDiskMemUsage(belowLimit(), aboveLimit()); - TEST_DO(f.assertStrategyConfig(2, 0.5, 20)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); } TEST_F("require that strategy is updated with all conservative values if both limits are reached", Fixture) { f.notifyDiskMemUsage(aboveLimit(), aboveLimit()); - TEST_DO(f.assertStrategyConfig(2, 0.5, 12)); + TEST_DO(f.assertStrategyConfig(2, 0, 12)); } TEST_F("require that last disk/memory usage state is remembered when setting new config", Fixture) @@ -129,11 +129,11 @@ TEST_F("require that we must go below low watermark for disk usage before using TEST_F("require that we must go below low watermark for memory usage before using normal max memory value again", Fixture) { f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.8)); - TEST_DO(f.assertStrategyConfig(2, 0.5, 20)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.7)); - TEST_DO(f.assertStrategyConfig(2, 0.5, 20)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.56)); - TEST_DO(f.assertStrategyConfig(2, 0.5, 20)); + TEST_DO(f.assertStrategyConfig(2, 0, 20)); f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.55)); TEST_DO(f.assertStrategyConfig(4, 1, 20)); f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.6)); diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 56596ec5bb6..23ef86a46b7 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -254,7 +254,7 @@ createSlimeRequestLarger(size_t num, array.addData(Memory(GID1, 12)); array.addData(Memory(GID2, 12)); } - return std::move(r); + return r; } Slime diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index c94a3048298..1492a34b241 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -60,7 +60,7 @@ std::unique_ptr<HwinfoConfig> readConfig(const vespalib::string &path) { ConfigSubscriber s(spec); std::unique_ptr<ConfigHandle<HwinfoConfig>> handle = s.subscribe<HwinfoConfig>("hwinfo"); s.nextConfig(0); - return std::move(handle->getConfig()); + return handle->getConfig(); } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index f7e0b7981bb..ed56599b6fe 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -377,7 +377,7 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler _pendingPrune.erase(result); } _pendingPrune.insert(flushHandler); - return std::move(result); + return result; } IFlushHandler::SP @@ -386,7 +386,7 @@ FlushEngine::removeFlushHandler(const DocTypeName &docTypeName) std::lock_guard<std::mutex> guard(_lock); IFlushHandler::SP result(_handlers.removeHandler(docTypeName)); _pendingPrune.erase(result); - return std::move(result); + return result; } FlushEngine::FlushMetaSet diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h index 1a7d3cf9ff6..7860acaab41 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h @@ -15,7 +15,6 @@ private: uint32_t _taskId; FlushEngine &_engine; FlushContext::SP _context; - search::SerialNum _serial; public: FlushTask(const FlushTask &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index e23a734fc2f..896ac83403a 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -23,9 +23,7 @@ private: HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; vespalib::SimpleThreadBundle::Pool _threadBundlePool; - bool _online; bool _nodeUp; - bool _inService; public: /** diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index b32af7e3e5a..100f3eff0e7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -150,7 +150,7 @@ Matcher::getStats() { std::lock_guard<std::mutex> guard(_statsLock); MatchingStats stats = std::move(_stats); - _stats = std::move(MatchingStats()); + _stats = MatchingStats(); _stats.softDoomFactor(stats.softDoomFactor()); return stats; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index feea15ddadd..06969167ab3 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -51,9 +51,10 @@ PersistenceHandlerMap::getHandlerSnapshot() const handlers.push_back(handlerItr.second); } } + size_t handlersSize = handlers.size(); return std::make_unique<HandlerSnapshot> (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)), - handlers.size()); + handlersSize); } namespace { diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h index 494edec78fc..4e5a0389add 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h @@ -18,8 +18,6 @@ namespace proton { */ class DummyGidToLidChangeHandler : public IGidToLidChangeHandler { - bool _closed; - public: DummyGidToLidChangeHandler(); virtual ~DummyGidToLidChangeHandler(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index e01fb048266..c673f615f25 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -35,13 +35,12 @@ DocumentDBMetricsUpdater::DocumentDBMetricsUpdater(const DocumentSubDBCollection DocumentDBJobTrackers &jobTrackers, matching::SessionManager &sessionManager, const AttributeUsageFilter &writeFilter, - const DDBState &state) + [[maybe_unused]] const DDBState &state) : _subDBs(subDBs), _writeService(writeService), _jobTrackers(jobTrackers), _sessionManager(sessionManager), - _writeFilter(writeFilter), - _state(state) + _writeFilter(writeFilter) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h index 7c27b273c59..dbf4c45007f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h @@ -34,7 +34,6 @@ private: DocumentDBJobTrackers &_jobTrackers; matching::SessionManager &_sessionManager; const AttributeUsageFilter &_writeFilter; - const DDBState &_state; // Last updated document store cache statistics. Necessary due to metrics implementation is upside down. DocumentStoreCacheStats _lastDocStoreCacheStats; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 1d1bbf21d70..82d31fbb6e6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -227,7 +227,6 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>; using IndexschemaConfigSP = DocumentDBConfig::IndexschemaConfigSP; - using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP; using SummaryConfigSP = DocumentDBConfig::SummaryConfigSP; using SummarymapConfigSP = DocumentDBConfig::SummarymapConfigSP; using JuniperrcConfigSP = DocumentDBConfig::JuniperrcConfigSP; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 531873fcd19..744b5060ca5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -33,8 +33,6 @@ namespace proton { namespace { -constexpr search::SerialNum ATTRIBUTE_INIT_SERIAL = 1; - struct AttributeGuardComp { vespalib::string name; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 97cc25e635f..a0d62f2052d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -327,7 +327,7 @@ FeedHandler::changeFeedState(FeedState::SP newState, const std::lock_guard<std:: FeedHandler::FeedHandler(IThreadingService &writeService, const vespalib::string &tlsSpec, const DocTypeName &docTypeName, - DDBState &state, + [[maybe_unused]] DDBState &state, IFeedHandlerOwner &owner, const IResourceWriteFilter &writeFilter, IReplayConfig &replayConfig, @@ -341,7 +341,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, IGetSerialNum(), _writeService(writeService), _docTypeName(docTypeName), - _state(state), _owner(owner), _writeFilter(writeFilter), _replayConfig(replayConfig), diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index ef554a2ee3a..af06f898716 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -74,7 +74,6 @@ private: IThreadingService &_writeService; DocTypeName _docTypeName; - DDBState &_state; IFeedHandlerOwner &_owner; const IResourceWriteFilter &_writeFilter; IReplayConfig &_replayConfig; diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index c68d794a5e0..744ce2aa7f4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -43,8 +43,7 @@ injectLidSpaceCompactionJobs(MaintenanceController &controller, config.getBlockableJobConfig(), clusterStateChangedNotifier, (calc ? calc->nodeRetired() : false))); - controller.registerJobInMasterThread(std::move(trackJob(tracker, - std::move(job)))); + controller.registerJobInMasterThread(trackJob(tracker, std::move(job))); } } @@ -76,8 +75,8 @@ injectBucketMoveJob(MaintenanceController &controller, diskMemUsageNotifier, blockableConfig, docTypeName, bucketSpace)); - controller.registerJobInMasterThread(std::move(trackJob(jobTrackers.getBucketMove(), - std::move(bmj)))); + controller.registerJobInMasterThread(trackJob(jobTrackers.getBucketMove(), + std::move(bmj))); } } @@ -115,7 +114,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, MUP pruneRDjob(new PruneRemovedDocumentsJob(config.getPruneRemovedDocumentsConfig(), *mRemSubDB._metaStore, mRemSubDB._subDbId, docTypeName, prdHandler, fbHandler)); controller.registerJobInMasterThread( - std::move(trackJob(jobTrackers.getRemovedDocumentsPrune(), std::move(pruneRDjob)))); + trackJob(jobTrackers.getRemovedDocumentsPrune(), std::move(pruneRDjob))); if (!config.getLidSpaceCompactionConfig().isDisabled()) { injectLidSpaceCompactionJobs(controller, config, lscHandlers, opStorer, fbHandler, jobTrackers.getLidSpaceCompact(), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 3542fcc3716..978720f88ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -107,9 +107,8 @@ VESPA_THREAD_STACK_TAG(close_executor) } -Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, const vespalib::string &creator) - : _proton(proton_), - _hostName(), +Proton::ProtonFileHeaderContext::ProtonFileHeaderContext([[maybe_unused]] const Proton &proton_, const vespalib::string &creator) + : _hostName(), _creator(creator), _cluster(), _pid(getpid()) @@ -204,7 +203,6 @@ Proton::Proton(const config::ConfigUri & configUri, _queryLimiter(), _clock(0.010), _threadPool(128 * 1024), - _configGen(0), _distributionKey(-1), _isInitializing(true), _isReplayDone(false), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 39e32c7f504..14ddcee3d5c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -75,7 +75,6 @@ private: class ProtonFileHeaderContext : public search::common::FileHeaderContext { - const Proton &_proton; vespalib::string _hostName; vespalib::string _creator; vespalib::string _cluster; @@ -120,7 +119,6 @@ private: matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; FastOS_ThreadPool _threadPool; - int64_t _configGen; uint32_t _distributionKey; bool _isInitializing; bool _isReplayDone; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index c0c9b961ac8..7a46e0769cf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -46,7 +46,6 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) _constantValueRepo(_constantValueCache), _configurer(_iSummaryMgr, _rSearchView, _rFeedView, ctx._queryLimiter, _constantValueRepo, ctx._clock, getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), - _numSearcherThreads(cfg._numSearcherThreads), _warmupExecutor(ctx._warmupExecutor), _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()), _flushConfig(), diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 52e0309dc16..e39ada31673 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -80,7 +80,6 @@ private: vespalib::eval::ConstantValueCache _constantValueCache; matching::ConstantValueRepo _constantValueRepo; SearchableDocSubDBConfigurer _configurer; - const size_t _numSearcherThreads; vespalib::ThreadExecutor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; DocumentDBFlushConfig _flushConfig; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 99186f89c6d..ef580cf44ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -505,12 +505,11 @@ StoreOnlyDocSubDB::getDocumentDBReference() } StoreOnlySubDBFileHeaderContext:: -StoreOnlySubDBFileHeaderContext(StoreOnlyDocSubDB &owner, +StoreOnlySubDBFileHeaderContext([[maybe_unused]] StoreOnlyDocSubDB &owner, const FileHeaderContext & parentFileHeaderContext, const DocTypeName &docTypeName, const vespalib::string &baseDir) : FileHeaderContext(), - _owner(owner), _parentFileHeaderContext(parentFileHeaderContext), _docTypeName(docTypeName), _subDB() diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 450779de7dd..2518ea38f82 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -64,7 +64,6 @@ class StoreOnlyDocSubDB; */ class StoreOnlySubDBFileHeaderContext : public search::common::FileHeaderContext { - StoreOnlyDocSubDB &_owner; const search::common::FileHeaderContext &_parentFileHeaderContext; const DocTypeName &_docTypeName; vespalib::string _subDB; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 22ff1b90de4..59b9a655d2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -353,7 +353,7 @@ void StoreOnlyFeedView::putSummary(SerialNum serialNum, Lid lid, summaryExecutor().execute( makeLambdaTask([serialNum, lid, futureStream = std::move(futureStream), onDone, this] () mutable { (void) onDone; - vespalib::nbostream os = std::move(futureStream.get()); + vespalib::nbostream os = futureStream.get(); if (!os.empty()) { _summaryAdapter->put(serialNum, lid, os); } diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index fa29eb33933..5cc547b4dcb 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -120,7 +120,7 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore virtual void setBucketState(const BucketId &bucketId, bool active) override { _store.setBucketState(bucketId, active); } - virtual void populateActiveBuckets(const BucketId::List &buckets) override { + virtual void populateActiveBuckets(const document::BucketId::List &buckets) override { _store.populateActiveBuckets(buckets); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 20c66993486..2d991c7d565 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -636,6 +636,6 @@ namespace attribute { } -template class vespalib::hash_map<search::EnumStoreIndex, search::EnumStoreIndex, - vespalib::hash<search::EnumStoreIndex>, std::equal_to<search::EnumStoreIndex>, - vespalib::hashtable_base::and_modulator>; +VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(search::EnumStoreIndex, search::EnumStoreIndex, + vespalib::hash<search::EnumStoreIndex>, std::equal_to<search::EnumStoreIndex>, + vespalib::hashtable_base::and_modulator); diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp index 38d9160de2a..6c0d9342f39 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multistringattribute.h" +#include "multistringattribute.hpp" #include <vespa/searchlib/query/queryterm.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index 3bb1481969a..fe6da8a4cf0 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -85,3 +85,5 @@ StoreByBucket::drain(IWrite & drainer) } } + +VESPALIB_HASH_MAP_INSTANTIATE(uint64_t, vespalib::ConstBufferRef); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 19dd096c46c..1ca1a336d2d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -149,9 +149,13 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inser using vespalib::slime::Cursor; using vespalib::Memory; const IAttributeVector & v = vec(*state); - uint32_t entries = v.getValueCount(docid); bool isWeightedSet = v.hasWeightedSetType(); + uint32_t entries = v.getValueCount(docid); + if (entries == 0) { + return; // Don't insert empty fields + } + Cursor &arr = target.insertArray(); BasicType::Type t = v.getBasicType(); switch (t) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 91a7fd45061..72cedb05f7c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -231,10 +231,6 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor FieldValue::UP _field_value; FieldValueConverter &_structuredFieldConverter; - void visit(const ArrayFieldValue &value) override { - _field_value = _structuredFieldConverter.convert(value); - } - template <typename T> void visitPrimitive(const T &t) { _field_value.reset(t.clone()); @@ -274,8 +270,16 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor visitPrimitive(value); } - void visit(const MapFieldValue & v) override { - _field_value = _structuredFieldConverter.convert(v); + void visit(const ArrayFieldValue &value) override { + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string + } + + void visit(const MapFieldValue & value) override { + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string } void visit(const StructFieldValue &value) override { @@ -292,7 +296,9 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor } void visit(const WeightedSetFieldValue &value) override { - _field_value = _structuredFieldConverter.convert(value); + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string } void visit(const TensorFieldValue &value) override { diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index c94121062ec..10f806f2e97 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -115,7 +115,6 @@ namespace { void LockableMapTest::testSimpleUsage() { // Tests insert, erase, size, empty, operator[] - typedef LockableMap<JudyMultiMap<A> > Map; Map map; // Do some insertions CPPUNIT_ASSERT(map.empty()); @@ -154,7 +153,6 @@ LockableMapTest::testSimpleUsage() { void LockableMapTest::testComparison() { - typedef LockableMap<JudyMultiMap<A> > Map; Map map1; Map map2; bool preExisted; @@ -579,7 +577,6 @@ printBuckets(const std::map<document::BucketId, Map::WrappedEntry>& results) { void LockableMapTest::testFindBucketsSimple() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(17, 0x0ffff); @@ -608,7 +605,6 @@ LockableMapTest::testFindBucketsSimple() { void LockableMapTest::testFindBuckets() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); @@ -637,7 +633,6 @@ LockableMapTest::testFindBuckets() { void LockableMapTest::testFindBuckets2() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); @@ -666,7 +661,6 @@ LockableMapTest::testFindBuckets2() { // ticket 3121525 void LockableMapTest::testFindBuckets3() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); @@ -689,7 +683,6 @@ LockableMapTest::testFindBuckets3() { // ticket 3121525 void LockableMapTest::testFindBuckets4() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); @@ -714,7 +707,6 @@ LockableMapTest::testFindBuckets4() { // ticket 3121525 void LockableMapTest::testFindBuckets5() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); @@ -739,7 +731,6 @@ LockableMapTest::testFindBuckets5() { // ticket 3121525 void LockableMapTest::testFindNoBuckets() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id(16, 0x0ffff); @@ -753,7 +744,6 @@ LockableMapTest::testFindNoBuckets() { void LockableMapTest::testFindAll() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0aaaa); // contains id2-id7 @@ -815,7 +805,6 @@ LockableMapTest::testFindAll() { void LockableMapTest::testFindAll2() { // Ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(17, 0x00001); @@ -839,7 +828,6 @@ LockableMapTest::testFindAll2() { // Ticket 3121525 void LockableMapTest::testFindAllUnusedBitIsSet() { // ticket 2938896 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(24, 0x000dc7089); @@ -868,7 +856,6 @@ LockableMapTest::testFindAllUnusedBitIsSet() { // ticket 2938896 void LockableMapTest::testFindAllInconsistentlySplit() { // Ticket 2938896 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 @@ -895,7 +882,6 @@ LockableMapTest::testFindAllInconsistentlySplit() { // Ticket 2938896 void LockableMapTest::testFindAllInconsistentlySplit2() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(17, 0x10000); @@ -923,7 +909,6 @@ LockableMapTest::testFindAllInconsistentlySplit2() { // ticket 3121525 void LockableMapTest::testFindAllInconsistentlySplit3() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); // contains id2 @@ -946,7 +931,6 @@ LockableMapTest::testFindAllInconsistentlySplit3() { // ticket 3121525 void LockableMapTest::testFindAllInconsistentlySplit4() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -972,7 +956,6 @@ LockableMapTest::testFindAllInconsistentlySplit4() { // ticket 3121525 void LockableMapTest::testFindAllInconsistentlySplit5() { // ticket 3121525 #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -997,7 +980,6 @@ LockableMapTest::testFindAllInconsistentlySplit5() { // ticket 3121525 void LockableMapTest::testFindAllInconsistentlySplit6() { - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -1022,7 +1004,6 @@ LockableMapTest::testFindAllInconsistentlySplit6() { void LockableMapTest::testFindAllInconsistentBelow16Bits() { - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(1, 0x1); // contains id2-id3 @@ -1048,7 +1029,6 @@ LockableMapTest::testFindAllInconsistentBelow16Bits() void LockableMapTest::testCreate() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(58, 0x43d6c878000004d2ull); @@ -1081,7 +1061,6 @@ LockableMapTest::testCreate() { void LockableMapTest::testCreate2() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(58, 0xeaf77782000004d2); @@ -1108,7 +1087,6 @@ LockableMapTest::testCreate2() { void LockableMapTest::testCreate3() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(58, 0xeaf77780000004d2); @@ -1137,7 +1115,6 @@ LockableMapTest::testCreate3() { void LockableMapTest::testCreate4() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(16, 0x00000000000004d1); @@ -1162,7 +1139,6 @@ LockableMapTest::testCreate4() { void LockableMapTest::testCreate6() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(0x8c000000000004d2); @@ -1199,7 +1175,6 @@ LockableMapTest::testCreate6() { void LockableMapTest::testCreate5() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(58, 0xeaf77780000004d2); @@ -1224,7 +1199,6 @@ LockableMapTest::testCreate5() { void LockableMapTest::testCreateEmpty() { #if __WORDSIZE == 64 - typedef LockableMap<JudyMultiMap<A> > Map; Map map; { document::BucketId id1(58, 0x00000000010004d2); @@ -1238,7 +1212,6 @@ LockableMapTest::testCreateEmpty() { void LockableMapTest::testIsConsistent() { - typedef LockableMap<JudyMultiMap<A> > Map; Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 document::BucketId id2(17, 0x00001); diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index 5551d0a5010..aa5fd80df91 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -209,7 +209,6 @@ PutOperationTest::testBucketDatabaseGetsSpecialEntryWhenCreateBucketSent() setupDistributor(2, 1, "storage:1 distributor:1"); Document::SP doc(createDummyDocument("test", "test")); - document::BucketId bucketId(getExternalOperationHandler().getBucketId(doc->getId())); sendPut(createPut(doc)); // Database updated before CreateBucket is sent diff --git a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp index 4656924c1b6..f9375790ebb 100644 --- a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp @@ -44,7 +44,6 @@ DeactivateBucketsTest::bucketsInDatabaseDeactivatedWhenNodeDownInClusterState() lib::ClusterState::CSP(new lib::ClusterState(upState))); document::BucketId bucket(8, 123); - spi::Bucket spiBucket(makeSpiBucket(bucket)); createBucket(bucket); api::BucketInfo serviceLayerInfo(1, 2, 3, 4, 5, true, true); diff --git a/storage/src/vespa/storage/bucketmover/htmltable.h b/storage/src/vespa/storage/bucketmover/htmltable.h index c032c15ee02..85e9438edcb 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.h +++ b/storage/src/vespa/storage/bucketmover/htmltable.h @@ -47,8 +47,7 @@ struct Column { } out << ">"; } - virtual void printElementStop(std::ostream& out, uint16_t row) { - std::map<uint16_t, Color>::iterator color(_colors.find(row)); + virtual void printElementStop(std::ostream& out, [[maybe_unused]] uint16_t row) { out << "</td>"; } diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h index eb7a6df2d17..914781eec30 100644 --- a/storage/src/vespa/storage/bucketmover/run.h +++ b/storage/src/vespa/storage/bucketmover/run.h @@ -38,7 +38,6 @@ class Run : public document::Printable { std::shared_ptr<const lib::Distribution> _distribution; lib::NodeState _nodeState; uint16_t _nodeIndex; - uint32_t _maxEntriesToKeep; std::list<Move> _entries; std::list<Move> _pending; bool _iterationDone; diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index 89c5000b542..ea67e7ea72a 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -191,13 +191,12 @@ private: public: NodeRemover(const lib::ClusterState& oldState, const lib::ClusterState& s, - const document::BucketIdFactory& factory, + [[maybe_unused]] const document::BucketIdFactory& factory, uint16_t localIndex, const lib::Distribution& distribution, const char* upStates) : _oldState(oldState), _state(s), - _factory(factory), _localIndex(localIndex), _distribution(distribution), _upStates(upStates) {} @@ -218,7 +217,6 @@ private: const lib::ClusterState _state; std::vector<document::BucketId> _removedBuckets; - const document::BucketIdFactory& _factory; uint16_t _localIndex; const lib::Distribution& _distribution; const char* _upStates; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index 4e2f8a3169a..8e6494a588d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -12,11 +12,10 @@ namespace storage { namespace distributor { StatBucketOperation::StatBucketOperation( - DistributorComponent& manager, + [[maybe_unused]] DistributorComponent& manager, DistributorBucketSpace &bucketSpace, const std::shared_ptr<api::StatBucketCommand> & cmd) : Operation(), - _manager(manager), _bucketSpace(bucketSpace), _command(cmd) { diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h index d40924e23f0..914e104943a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h @@ -30,7 +30,6 @@ public: void onStart(DistributorMessageSender& sender) override; void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) override; private: - DistributorComponent& _manager; DistributorBucketSpace &_bucketSpace; std::shared_ptr<api::StatBucketCommand> _command; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 3ccb35ef5a7..d7207f43c7b 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -23,10 +23,9 @@ using document::BucketSpace; namespace storage { FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, - const spi::PartitionStateList& partitions, + [[maybe_unused]] const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _partitions(partitions), - _component(compReg, "filestorhandlerimpl"), + : _component(compReg, "filestorhandlerimpl"), _diskInfo(), _messageSender(sender), _bucketIdFactory(_component.getBucketIdFactory()), diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index fe9982018b9..390b7284b85 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -282,7 +282,6 @@ public: void abortQueuedOperations(const AbortBucketOperationsCommand& cmd); private: - const spi::PartitionStateList& _partitions; ServiceLayerComponent _component; std::vector<Disk> _diskInfo; MessageSender& _messageSender; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index b6ac2a4b219..f9deb76c3e2 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -152,7 +152,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) toMsg->toString().c_str(), toMsg->getLoadType().getId(), fromMsg.getPriority(), toMsg->getPriority()); } - return std::move(toMsg); + return toMsg; } std::unique_ptr<api::StorageReply> @@ -195,7 +195,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentReply& fromReply, toMsg->setPriority(_priConverter->toStoragePriority(fromReply.getPriority())); } } - return std::move(toMsg); + return toMsg; } std::unique_ptr<mbus::Message> @@ -318,7 +318,7 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) toMsg->getTrace().setLevel(9); } } - return std::move(toMsg); + return toMsg; } void diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 60b94b6b33e..9e927a24d9c 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -1105,8 +1105,6 @@ Visitor::getStatus(std::ostream& out, bool verbose) const out << "</table>\n"; out << "<h4>Buckets to visit</h4>"; - typedef std::pair<api::Timestamp, api::Timestamp> TimePair; - TimePair lastTime; for (uint32_t i=0; i<_buckets.size(); ++i) { out << _buckets[i] << "\n<br>"; } diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index f8da49202ad..c8d34139364 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -105,7 +105,6 @@ public: uint32_t _secondPassHits; uint64_t _secondPassBytes; const document::OrderingSpecification* _ordering; - bool _allowFirstPass; }; enum VisitorState diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 5408c714eba..76feec9ffbb 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -230,7 +230,7 @@ auto readFile(const std::string & filename) { off_t read = file.read(&buf[0], buf.size(), 0); CPPUNIT_ASSERT_EQUAL(read, file.getFileSize()); - return std::move(buf); + return buf; } void diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index 5a834929981..52d523071e6 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -233,27 +233,26 @@ Distribution::getStorageSeed( uint32_t Distribution::getDiskSeed(const document::BucketId& bucket, uint16_t nodeIndex) const { - typedef vespa::config::content::StorDistributionConfig Config; switch (_diskDistribution) { - case Config::MODULO: + case DiskDistribution::MODULO: { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[16]); return 0xdeadbeef ^ seed; } - case Config::MODULO_INDEX: + case DiskDistribution::MODULO_INDEX: { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[16]); return 0xdeadbeef ^ seed ^ nodeIndex; } - case Config::MODULO_KNUTH: + case DiskDistribution::MODULO_KNUTH: { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[16]); return 0xdeadbeef ^ seed ^ (1664525L * nodeIndex + 1013904223L); } - case Config::MODULO_BID: + case DiskDistribution::MODULO_BID: { uint64_t currentid = bucket.withoutCountBits(); char ordered[8]; @@ -305,7 +304,7 @@ Distribution::getIdealDisk(const NodeState& nodeState, uint16_t nodeIndex, } RandomGen randomizer(getDiskSeed(bucket, nodeIndex)); switch (_diskDistribution) { - case vespa::config::content::StorDistributionConfig::MODULO_BID: + case DiskDistribution::MODULO_BID: { double maxScore = 0.0; uint16_t idealDisk = 0xffff; |