aboutsummaryrefslogtreecommitdiffstats
path: root/node-admin
diff options
context:
space:
mode:
authorValerij Fredriksen <freva@users.noreply.github.com>2023-11-04 09:25:36 +0100
committerGitHub <noreply@github.com>2023-11-04 09:25:36 +0100
commitbd42627870b4066b4b8085e17cf67cc7656468f0 (patch)
tree3ba27ec3212201516afe43745950c978c115999a /node-admin
parentf547fce384fb465dc04bbbb95063be69b6b89430 (diff)
Revert "Move node-admin"
Diffstat (limited to 'node-admin')
-rw-r--r--node-admin/.gitignore1
-rw-r--r--node-admin/CMakeLists.txt7
-rw-r--r--node-admin/OWNERS1
-rw-r--r--node-admin/README.md4
-rw-r--r--node-admin/pom.xml175
-rw-r--r--node-admin/src/main/application/services.xml12
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java166
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java34
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java111
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java111
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java92
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java68
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java47
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java43
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java48
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java35
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java78
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java273
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java32
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java10
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java43
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java72
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java70
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java22
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java96
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java16
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java97
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java29
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java236
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java86
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java54
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java192
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java115
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java103
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java38
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java10
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java880
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java23
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java406
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java48
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java114
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java25
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java50
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java279
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java147
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java55
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java36
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java16
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java143
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java32
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java12
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java29
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java36
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java85
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java70
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java36
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java59
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java29
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java153
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java135
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java94
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java168
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java139
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java25
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java66
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java164
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java50
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java28
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java76
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java30
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java24
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java139
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java16
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java196
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java138
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java61
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java49
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java132
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java338
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java106
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java59
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java20
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java48
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java433
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java29
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java55
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java37
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java109
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java35
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java103
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java40
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java24
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java143
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java47
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java269
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java27
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java22
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java143
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java83
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java41
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java56
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java261
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java180
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java12
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java42
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java42
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java15
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java43
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java66
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java283
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java124
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java20
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java10
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java633
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java25
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java31
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java57
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java67
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java52
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java50
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java18
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java116
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java96
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java356
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java58
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java52
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java53
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java72
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java30
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java175
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java117
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java59
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java52
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java125
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java71
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java135
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java37
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java32
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java35
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java34
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java272
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java55
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java83
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java107
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java92
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java33
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java40
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java21
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java70
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java67
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java53
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java46
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java42
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java58
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java350
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java58
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java81
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java98
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java348
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java224
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java130
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java142
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java21
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java85
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java85
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java16
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java139
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java47
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java15
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java382
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java92
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java15
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java36
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java10
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java106
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java10
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java14
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java52
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java113
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java67
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java18
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java21
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java227
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java91
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java32
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java69
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java61
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java14
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java11
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java35
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java69
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java114
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java76
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java42
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java18
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java13
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java156
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java60
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java37
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java165
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java38
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java30
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java85
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java305
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java245
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java178
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java30
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java5
-rwxr-xr-xnode-admin/src/main/sh/node-admin.sh112
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java162
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java19
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java194
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java151
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java40
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java182
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java26
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java249
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java72
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java73
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java172
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java54
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java39
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java256
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java52
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java70
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java49
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java147
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java37
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java184
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java99
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java52
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java182
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java58
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java91
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java44
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java50
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java178
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java351
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java88
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java96
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java234
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java300
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java103
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java129
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java58
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java31
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java319
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java134
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java58
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java166
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java277
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java103
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java152
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java889
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java29
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java28
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java68
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java148
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java59
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java26
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java122
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java39
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java20
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java62
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java27
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java238
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java73
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java60
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java79
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java62
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java87
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java52
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java39
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java199
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java211
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java120
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java41
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java32
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java78
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java69
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java147
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java190
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java88
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java149
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java52
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java218
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java194
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java335
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java80
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java39
-rw-r--r--node-admin/src/test/resources/default-env-example.txt5
-rw-r--r--node-admin/src/test/resources/default-env-rewritten.txt4
-rw-r--r--node-admin/src/test/resources/template1.tmp10
-rw-r--r--node-admin/src/test/resources/template2.tmp4
-rw-r--r--node-admin/src/test/resources/template3.tmp6
344 files changed, 29790 insertions, 0 deletions
diff --git a/node-admin/.gitignore b/node-admin/.gitignore
new file mode 100644
index 00000000000..adbb97d2d31
--- /dev/null
+++ b/node-admin/.gitignore
@@ -0,0 +1 @@
+data/ \ No newline at end of file
diff --git a/node-admin/CMakeLists.txt b/node-admin/CMakeLists.txt
new file mode 100644
index 00000000000..1056a09976a
--- /dev/null
+++ b/node-admin/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install(DIRECTORY DESTINATION logs/vespa/node-admin)
+install(FILES target/node-admin-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components)
+install_symlink(lib/jars/application-model-jar-with-dependencies.jar conf/node-admin-app/components/application-model-jar-with-dependencies.jar)
+install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/node-admin-app/components/flags-jar-with-dependencies.jar)
+install(FILES src/main/application/services.xml DESTINATION conf/node-admin-app)
+install(PROGRAMS src/main/sh/node-admin.sh DESTINATION libexec/vespa)
diff --git a/node-admin/OWNERS b/node-admin/OWNERS
new file mode 100644
index 00000000000..e131dacde49
--- /dev/null
+++ b/node-admin/OWNERS
@@ -0,0 +1 @@
+hakonhall
diff --git a/node-admin/README.md b/node-admin/README.md
new file mode 100644
index 00000000000..d366400ea7e
--- /dev/null
+++ b/node-admin/README.md
@@ -0,0 +1,4 @@
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# Node Admin
+
+Manages docker containers that run different applications on a host.
diff --git a/node-admin/pom.xml b/node-admin/pom.xml
new file mode 100644
index 00000000000..75adc2fb380
--- /dev/null
+++ b/node-admin/pom.xml
@@ -0,0 +1,175 @@
+<?xml version="1.0"?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>8-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>node-admin</artifactId>
+ <version>8-SNAPSHOT</version>
+ <packaging>container-plugin</packaging>
+ <name>${project.artifactId}</name>
+
+ <dependencies>
+ <!-- Provided -->
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-provisioning</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-dev</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespa-athenz</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>flags</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Compile -->
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>orchestrator-restapi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity-engine-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. -->
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>http-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <!-- Test -->
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application-model</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>orchestrator</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>node-repository</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>service-monitor</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>testutil</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Needed for node repo mock -->
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>zkfacade</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <attachBundleArtifact>true</attachBundleArtifact>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!-- Explicit for IntelliJ to detect correct language level from parent -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml
new file mode 100644
index 00000000000..d1ea4f2b2ff
--- /dev/null
+++ b/node-admin/src/main/application/services.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0" xmlns:preprocess="properties">
+ <container id="node-admin" version="1.0">
+ <!-- Please update container test when changing this file -->
+ <accesslog type="json" fileNamePattern="logs/vespa/node-admin/access-json.log.%Y%m%d%H%M%S" symlinkName="access-json.log" compressOnRotation="true" compressionType="zstd" bufferSize='262144' queueSize='1024'/>
+
+ <component id="metrics" class="com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics" bundle="node-admin"/>
+
+ <preprocess:include file="variant.xml" required="false"/>
+ </container>
+</services>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java
new file mode 100644
index 00000000000..034c7a381ed
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java
@@ -0,0 +1,166 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.logging.Logger;
+
+/**
+ * Represents a cgroup in the control group v2 hierarchy, see
+ * <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">Control Group v2</a>.
+ *
+ * @author hakonhall
+ */
+public class Cgroup {
+ private static final Logger logger = Logger.getLogger(Cgroup.class.getName());
+
+ private static final Map<String, Consumer<UnixPath>> cgroupDirectoryCallbacks = new HashMap<>();
+
+ private final Path root;
+ private final Path relativePath;
+
+ public static Cgroup root(FileSystem fileSystem) {
+ return new Cgroup(fileSystem.getPath("/sys/fs/cgroup"), fileSystem.getPath(""));
+ }
+
+ private Cgroup(Path root, Path relativePath) {
+ this.root = root.normalize();
+ this.relativePath = this.root.relativize(this.root.resolve(relativePath).normalize());
+ if (this.relativePath.toString().equals("..") || this.relativePath.toString().startsWith("../")) {
+ throw new IllegalArgumentException("Invalid cgroup relative path: " + relativePath);
+ }
+ }
+
+ /** Whether this cgroup actually exists in the kernel / on the file system. */
+ public boolean exists() { return unixPath().resolve("cgroup.controllers").exists(); }
+
+ /** Creates this cgroup if it does not already exist, and return this. */
+ public Cgroup create() {
+ if (unixPath().createDirectory()) {
+ // cgroup automatically creates various files in a newly created cgroup directory. A unit test may simulate
+ // this by registering consumers before the test is run.
+ Consumer<UnixPath> callback = cgroupDirectoryCallbacks.get(relativePath.toString());
+ if (callback != null)
+ callback.accept(unixPath());
+ }
+ return this;
+ }
+
+ /** Whether v2 cgroup is enabled on this host. */
+ public boolean v2CgroupIsEnabled() { return resolveRoot().exists(); }
+
+ /**
+ * Resolve the given path against the path of this cgroup, and return the resulting cgroup.
+ * If the given path is absolute, it is resolved against the root of the cgroup hierarchy.
+ */
+ public Cgroup resolve(String path) {
+ Path effectivePath = fileSystem().getPath(path);
+ if (effectivePath.isAbsolute()) {
+ return new Cgroup(root, fileSystem().getPath("/").relativize(effectivePath));
+ } else {
+ return new Cgroup(root, relativePath.resolve(path));
+ }
+ }
+
+ /** Returns the root cgroup, possibly this. */
+ public Cgroup resolveRoot() { return isRoot() ? this : new Cgroup(root, fileSystem().getPath("")); }
+
+ /** Returns the cgroup of a system service assuming this is the root, e.g. vespa-host-admin -> system.slice/vespa-host-admin.service. */
+ public Cgroup resolveSystemService(String name) { return resolve("system.slice").resolve(serviceNameOf(name)); }
+
+ /** Returns the root cgroup of the given Podman container. */
+ public Cgroup resolveContainer(ContainerId containerId) { return resolve("/machine.slice/libpod-" + containerId + ".scope/container"); }
+
+ /** Returns the root cgroup of the container, or otherwise the root cgroup. */
+ public Cgroup resolveRoot(Optional<ContainerId> containerId) { return containerId.map(this::resolveContainer).orElseGet(this::resolveRoot); }
+
+ /** Returns the absolute path to this cgroup. */
+ public Path path() { return root.resolve(relativePath); }
+
+ /** Returns the UnixPath of {@link #path()}. */
+ public UnixPath unixPath() { return new UnixPath(path()); }
+
+ public String read(String filename) {
+ return unixPath().resolve(filename).readUtf8File();
+ }
+
+ public Optional<String> readIfExists(String filename) {
+ return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip);
+ }
+
+ public List<String> readLines(String filename) {
+ return unixPath().resolve(filename).readUtf8File().lines().toList();
+ }
+
+ public Optional<Integer> readIntIfExists(String filename) {
+ return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip).map(Integer::parseInt);
+ }
+
+ public Size readSize(String filename) { return Size.from(read(filename).stripTrailing()); }
+
+ public boolean convergeFileContent(TaskContext context, String filename, String content, boolean apply) {
+ UnixPath path = unixPath().resolve(filename);
+ String currentContent = path.readUtf8File();
+ if (ensureSuffixNewline(currentContent).equals(ensureSuffixNewline(content))) return false;
+
+ if (apply) {
+ context.recordSystemModification(logger, "Updating " + path + " from '" + currentContent.stripTrailing() +
+ "' to '" + content.stripTrailing() + "'");
+ path.writeUtf8File(content);
+ }
+ return true;
+ }
+
+ /** The kernel appears to append a newline if none exist, when writing to files in cgroupfs. */
+ private static String ensureSuffixNewline(String content) {
+ return content.endsWith("\n") ? content : content + "\n";
+ }
+
+ /** Returns an instance representing core interface files (cgroup.* files). */
+ public CgroupCore core() { return new CgroupCore(this); }
+
+ /** Returns the CPU controller of this cgroup (cpu.* files). */
+ public CpuController cpu() { return new CpuController(this); }
+
+ /** Returns the memory controller of this cgroup (memory.* files). */
+ public MemoryController memory() { return new MemoryController(this); }
+
+ /** Returns the IO controller of this cgroup (io.* files). */
+ public IoController io() { return new IoController(this); }
+
+ /**
+ * Wraps {@code command} to ensure it is executed in this cgroup.
+ *
+ * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p>
+ */
+ public String[] wrapCommandForExecutionInCgroup(String... command) {
+ String[] fullCommand = new String[3 + command.length];
+ fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec";
+ fullCommand[1] = "-g";
+ fullCommand[2] = relativePath.toString();
+ System.arraycopy(command, 0, fullCommand, 3, command.length);
+ return fullCommand;
+ }
+
+ public static void unitTesting_atCgroupCreation(String relativePath, Consumer<UnixPath> callback) {
+ cgroupDirectoryCallbacks.put(relativePath, callback);
+ }
+
+ private boolean isRoot() { return relativePath.toString().isEmpty(); }
+
+ private static String serviceNameOf(String name) {
+ return name.indexOf('.') == -1 ? name + ".service" : name;
+ }
+
+ private FileSystem fileSystem() { return root.getFileSystem(); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java
new file mode 100644
index 00000000000..ecee819cc66
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java
@@ -0,0 +1,34 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import java.util.List;
+
+/**
+ * Utility methods for accessing the cgroup core interface files, i.e. all cgroup.* files.
+ *
+ * @author hakonhall
+ */
+public class CgroupCore {
+ private final Cgroup cgroup;
+
+ CgroupCore(Cgroup cgroup) { this.cgroup = cgroup; }
+
+ public List<Integer> getPidsInCgroup() {
+ return cgroup.readLines("cgroup.procs")
+ .stream()
+ .map(Integer::parseInt)
+ .toList();
+ }
+
+ /** Whether the given PID is a member of this cgroup. */
+ public boolean isMember(int pid) {
+ return getPidsInCgroup().contains(pid);
+ }
+
+ /** Move the given PID to this cgroup, but return false if it was already a member. */
+ public boolean addMember(int pid) {
+ if (isMember(pid)) return false;
+ cgroup.unixPath().resolve("cgroup.procs").writeUtf8File(Integer.toString(pid));
+ return true;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java
new file mode 100644
index 00000000000..5ca8a84cad6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java
@@ -0,0 +1,111 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static java.lang.Integer.parseInt;
+
+/**
+ * Represents a cgroup v2 CPU controller, i.e. all cpu.* files.
+ *
+ * @author hakonhall
+ */
+public class CpuController {
+ private final Cgroup cgroup;
+
+ CpuController(Cgroup cgroup) {
+ this.cgroup = cgroup;
+ }
+
+ /**
+ * The maximum bandwidth limit of the format "QUOTA PERIOD", which indicates that the cgroup may consume
+ * up to QUOTA in each PERIOD duration. A quota of "max" indicates no limit.
+ */
+ public record Max(Size quota, int period) {
+ public String toFileContent() { return quota + " " + period + '\n'; }
+ }
+
+ /**
+ * Returns the maximum CPU usage, or empty if cgroup is not found.
+ *
+ * @see Max
+ */
+ public Optional<Max> readMax() {
+ return cgroup.readIfExists("cpu.max")
+ .map(content -> {
+ String[] parts = content.strip().split(" ");
+ return new Max(Size.from(parts[0]), parseInt(parts[1]));
+ });
+ }
+
+ /**
+ * Update CPU quota and period for the given container ID. Set quota to -1 value for unlimited.
+ *
+ * @see #readMax()
+ * @see Max
+ */
+ public boolean updateMax(TaskContext context, int quota, int period) {
+ Max max = new Max(quota < 0 ? Size.max() : Size.from(quota), period);
+ return cgroup.convergeFileContent(context, "cpu.max", max.toFileContent(), true);
+ }
+
+ /** @return The weight in the range [1, 10000], or empty if not found. */
+ private Optional<Integer> readWeight() {
+ return cgroup.readIntIfExists("cpu.weight");
+ }
+
+ /** @return The number of shares allocated to this cgroup for purposes of CPU time scheduling, or empty if not found. */
+ public Optional<Integer> readShares() {
+ return readWeight().map(CpuController::weightToShares);
+ }
+
+ public boolean updateShares(TaskContext context, int shares) {
+ return cgroup.convergeFileContent(context, "cpu.weight", sharesToWeight(shares) + "\n", true);
+ }
+
+ // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061
+ // TODO: Migrate to weights
+ public static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); }
+ public static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); }
+
+ public enum StatField {
+ TOTAL_USAGE_USEC("usage_usec"),
+ USER_USAGE_USEC("user_usec"),
+ SYSTEM_USAGE_USEC("system_usec"),
+ TOTAL_PERIODS("nr_periods"),
+ THROTTLED_PERIODS("nr_throttled"),
+ THROTTLED_TIME_USEC("throttled_usec");
+
+ private final String name;
+
+ StatField(String name) {
+ this.name = name;
+ }
+
+ long parseValue(String value) {
+ return Long.parseLong(value);
+ }
+
+ static Optional<StatField> fromField(String fieldName) {
+ return Arrays.stream(values())
+ .filter(field -> fieldName.equals(field.name))
+ .findFirst();
+ }
+ }
+
+ public Map<StatField, Long> readStats() {
+ return cgroup.readLines("cpu.stat")
+ .stream()
+ .map(line -> line.split("\\s+"))
+ .filter(parts -> parts.length == 2)
+ .flatMap(parts -> StatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1]))))
+ .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java
new file mode 100644
index 00000000000..f6676347605
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java
@@ -0,0 +1,111 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import ai.vespa.validation.Validation;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.lang.Integer.parseInt;
+
+/**
+ * Represents a cgroup v2 IO controller, i.e. all io.* files.
+ *
+ * @author freva
+ */
+public class IoController {
+ private static final Logger logger = Logger.getLogger(IoController.class.getName());
+ private final Cgroup cgroup;
+
+ IoController(Cgroup cgroup) {
+ this.cgroup = cgroup;
+ }
+
+ public record Device(int major, int minor) implements Comparable<Device> {
+ public Device {
+ // https://www.halolinux.us/kernel-architecture/representation-of-major-and-minor-numbers.html
+ Validation.requireInRange(major, "device major", 0, 0xFFF);
+ Validation.requireInRange(minor, "device minor", 0, 0xFFFFF);
+ }
+
+ private String toFileContent() { return major + ":" + minor; }
+ private static Device fromString(String device) {
+ String[] parts = device.split(":");
+ return new Device(parseInt(parts[0]), parseInt(parts[1]));
+ }
+
+ @Override
+ public int compareTo(Device o) {
+ return major != o.major ? Integer.compare(major, o.major) : Integer.compare(minor, o.minor);
+ }
+ }
+
+ /**
+ * Defines max allowed IO:
+ * <ul>
+ * <li><b>rbps</b>: Read bytes per seconds</li>
+ * <li><b>riops</b>: Read IO operations per seconds</li>
+ * <li><b>wbps</b>: Write bytes per seconds</li>
+ * <li><b>wiops</b>: Write IO operations per seconds</li>
+ * </ul>.
+ */
+ public record Max(Size rbps, Size wbps, Size riops, Size wiops) {
+ public static Max UNLIMITED = new Max(Size.max(), Size.max(), Size.max(), Size.max());
+
+ // Keys can be specified in any order, this is the order they are outputted in from io.max
+ // https://github.com/torvalds/linux/blob/c1a515d3c0270628df8ae5f5118ba859b85464a2/block/blk-throttle.c#L1541
+ private String toFileContent() { return "rbps=%s wbps=%s riops=%s wiops=%s".formatted(rbps, wbps, riops, wiops); }
+
+ public static Max fromString(String max) {
+ String[] parts = max.split(" ");
+ Size rbps = Size.max(), riops = Size.max(), wbps = Size.max(), wiops = Size.max();
+ for (String part : parts) {
+ if (part.isEmpty()) continue;
+ String[] kv = part.split("=");
+ if (kv.length != 2) throw new IllegalArgumentException("Invalid io.max format: " + max);
+ switch (kv[0]) {
+ case "rbps" -> rbps = Size.from(kv[1]);
+ case "riops" -> riops = Size.from(kv[1]);
+ case "wbps" -> wbps = Size.from(kv[1]);
+ case "wiops" -> wiops = Size.from(kv[1]);
+ default -> throw new IllegalArgumentException("Unknown key " + kv[0]);
+ }
+ }
+ return new Max(rbps, wbps, riops, wiops);
+ }
+ }
+
+ /**
+ * Returns the maximum allowed IO usage, by device, or empty if cgroup is not found.
+ *
+ * @see Max
+ */
+ public Optional<Map<Device, Max>> readMax() {
+ return cgroup.readIfExists("io.max")
+ .map(content -> content
+ .lines()
+ .map(line -> {
+ String[] parts = line.strip().split(" ", 2);
+ return Map.entry(Device.fromString(parts[0]), Max.fromString(parts[1]));
+ })
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+ }
+
+ public boolean updateMax(TaskContext context, Device device, Max max) {
+ Max prevMax = readMax()
+ .map(maxByDevice -> maxByDevice.get(device))
+ .orElse(Max.UNLIMITED);
+ if (prevMax.equals(max)) return false;
+
+ UnixPath path = cgroup.unixPath().resolve("io.max");
+ context.recordSystemModification(logger, "Updating %s for device %s from '%s' to '%s'",
+ path, device.toFileContent(), prevMax.toFileContent(), max.toFileContent());
+ path.writeUtf8File(device.toFileContent() + ' ' + max.toFileContent() + '\n');
+ return true;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java
new file mode 100644
index 00000000000..28da683ea69
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java
@@ -0,0 +1,92 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+/**
+ * Represents a cgroup v2 memory controller, i.e. all memory.* files.
+ *
+ * @author hakonhall
+ */
+public class MemoryController {
+ private final Cgroup cgroup;
+
+ MemoryController(Cgroup cgroup) {
+ this.cgroup = cgroup;
+ }
+
+ /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */
+ public Size readMax() {
+ return cgroup.readSize("memory.max");
+ }
+
+ /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */
+ public Size readCurrent() {
+ return cgroup.readSize("memory.current");
+ }
+
+ /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */
+ public Optional<Size> readCurrentIfExists() {
+ return cgroup.readIfExists("memory.current").map(Size::from);
+ }
+
+ public Stats readStat() {
+ var lines = cgroup.readLines("memory.stat");
+ return new Stats(
+ Size.from(readField(lines, "file")), Size.from(readField(lines, "sock")), Size.from(readField(lines, "slab")),
+ Size.from(readField(lines, "slab_reclaimable")), Size.from(readField(lines, "anon")));
+ }
+
+ public Optional<Pressure> readPressureIfExists() {
+ return cgroup.readIfExists("memory.pressure")
+ .map(fileContent ->
+ new Pressure(
+ readPressureField(fileContent, "some"),
+ readPressureField(fileContent, "full")
+ )
+ );
+ }
+
+ private static String readField(List<String> lines, String fieldName) {
+ return lines.stream()
+ .map(line -> line.split("\\s+"))
+ .filter(fields -> fields.length == 2)
+ .filter(fields -> fieldName.equals(fields[0]))
+ .map(fields -> fields[1])
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("No such field: " + fieldName));
+ }
+
+ /**
+ * Fetches the avg60 value from the specified type, i.e. "some" or "full".
+ */
+ private static Double readPressureField(String fileContent, String type) {
+ var pattern = Pattern.compile(type + ".*avg60=(?<avg60>\\d+\\.\\d+).*");
+ return Stream.of(fileContent.split("\n"))
+ .map(pattern::matcher)
+ .filter(Matcher::matches)
+ .map(matcher -> matcher.group("avg60"))
+ .findFirst()
+ .map(Double::parseDouble)
+ .orElseThrow(() -> new IllegalArgumentException("No such field: " + type));
+ }
+
+ /**
+ * @param file Number of bytes used to cache filesystem data, including tmpfs and shared memory.
+ * @param sock Amount of memory used in network transmission buffers.
+ * @param slab Amount of memory used for storing in-kernel data structures.
+ * @param slabReclaimable Part of "slab" that might be reclaimed, such as dentries and inodes.
+ * @param anon Amount of memory used in anonymous mappings such as brk(), sbrk(), and mmap(MAP_ANONYMOUS).
+ */
+ public record Stats(Size file, Size sock, Size slab, Size slabReclaimable, Size anon) {}
+
+ /**
+ * @param some The avg60 value of the "some" pressure level.
+ * @param full The avg60 value of the "full" pressure level.
+ */
+ public record Pressure(double some, double full) {}
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java
new file mode 100644
index 00000000000..d89db56e4d2
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java
@@ -0,0 +1,68 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import java.util.Objects;
+
+/**
+ * Represents a number of bytes or possibly "max".
+ *
+ * @author hakonhall
+ */
+public class Size {
+ private static final String MAX = "max";
+ private static final Size MAX_SIZE = new Size(true, 0);
+
+ private final boolean max;
+ private final long value;
+
+ public static Size max() {
+ return MAX_SIZE;
+ }
+
+ public static Size from(long value) {
+ return new Size(false, value);
+ }
+
+ public static Size from(String value) {
+ return value.equals(MAX) ? MAX_SIZE : new Size(false, Long.parseLong(value));
+ }
+
+ private Size(boolean max, long value) {
+ this.max = max;
+ this.value = value;
+ }
+
+ public boolean isMax() {
+ return max;
+ }
+
+ /** Returns the value, i.e. the number of "bytes" if applicable. Throws if this is max. */
+ public long value() {
+ if (max) throw new IllegalStateException("Value is max");
+ return value;
+ }
+
+ public String toFileContent() { return toString() + '\n'; }
+
+ @Override
+ public String toString() { return max ? MAX : Long.toString(value); }
+
+ public boolean isGreaterThan(Size that) {
+ if (that.max) return false;
+ if (this.max) return true;
+ return this.value > that.value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Size size = (Size) o;
+ return max == size.max && value == size.value;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(max, value);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java
new file mode 100644
index 00000000000..b4c1a5228f8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java
@@ -0,0 +1,9 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @author hakonhall
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
new file mode 100644
index 00000000000..64c6b19b8bb
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
@@ -0,0 +1,47 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Information necessary to e.g. establish communication with the config servers
+ *
+ * @author hakon
+ */
+public class ConfigServerInfo {
+ private final URI loadBalancerEndpoint;
+ private final AthenzIdentity configServerIdentity;
+ private final Function<String, URI> configServerHostnameToUriMapper;
+ private final List<URI> configServerURIs;
+
+ public ConfigServerInfo(URI loadBalancerEndpoint, List<String> configServerHostNames,
+ AthenzIdentity configServerAthenzIdentity) {
+ this.loadBalancerEndpoint = loadBalancerEndpoint;
+ this.configServerIdentity = configServerAthenzIdentity;
+ this.configServerHostnameToUriMapper = hostname -> URI.create("https://" + hostname + ":4443");
+ this.configServerURIs = configServerHostNames.stream()
+ .map(configServerHostnameToUriMapper)
+ .toList();
+ }
+
+ public List<URI> getConfigServerUris() {
+ return configServerURIs;
+ }
+
+ public URI getConfigServerUri(String hostname) {
+ return configServerHostnameToUriMapper.apply(hostname);
+ }
+
+ public URI getLoadBalancerEndpoint() {
+ return loadBalancerEndpoint;
+ }
+
+ public AthenzIdentity getConfigServerIdentity() {
+ return configServerIdentity;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java
new file mode 100644
index 00000000000..492020b7ae4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java
@@ -0,0 +1,43 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.component;
+
+/**
+ * <p>This class is thread unsafe: All method calls MUST be exclusive and serialized.</p>
+ *
+ * <dl>
+ * <dt>In a specialized environment it is possible to provide a richer context than TaskContext:</dt>
+ * <dd>- Define a subclass T of TaskContext with the additional functionality.</dd>
+ * <dd>- Define task classes that implement IdempotentTask&lt;T&gt;.</dd>
+ * </dl>
+ */
+public interface IdempotentTask<T extends TaskContext> {
+ /**
+ * <p>A short id of the task to e.g. identify the task in the log.</p>
+ *
+ * <p>Prefer PascalCase and without white-space.</p>
+ *
+ * <p>Example: "EnableDocker"</p>
+ */
+ default String name() { return getClass().getSimpleName(); }
+
+ /**
+ * <p>Execute an administrative task to converge towards some ideal state, whether it is
+ * system state or in-memory Java state.</p>
+ *
+ * <p>converge() must be idempotent: it may be called any number of times, or
+ * interrupted at any time e.g. by `kill -9`.</p>
+ *
+ * <p>converge() is not thread safe: The caller must ensure there is at most one invocation
+ * of converge() at any given time.</p>
+ *
+ * @return false if already converged, i.e. was a no-op. A typical sequence of converge()
+ * calls on a IdempotentTask will consist of:
+ * - Any number of calls that throws an exception due to some issues. Assuming
+ * no exceptions were thrown, or the issue eventually resolved itself...
+ * (convergence failure)
+ * - Returns true once (converged just now)
+ * - Returns false for all further calls (already converged)
+ * @throws RuntimeException (or a subclass) if the task is unable to converge.
+ */
+ boolean converge(T context);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
new file mode 100644
index 00000000000..0e8fdb6e1f6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
@@ -0,0 +1,48 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public interface TaskContext {
+ /**
+ * Record a system modification. IdempotentTask is supposed to converge the system (files,
+ * directory permission, iptable rules, etc) to some wanted state. It is especially important
+ * to produce a truthful log of system changes to understand what may or may not be going on.
+ *
+ * All tasks should:
+ * 1. Record any and all modifications to the system
+ * 2. Avoid recording system interactions that does not actually change the system.
+ * 3. Record system modifications as early as possible and preferably before they are
+ * performed (sometimes this is not possible).
+ *
+ * @param logger Used to log the modification to help locate the source of the modification.
+ * @param message Description of the modification, e.g. "Changing owner of /foo from alice
+ * to bob".
+ */
+ void recordSystemModification(Logger logger, String message);
+ default void recordSystemModification(Logger logger, String messageFormat, Object... args) {
+ recordSystemModification(logger, String.format(messageFormat, args));
+ }
+
+ /**
+ * Log message at Level.INFO, scoped to denote the current task. The message may
+ * also be directed to status pages or similar.
+ *
+ * Please do not call this too many times as that spams the log. Typically a task may call
+ * this zero times, or up to a few times.
+ *
+ * Do not log a message that is also recorded with recordSystemModification.
+ */
+ default void log(Logger logger, String message) {
+ log(logger, Level.INFO, message);
+ }
+
+ default void log(Logger logger, String messageFormat, Object... args) {
+ log(logger, String.format(messageFormat, args));
+ }
+
+ void log(Logger logger, Level level, String message);
+
+ void log(Logger logger, Level level, String message, Throwable throwable);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java
new file mode 100644
index 00000000000..beedb56941a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java
@@ -0,0 +1,35 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class TestTaskContext implements TaskContext {
+ private final List<String> systemModifications = new ArrayList<>();
+
+ @Override
+ public void recordSystemModification(Logger logger, String description) {
+ systemModifications.add(description);
+ }
+
+ @Override
+ public void log(Logger logger, Level level, String message) {
+ logger.log(level, message);
+ }
+
+ @Override
+ public void log(Logger logger, Level level, String message, Throwable throwable) {
+ logger.log(level, message, throwable);
+ }
+
+ public List<String> getSystemModificationLog() {
+ return systemModifications;
+ }
+
+ public void clearSystemModificationLog() {
+ systemModifications.clear();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java
new file mode 100644
index 00000000000..53cb32300b4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java
new file mode 100644
index 00000000000..b401e2f3d08
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java
@@ -0,0 +1,78 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * Interface to execute basic HTTP/HTTPS request against config server(s)
+ *
+ * @author freva
+ */
+public interface ConfigServerApi extends AutoCloseable {
+
+ /**
+ * The result of sending a request to a config server results in a jackson response or exception. If a response
+ * is returned, an instance of this interface is conferred to discard the result and try the next config server,
+ * unless it was the last attempt.
+ *
+ * @param <T> the type of the returned jackson response
+ */
+ interface RetryPolicy<T> {
+ boolean tryNextConfigServer(URI configServerEndpoint, T response);
+ }
+
+ class Params<T> {
+ private Optional<Duration> connectionTimeout = Optional.empty();
+
+ private RetryPolicy<T> retryPolicy = (configServerEndpoint, response) -> false;
+
+ public Params() {}
+
+ /** Set the socket connect and read timeouts. */
+ public Params<T> setConnectionTimeout(Duration connectionTimeout) {
+ this.connectionTimeout = Optional.of(connectionTimeout);
+ return this;
+ }
+
+ public Optional<Duration> getConnectionTimeout() { return connectionTimeout; }
+
+ /** Set the retry policy to use against the config servers. */
+ public Params<T> setRetryPolicy(RetryPolicy<T> retryPolicy) {
+ this.retryPolicy = retryPolicy;
+ return this;
+ }
+
+ public RetryPolicy<T> getRetryPolicy() { return retryPolicy; }
+ }
+
+ <T> T get(String path, Class<T> wantedReturnType, Params<T> params);
+ default <T> T get(String path, Class<T> wantedReturnType) {
+ return get(path, wantedReturnType, new Params<>());
+ }
+
+ <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params);
+ default <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) {
+ return post(path, bodyJsonPojo, wantedReturnType, new Params<>());
+ }
+
+ <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params);
+ default <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) {
+ return put(path, bodyJsonPojo, wantedReturnType, new Params<>());
+ }
+
+ <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params);
+ default <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) {
+ return patch(path, bodyJsonPojo, wantedReturnType, new Params<>());
+ }
+
+ <T> T delete(String path, Class<T> wantedReturnType, Params<T> params);
+ default <T> T delete(String path, Class<T> wantedReturnType) {
+ return delete(path, wantedReturnType, new Params<>());
+ }
+
+ /** Close the underlying HTTP client and any threads this class might have started. */
+ @Override
+ void close();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
new file mode 100644
index 00000000000..b645e993a05
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
@@ -0,0 +1,273 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import ai.vespa.util.http.hc4.SslConnectionSocketFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory;
+import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo;
+import com.yahoo.yolean.Exceptions;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPatch;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
+
+import javax.net.ssl.HostnameVerifier;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Retries request on config server a few times before giving up. Assumes that all requests should be sent with
+ * content-type application/json
+ *
+ * @author dybdahl
+ * @author bjorncs
+ */
+public class ConfigServerApiImpl implements ConfigServerApi {
+
+ private static final Logger logger = Logger.getLogger(ConfigServerApiImpl.class.getName());
+ private static final RequestConfig DEFAULT_REQUEST_CONFIG = RequestConfig.custom()
+ .setConnectionRequestTimeout(1_000) // connection from connection manager
+ .setConnectTimeout(10_000) // establishment of connection
+ .setSocketTimeout(10_000) // waiting for data
+ .build();
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ private final List<URI> configServers;
+
+ private final CloseableHttpClient client;
+
+ public static ConfigServerApiImpl create(ConfigServerInfo info,
+ ServiceIdentityProvider provider,
+ HostnameVerifier hostnameVerifier) {
+ return new ConfigServerApiImpl(
+ info.getConfigServerUris(),
+ hostnameVerifier,
+ provider);
+ }
+
+ public static ConfigServerApiImpl createFor(URI uri,
+ ServiceIdentityProvider provider,
+ HostnameVerifier hostnameVerifier) {
+ return new ConfigServerApiImpl(List.of(uri), hostnameVerifier, provider);
+ }
+
+ private ConfigServerApiImpl(Collection<URI> configServers,
+ HostnameVerifier verifier,
+ ServiceIdentityProvider identityProvider) {
+ this(configServers, createClient(SslConnectionSocketFactory.of(new ServiceIdentitySslSocketFactory(identityProvider), verifier)));
+ }
+
+ private ConfigServerApiImpl(Collection<URI> configServers, CloseableHttpClient client) {
+ this.configServers = randomizeConfigServerUris(configServers);
+ this.client = client;
+ }
+
+ public static ConfigServerApiImpl createForTesting(List<URI> configServerHosts) {
+ return new ConfigServerApiImpl(configServerHosts, createClient(SslConnectionSocketFactory.of()));
+ }
+
+ static ConfigServerApiImpl createForTestingWithClient(List<URI> configServerHosts,
+ CloseableHttpClient client) {
+ return new ConfigServerApiImpl(configServerHosts, client);
+ }
+
+ interface CreateRequest {
+ HttpUriRequest createRequest(URI configServerUri) throws JsonProcessingException, UnsupportedEncodingException;
+ }
+
+ private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType, Params<T> params) {
+ T lastResult = null;
+ Exception lastException = null;
+
+ for (URI configServer : configServers) {
+ var request = Exceptions.uncheck(() -> requestFactory.createRequest(configServer));
+ try (CloseableHttpResponse response = client.execute(request)) {
+ var responseBody = EntityUtils.toString(response.getEntity());
+ HttpException.handleStatusCode(response.getStatusLine().getStatusCode(),
+ request.getMethod() + " " + request.getURI() +
+ " failed with response '" + responseBody + "'");
+
+ T result;
+ try {
+ result = mapper.readValue(responseBody, wantedReturnType);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed parse response from config server", e);
+ }
+
+ if (params.getRetryPolicy().tryNextConfigServer(configServer, result)) {
+ lastResult = result;
+ lastException = null;
+ } else {
+ return result;
+ }
+ } catch (HttpException e) {
+ if (!e.isRetryable()) throw e;
+ lastResult = null;
+ lastException = e;
+ } catch (Exception e) {
+ lastResult = null;
+ lastException = e;
+ if (configServers.size() == 1) break;
+
+ // Failure to communicate with a config server is not abnormal during upgrades
+ if (ConnectionException.isKnownConnectionException(e)) {
+ logger.info("Failed to connect to " + configServer + ", will try next: " + e.getMessage());
+ } else {
+ logger.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage());
+ }
+ }
+ }
+
+ if (lastResult != null) {
+ logger.warning("Giving up after trying all config servers: returning result: " + lastResult);
+ return lastResult;
+ }
+
+ String prefix = configServers.size() == 1 ?
+ "Request against " + configServers.get(0) + " failed: " :
+ "All requests against the config servers (" + configServers + ") failed, last as follows: ";
+ throw ConnectionException.handleException(prefix, lastException);
+ }
+
+ @Override
+ public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) {
+ Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params);
+ return tryAllConfigServers(configServer -> {
+ HttpPut put = new HttpPut(configServer.resolve(path));
+ requestConfigOverride.ifPresent(put::setConfig);
+ setContentTypeToApplicationJson(put);
+ if (bodyJsonPojo.isPresent()) {
+ put.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo.get())));
+ }
+ return put;
+ }, wantedReturnType, params);
+ }
+
+ @Override
+ public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) {
+ Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params);
+ return tryAllConfigServers(configServer -> {
+ HttpPatch patch = new HttpPatch(configServer.resolve(path));
+ requestConfigOverride.ifPresent(patch::setConfig);
+ setContentTypeToApplicationJson(patch);
+ patch.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo)));
+ return patch;
+ }, wantedReturnType, params);
+ }
+
+ @Override
+ public <T> T delete(String path, Class<T> wantedReturnType, Params<T> params) {
+ Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params);
+ return tryAllConfigServers(configServer -> {
+ HttpDelete delete = new HttpDelete(configServer.resolve(path));
+ requestConfigOverride.ifPresent(delete::setConfig);
+ return delete;
+ }, wantedReturnType, params);
+ }
+
+ @Override
+ public <T> T get(String path, Class<T> wantedReturnType, Params<T> params) {
+ Optional<RequestConfig> requestConfig = getRequestConfigOverride(params);
+ return tryAllConfigServers(configServer -> {
+ HttpGet get = new HttpGet(configServer.resolve(path));
+ requestConfig.ifPresent(get::setConfig);
+ return get;
+ }, wantedReturnType, params);
+ }
+
+ @Override
+ public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) {
+ Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params);
+ return tryAllConfigServers(configServer -> {
+ HttpPost post = new HttpPost(configServer.resolve(path));
+ requestConfigOverride.ifPresent(post::setConfig);
+ setContentTypeToApplicationJson(post);
+ post.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo)));
+ return post;
+ }, wantedReturnType, params);
+ }
+
+ @Override
+ public void close() {
+ // Need to do try and catch, using e.g. uncheck(client::close) might fail because
+ // components are deconstructed in random order and if the bundle containing uncheck has been
+ // unloaded it will fail with NoClassDefFoundError
+ try {
+ client.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private void setContentTypeToApplicationJson(HttpRequestBase request) {
+ request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
+ }
+
+ private static CloseableHttpClient createClient(SSLConnectionSocketFactory socketFactory) {
+ Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", socketFactory)
+ .build();
+
+ PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough
+
+ // Have experienced hang in socket read, which may have been because of
+ // system defaults, therefore set explicit timeouts.
+ return HttpClientBuilder.create()
+ .setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG)
+ .disableAutomaticRetries()
+ .disableConnectionState() // Share connections between subsequent requests.
+ .setUserAgent("node-admin") // Node-repository depends on this value to identify agent of node-admin/host-admin requests
+ .setConnectionManager(cm)
+ .build();
+ }
+
+ private static <T> Optional<RequestConfig> getRequestConfigOverride(Params<T> params) {
+ if (params.getConnectionTimeout().isEmpty()) return Optional.empty();
+
+ RequestConfig.Builder builder = RequestConfig.copy(DEFAULT_REQUEST_CONFIG);
+
+ params.getConnectionTimeout().ifPresent(connectionTimeout -> {
+ builder.setConnectTimeout((int) connectionTimeout.toMillis());
+ builder.setSocketTimeout((int) connectionTimeout.toMillis());
+ });
+
+ return Optional.of(builder.build());
+ }
+
+ // Shuffle config server URIs to balance load
+ private static List<URI> randomizeConfigServerUris(Collection<URI> configServerUris) {
+ List<URI> shuffledConfigServerHosts = new ArrayList<>(configServerUris);
+ Collections.shuffle(shuffledConfigServerHosts);
+ return shuffledConfigServerHosts;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java
new file mode 100644
index 00000000000..8c6212f83f4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java
@@ -0,0 +1,32 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.yahoo.vespa.flags.FlagRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.State;
+
+/**
+ * The available (and implemented) APIs of the config server
+ *
+ * @author freva
+ */
+public interface ConfigServerClients {
+ /** Get handle to /nodes/v2/ REST API */
+ NodeRepository nodeRepository();
+
+ /** Get handle to /orchestrator/v1/ REST API */
+ Orchestrator orchestrator();
+
+ /** Get handle to the /state/v1 REST API */
+ State state();
+
+ /** Get handle to the /flags/v1 REST API */
+ FlagRepository flagRepository();
+
+ /** Get handle to the /cores/v1 REST API */
+ Cores cores();
+
+ void stop();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java
new file mode 100644
index 00000000000..e957a56c0ae
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java
@@ -0,0 +1,10 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+/**
+ * @author hakonhall
+ */
+public class ConfigServerException extends RuntimeException {
+ public ConfigServerException(String message) { super(message); }
+ public ConfigServerException(String message, Throwable cause) { super(message, cause); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java
new file mode 100644
index 00000000000..86c52efe282
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java
@@ -0,0 +1,43 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import org.apache.http.NoHttpResponseException;
+
+import java.io.EOFException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+
+/**
+ * @author freva
+ */
+@SuppressWarnings("serial")
+public class ConnectionException extends ConvergenceException {
+
+ private ConnectionException(String message, Throwable cause) {
+ super(message, cause, true);
+ }
+
+ /**
+ * Returns {@link ConnectionException} if the given Throwable is of a known and well understood error or
+ * a RuntimeException with the given exception as cause otherwise.
+ */
+ public static RuntimeException handleException(String prefix, Throwable t) {
+ if (isKnownConnectionException(t))
+ return new ConnectionException(prefix + t.getMessage(), t);
+
+ return new RuntimeException(prefix, t);
+ }
+
+ static boolean isKnownConnectionException(Throwable t) {
+ for (; t != null; t = t.getCause()) {
+ if (t instanceof SocketException ||
+ t instanceof SocketTimeoutException ||
+ t instanceof NoHttpResponseException ||
+ t instanceof EOFException)
+ return true;
+ }
+
+ return false;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java
new file mode 100644
index 00000000000..64b1ebe239d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java
@@ -0,0 +1,72 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+
+import javax.ws.rs.core.Response;
+
+/**
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class HttpException extends ConvergenceException {
+
+ private final boolean isRetryable;
+
+ private HttpException(int statusCode, String message, boolean isRetryable) {
+ super("HTTP status code " + statusCode + ": " + message, null, !isRetryable);
+ this.isRetryable = isRetryable;
+ }
+
+ private HttpException(Response.Status status, String message, boolean isRetryable) {
+ super(status.toString() + " (" + status.getStatusCode() + "): " + message, null, !isRetryable);
+ this.isRetryable = isRetryable;
+ }
+
+ boolean isRetryable() {
+ return isRetryable;
+ }
+
+ /**
+ * Returns on success.
+ * @throws HttpException for all non-expected status codes.
+ */
+ static void handleStatusCode(int statusCode, String message) {
+ Response.Status status = Response.Status.fromStatusCode(statusCode);
+ if (status == null) {
+ throw new HttpException(statusCode, message, true);
+ }
+
+ switch (status.getFamily()) {
+ case SUCCESSFUL: return;
+ case CLIENT_ERROR:
+ switch (status) {
+ case FORBIDDEN:
+ throw new ForbiddenException(message);
+ case NOT_FOUND:
+ throw new NotFoundException(message);
+ case CONFLICT:
+ // A response body is assumed to be present, and
+ // will later be interpreted as an error.
+ return;
+ }
+ throw new HttpException(status, message, false);
+ }
+
+ // Other errors like server-side errors are assumed to be NOT retryable,
+ // in case retries would put additional load on a bogged down server.
+ throw new HttpException(status, message, false);
+ }
+
+ public static class NotFoundException extends HttpException {
+ public NotFoundException(String message) {
+ super(Response.Status.NOT_FOUND, message, false);
+ }
+ }
+
+ public static class ForbiddenException extends HttpException {
+ public ForbiddenException(String message) {
+ super(Response.Status.FORBIDDEN, message, false);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java
new file mode 100644
index 00000000000..8ee346246ae
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java
@@ -0,0 +1,70 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.yahoo.vespa.flags.FlagRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoresImpl;
+import com.yahoo.vespa.hosted.node.admin.configserver.flags.RealFlagRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorImpl;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.State;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.StateImpl;
+
+/**
+ * {@link ConfigServerClients} using the default implementation for the various clients,
+ * and backed by a {@link ConfigServerApi}.
+ *
+ * @author freva
+ */
+public class RealConfigServerClients implements ConfigServerClients {
+ private final ConfigServerApi configServerApi;
+ private final NodeRepository nodeRepository;
+ private final Orchestrator orchestrator;
+ private final State state;
+ private final RealFlagRepository flagRepository;
+ private final Cores cores;
+
+ /**
+ * @param configServerApi the backend API to use - will be closed at {@link #stop()}.
+ */
+ public RealConfigServerClients(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ nodeRepository = new RealNodeRepository(configServerApi);
+ orchestrator = new OrchestratorImpl(configServerApi);
+ state = new StateImpl(configServerApi);
+ flagRepository = new RealFlagRepository(configServerApi);
+ cores = new CoresImpl(configServerApi);
+ }
+
+ @Override
+ public NodeRepository nodeRepository() {
+ return nodeRepository;
+ }
+
+ @Override
+ public Orchestrator orchestrator() {
+ return orchestrator;
+ }
+
+ @Override
+ public State state() {
+ return state;
+ }
+
+ @Override
+ public FlagRepository flagRepository() {
+ return flagRepository;
+ }
+
+ @Override
+ public Cores cores() {
+ return cores;
+ }
+
+ @Override
+ public void stop() {
+ configServerApi.close();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java
new file mode 100644
index 00000000000..c967091ccbf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java
@@ -0,0 +1,22 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Strings;
+
+/**
+ * @author hakonhall
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class StandardConfigServerResponse {
+ @JsonProperty("message") public String message;
+ @JsonProperty("error-code") public String errorCode;
+
+ public void throwOnError(String detail) {
+ if (!Strings.isNullOrEmpty(errorCode))
+ throw new ConfigServerException(detail + ": " + message + " " + errorCode);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
new file mode 100644
index 00000000000..2f4595ce5d1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
@@ -0,0 +1,96 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.cores;
+
+import com.yahoo.config.provision.DockerImage;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public class CoreDumpMetadata {
+ public enum Type { CORE_DUMP, JVM_HEAP, OOM }
+
+ private Type type;
+ private String binPath;
+ private Instant created;
+ private List<String> backtrace;
+ private List<String> backtraceAllThreads;
+ private Path coreDumpPath;
+ private String decryptionToken;
+ private String kernelVersion;
+ private String cpuMicrocodeVersion;
+ private DockerImage dockerImage;
+ private String vespaVersion;
+
+ public CoreDumpMetadata() {}
+
+ public Optional<Type> type() { return Optional.ofNullable(type); }
+ public Optional<String> binPath() { return Optional.ofNullable(binPath); }
+ public Optional<Instant> created() { return Optional.ofNullable(created); }
+ public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }
+ public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }
+ public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }
+ public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); }
+ public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }
+ public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }
+ public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }
+ public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); }
+
+ public CoreDumpMetadata setType(Type type) { this.type = type; return this; }
+ public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; }
+ public CoreDumpMetadata setCreated(Instant created) { this.created = created; return this; }
+ public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }
+ public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }
+ public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }
+ public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; }
+ public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }
+ public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }
+ public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }
+ public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; }
+
+ @Override
+ public String toString() {
+ return "CoreDumpMetadata{" +
+ "type=" + type +
+ ", binPath=" + binPath +
+ ", created=" + created +
+ ", backtrace=" + backtrace +
+ ", backtraceAllThreads=" + backtraceAllThreads +
+ ", coreDumpPath=" + coreDumpPath +
+ ", decryptionToken=" + decryptionToken +
+ ", kernelVersion='" + kernelVersion + '\'' +
+ ", cpuMicrocodeVersion='" + cpuMicrocodeVersion + '\'' +
+ ", dockerImage=" + dockerImage +
+ ", vespaVersion=" + vespaVersion +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CoreDumpMetadata metadata = (CoreDumpMetadata) o;
+ return type == metadata.type &&
+ Objects.equals(binPath, metadata.binPath) &&
+ Objects.equals(created, metadata.created) &&
+ Objects.equals(backtrace, metadata.backtrace) &&
+ Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) &&
+ Objects.equals(coreDumpPath, metadata.coreDumpPath) &&
+ Objects.equals(decryptionToken, metadata.decryptionToken) &&
+ Objects.equals(kernelVersion, metadata.kernelVersion) &&
+ Objects.equals(cpuMicrocodeVersion, metadata.cpuMicrocodeVersion) &&
+ Objects.equals(dockerImage, metadata.dockerImage) &&
+ Objects.equals(vespaVersion, metadata.vespaVersion);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, binPath, created, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion,
+ cpuMicrocodeVersion, dockerImage, vespaVersion);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java
new file mode 100644
index 00000000000..b168c6f6dbe
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java
@@ -0,0 +1,16 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.cores;
+
+import com.yahoo.config.provision.HostName;
+
+/**
+ * @author hakonhall
+ */
+public interface Cores {
+ /**
+ * @param hostname Hostname of the node that produced the core.
+ * @param id The ID (aka UUID aka docid) of the core.
+ * @param metadata Core dump metadata.
+ */
+ void report(HostName hostname, String id, CoreDumpMetadata metadata);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java
new file mode 100644
index 00000000000..200fe97283c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.cores;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
+
+/**
+ * @author hakonhall
+ */
+public class CoresImpl implements Cores {
+ private final ConfigServerApi configServerApi;
+
+ public CoresImpl(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ }
+
+ @Override
+ public void report(HostName hostname, String id, CoreDumpMetadata metadata) {
+ var request = new ReportCoreDumpRequest().fillFrom(metadata);
+ String uriPath = "/cores/v1/report/" + hostname.value() + "/" + id;
+ configServerApi.post(uriPath, request, StandardConfigServerResponse.class)
+ .throwOnError("Failed to report core dump at " + metadata.coredumpPath());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
new file mode 100644
index 00000000000..435367cd1ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
@@ -0,0 +1,97 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Jackson class of JSON request, with names of fields verified in unit test.
+ *
+ * @author hakonhall
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ReportCoreDumpRequest {
+ private static final ObjectMapper objectMapper = new ObjectMapper();
+
+ public List<String> backtrace;
+ public List<String> backtrace_all_threads;
+ public Long created;
+ public String type;
+ public String bin_path;
+ public String coredump_path;
+ public String cpu_microcode_version;
+ public String decryption_token;
+ public String docker_image;
+ public String kernel_version;
+ public String vespa_version;
+
+ public ReportCoreDumpRequest() {}
+
+ /** Fill this from metadata and return this. */
+ @JsonIgnore
+ public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) {
+ metadata.type().ifPresent(type -> this.type = type.name());
+ metadata.binPath().ifPresent(binPath -> this.bin_path = binPath);
+ metadata.created().ifPresent(created -> this.created = created.toEpochMilli());
+ metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace));
+ metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads));
+ metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString());
+ metadata.decryptionToken().ifPresent(decryptionToken -> this.decryption_token = decryptionToken);
+ metadata.kernelVersion().ifPresent(kernelVersion -> this.kernel_version = kernelVersion);
+ metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> this.cpu_microcode_version = cpuMicrocodeVersion);
+ metadata.dockerImage().ifPresent(dockerImage -> this.docker_image = dockerImage.asString());
+ metadata.vespaVersion().ifPresent(vespaVersion -> this.vespa_version = vespaVersion);
+ return this;
+ }
+
+ @JsonIgnore
+ public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) {
+ if (type != null) metadata.setType(CoreDumpMetadata.Type.valueOf(type));
+ if (bin_path != null) metadata.setBinPath(bin_path);
+ if (created != null) metadata.setCreated(Instant.ofEpochMilli(created));
+ if (backtrace != null) metadata.setBacktrace(backtrace);
+ if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads);
+ if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path));
+ if (decryption_token != null) metadata.setDecryptionToken(decryption_token);
+ if (kernel_version != null) metadata.setKernelVersion(kernel_version);
+ if (cpu_microcode_version != null) metadata.setCpuMicrocodeVersion(cpu_microcode_version);
+ if (docker_image != null) metadata.setDockerImage(DockerImage.fromString(docker_image));
+ if (vespa_version != null) metadata.setVespaVersion(vespa_version);
+ }
+
+ @JsonIgnore
+ public void save(Path path) {
+ String serialized = uncheck(() -> objectMapper.writeValueAsString(this));
+ uncheck(() -> Files.writeString(path, serialized));
+ }
+
+ @JsonIgnore
+ public static Optional<ReportCoreDumpRequest> load(Path path) {
+ final String serialized;
+ try {
+ serialized = Files.readString(path);
+ } catch (NoSuchFileException e) {
+ return Optional.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return Optional.of(uncheck(() -> objectMapper.readValue(serialized, ReportCoreDumpRequest.class)));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java
new file mode 100644
index 00000000000..d8a07b2b0df
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java
@@ -0,0 +1,9 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @author hakonhall
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.cores;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java
new file mode 100644
index 00000000000..97c93e6a48a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java
@@ -0,0 +1,29 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.flags;
+
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.FlagRepository;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.flags.json.wire.WireFlagDataList;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * @author hakonhall
+ */
+public class RealFlagRepository implements FlagRepository {
+ private final ConfigServerApi configServerApi;
+
+ public RealFlagRepository(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ }
+
+ @Override
+ public Map<FlagId, FlagData> getAllFlagData() {
+ WireFlagDataList list = configServerApi.get("/flags/v1/data?recursive=true", WireFlagDataList.class);
+ return FlagData.listFromWire(list).stream().collect(Collectors.toMap(FlagData::id, Function.identity()));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java
new file mode 100644
index 00000000000..b5f1bc2a3bc
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.flags;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java
new file mode 100644
index 00000000000..dd13658ba27
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java
@@ -0,0 +1,236 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.google.common.net.InetAddresses;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * This class represents an ACL for a specific container instance.
+ *
+ * @author mpolden
+ * @author smorgrav
+ */
+public class Acl {
+
+ public static final Acl EMPTY = new Acl(Set.of(), Set.of(), Set.of(), Set.of());
+
+ private final Set<Node> trustedNodes;
+ private final Set<Integer> trustedPorts;
+ private final Set<Integer> trustedUdpPorts;
+ private final Set<String> trustedNetworks;
+
+ /**
+ * @param trustedPorts TCP Ports to trust
+ * @param trustedUdpPorts UDP ports to trust
+ * @param trustedNodes Nodes to trust
+ * @param trustedNetworks Networks (in CIDR notation) to trust
+ */
+ public Acl(Set<Integer> trustedPorts, Set<Integer> trustedUdpPorts, Set<Node> trustedNodes, Set<String> trustedNetworks) {
+ this.trustedNodes = copyOfNullable(trustedNodes);
+ this.trustedPorts = copyOfNullable(trustedPorts);
+ this.trustedUdpPorts = copyOfNullable(trustedUdpPorts);
+ this.trustedNetworks = copyOfNullable(trustedNetworks);
+ }
+
+ public Acl(Set<Integer> trustedPorts, Set<Node> trustedNodes) {
+ this(trustedPorts, Set.of(), trustedNodes, Set.of());
+ }
+
+ public List<String> toRules(IPVersion ipVersion) {
+ List<String> rules = new LinkedList<>();
+
+ // We reject with rules instead of using policies
+ rules.add("-P INPUT ACCEPT");
+ rules.add("-P FORWARD ACCEPT");
+ rules.add("-P OUTPUT ACCEPT");
+
+ // Allow packets belonging to established connections
+ rules.add( "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT");
+
+ // Allow any loopback traffic
+ rules.add("-A INPUT -i lo -j ACCEPT");
+
+ // Allow ICMP packets. See http://shouldiblockicmp.com/
+ rules.add("-A INPUT -p " + ipVersion.icmpProtocol() + " -j ACCEPT");
+
+ // Allow trusted ports if any
+ if (!trustedPorts.isEmpty()) {
+ rules.add("-A INPUT -p tcp -m multiport --dports " + joinPorts(trustedPorts) + " -j ACCEPT");
+ }
+
+ // Allow trusted UDP ports if any
+ if (!trustedUdpPorts.isEmpty()) {
+ rules.add("-A INPUT -p udp -m multiport --dports " + joinPorts(trustedUdpPorts) + " -j ACCEPT");
+ }
+
+ // Allow traffic from trusted nodes, limited to specific ports, if any
+ getTrustedNodes(ipVersion).stream()
+ .map(node -> {
+ StringBuilder rule = new StringBuilder();
+ rule.append("-A INPUT -s ")
+ .append(node.inetAddressString())
+ .append(ipVersion.singleHostCidr());
+ if (!node.ports.isEmpty()) {
+ rule.append(" -p tcp -m multiport --dports ")
+ .append(joinPorts(node.ports()));
+ }
+ rule.append(" -j ACCEPT");
+ return rule.toString();
+ })
+ .sorted()
+ .forEach(rules::add);
+
+ // Allow traffic from trusted networks
+ addressesOf(ipVersion, trustedNetworks).stream()
+ .map(network -> "-A INPUT -s " + network + " -j ACCEPT")
+ .sorted()
+ .forEach(rules::add);
+
+ // We reject instead of dropping to give us an easier time to figure out potential network issues
+ rules.add("-A INPUT -j REJECT --reject-with " + ipVersion.icmpPortUnreachable());
+
+ return Collections.unmodifiableList(rules);
+ }
+
+ private static String joinPorts(Collection<Integer> ports) {
+ return ports.stream().sorted().map(String::valueOf).collect(Collectors.joining(","));
+ }
+
+ public Set<Node> getTrustedNodes() {
+ return trustedNodes;
+ }
+
+ public Set<Node> getTrustedNodes(IPVersion ipVersion) {
+ return trustedNodes.stream()
+ .filter(node -> ipVersion.match(node.inetAddress()))
+ .collect(Collectors.toSet());
+ }
+
+ public Set<Integer> getTrustedPorts() {
+ return trustedPorts;
+ }
+
+ public Set<Integer> getTrustedUdpPorts() {
+ return trustedUdpPorts;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Acl acl = (Acl) o;
+ return trustedNodes.equals(acl.trustedNodes) &&
+ trustedPorts.equals(acl.trustedPorts) &&
+ trustedUdpPorts.equals(acl.trustedUdpPorts) &&
+ trustedNetworks.equals(acl.trustedNetworks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(trustedNodes, trustedPorts, trustedUdpPorts, trustedNetworks);
+ }
+
+ @Override
+ public String toString() {
+ return "Acl{" +
+ "trustedNodes=" + trustedNodes +
+ ", trustedPorts=" + trustedPorts +
+ ", trustedUdpPorts=" + trustedUdpPorts +
+ ", trustedNetworks=" + trustedNetworks +
+ '}';
+ }
+
+ private static Set<String> addressesOf(IPVersion version, Set<String> addresses) {
+ return addresses.stream()
+ .filter(version::match)
+ .collect(Collectors.toUnmodifiableSet());
+ }
+
+ private static <T> Set<T> copyOfNullable(Set<T> set) {
+ return Optional.ofNullable(set).map(Set::copyOf).orElseGet(Set::of);
+ }
+
+ public record Node(String hostname, InetAddress inetAddress, Set<Integer> ports) {
+
+ public Node(String hostname, String ipAddress, Set<Integer> ports) {
+ this(hostname, InetAddresses.forString(ipAddress), ports);
+ }
+
+ public String inetAddressString() {
+ return InetAddresses.toAddrString(inetAddress);
+ }
+
+ @Override
+ public String toString() {
+ return "Node{" +
+ "hostname='" + hostname + '\'' +
+ ", inetAddress=" + inetAddress +
+ ", ports=" + ports +
+ '}';
+ }
+ }
+
+ public static class Builder {
+
+ private final Set<Node> trustedNodes = new HashSet<>();
+ private final Set<Integer> trustedPorts = new HashSet<>();
+ private final Set<Integer> trustedUdpPorts = new HashSet<>();
+ private final Set<String> trustedNetworks = new HashSet<>();
+
+ public Builder() { }
+
+ public Builder(Acl acl) {
+ trustedNodes.addAll(acl.trustedNodes);
+ trustedPorts.addAll(acl.trustedPorts);
+ trustedNetworks.addAll(acl.trustedNetworks);
+ }
+
+ public Builder withTrustedNode(Node node) {
+ trustedNodes.add(node);
+ return this;
+ }
+
+ public Builder withTrustedNode(String hostname, String ipAddress) {
+ return withTrustedNode(hostname, ipAddress, Set.of());
+ }
+
+ public Builder withTrustedNode(String hostname, String ipAddress, Set<Integer> ports) {
+ return withTrustedNode(new Node(hostname, ipAddress, ports));
+ }
+
+ public Builder withTrustedNode(String hostname, InetAddress inetAddress, Set<Integer> ports) {
+ return withTrustedNode(new Node(hostname, inetAddress, ports));
+ }
+
+ public Builder withTrustedPorts(Integer... ports) {
+ trustedPorts.addAll(List.of(ports));
+ return this;
+ }
+
+ public Builder withTrustedUdpPorts(Integer... ports) {
+ trustedUdpPorts.addAll(List.of(ports));
+ return this;
+ }
+
+ public Builder withTrustedNetworks(Set<String> networks) {
+ trustedNetworks.addAll(networks);
+ return this;
+ }
+
+ public Acl build() {
+ return new Acl(trustedPorts, trustedUdpPorts, trustedNodes, trustedNetworks);
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
new file mode 100644
index 00000000000..47b59414efd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
@@ -0,0 +1,86 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.host.FlavorOverrides;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * @author freva
+ */
+public class AddNode {
+
+ public final String hostname;
+ public final String id;
+ public final Optional<String> parentHostname;
+ public final Optional<String> nodeFlavor;
+ public final Optional<FlavorOverrides> flavorOverrides;
+ public final Optional<NodeResources> nodeResources;
+ public final NodeType nodeType;
+ public final Set<String> ipAddresses;
+ public final Set<String> additionalIpAddresses;
+
+ public static AddNode forHost(String hostname, String id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
+ return new AddNode(hostname, id, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses);
+ }
+
+ public static AddNode forNode(String hostname, String id, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) {
+ return new AddNode(hostname, id, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of());
+ }
+
+ private AddNode(String hostname, String id, Optional<String> parentHostname,
+ Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides,
+ Optional<NodeResources> nodeResources,
+ NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
+ this.hostname = hostname;
+ this.id = id;
+ this.parentHostname = parentHostname;
+ this.nodeFlavor = nodeFlavor;
+ this.flavorOverrides = flavorOverrides;
+ this.nodeResources = nodeResources;
+ this.nodeType = nodeType;
+ this.ipAddresses = ipAddresses;
+ this.additionalIpAddresses = additionalIpAddresses;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AddNode addNode = (AddNode) o;
+ return Objects.equals(hostname, addNode.hostname) &&
+ Objects.equals(id, addNode.id) &&
+ Objects.equals(parentHostname, addNode.parentHostname) &&
+ Objects.equals(nodeFlavor, addNode.nodeFlavor) &&
+ Objects.equals(flavorOverrides, addNode.flavorOverrides) &&
+ Objects.equals(nodeResources, addNode.nodeResources) &&
+ nodeType == addNode.nodeType &&
+ Objects.equals(ipAddresses, addNode.ipAddresses) &&
+ Objects.equals(additionalIpAddresses, addNode.additionalIpAddresses);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(hostname, id, parentHostname, nodeFlavor, flavorOverrides, nodeResources, nodeType, ipAddresses, additionalIpAddresses);
+ }
+
+ @Override
+ public String toString() {
+ return "AddNode{" +
+ "hostname='" + hostname + '\'' +
+ ", id=" + id +
+ ", parentHostname=" + parentHostname +
+ ", nodeFlavor='" + nodeFlavor + '\'' +
+ ", flavorOverrides='" + flavorOverrides + '\'' +
+ ", nodeResources='" + nodeResources + '\'' +
+ ", nodeType=" + nodeType +
+ ", ipAddresses=" + ipAddresses +
+ ", additionalIpAddresses=" + additionalIpAddresses +
+ '}';
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java
new file mode 100644
index 00000000000..554e9f4df13
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java
@@ -0,0 +1,54 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class Event {
+ private final String agent;
+ private final String type;
+ private final Instant at;
+
+ public Event(String agent, String type, Instant at) {
+ this.agent = Objects.requireNonNull(agent);
+ this.type = Objects.requireNonNull(type);
+ this.at = Objects.requireNonNull(at);
+ }
+
+ public String agent() {
+ return agent;
+ }
+
+ public String type() {
+ return type;
+ }
+
+ public Instant at() {
+ return at;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Event event1 = (Event) o;
+ return agent.equals(event1.agent) && type.equals(event1.type) && at.equals(event1.at);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(agent, type, at);
+ }
+
+ @Override
+ public String toString() {
+ return "Event{" +
+ "agent='" + agent + '\'' +
+ ", type='" + type + '\'' +
+ ", at=" + at +
+ '}';
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java
new file mode 100644
index 00000000000..4c77019f9ba
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+public class NoSuchNodeException extends NodeRepositoryException {
+ public NoSuchNodeException(String message) {
+ super(message);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
new file mode 100644
index 00000000000..9b22de3f279
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
@@ -0,0 +1,192 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A node in the node repository is modified by setting which attributes to modify in this class,
+ * and then patching the node repository node through {@link NodeRepository#updateNodeAttributes(String, NodeAttributes)}.
+ *
+ * @author Haakon Dybdahl
+ * @author Valerij Fredriksen
+ */
+public class NodeAttributes {
+
+ private Optional<String> hostId = Optional.empty();
+ private Optional<Long> restartGeneration = Optional.empty();
+ private Optional<Long> rebootGeneration = Optional.empty();
+ private Optional<DockerImage> dockerImage = Optional.empty();
+ private Optional<Version> vespaVersion = Optional.empty();
+ private Optional<Version> currentOsVersion = Optional.empty();
+ private Optional<Instant> currentFirmwareCheck = Optional.empty();
+ private List<TrustStoreItem> trustStore = List.of();
+ private Optional<WireguardKey> wireguardPubkey = Optional.empty();
+ /** The list of reports to patch. A null value is used to remove the report. */
+ private Map<String, JsonNode> reports = new TreeMap<>();
+
+ public NodeAttributes() { }
+
+ public NodeAttributes withHostId(String hostId) {
+ this.hostId = Optional.of(hostId);
+ return this;
+ }
+
+ public NodeAttributes withRestartGeneration(Optional<Long> restartGeneration) {
+ this.restartGeneration = restartGeneration;
+ return this;
+ }
+
+ public NodeAttributes withRestartGeneration(long restartGeneration) {
+ return withRestartGeneration(Optional.of(restartGeneration));
+ }
+
+ public NodeAttributes withRebootGeneration(long rebootGeneration) {
+ this.rebootGeneration = Optional.of(rebootGeneration);
+ return this;
+ }
+
+ public NodeAttributes withDockerImage(DockerImage dockerImage) {
+ this.dockerImage = Optional.of(dockerImage);
+ return this;
+ }
+
+ public NodeAttributes withVespaVersion(Version vespaVersion) {
+ this.vespaVersion = Optional.of(vespaVersion);
+ return this;
+ }
+
+ public NodeAttributes withCurrentOsVersion(Version currentOsVersion) {
+ this.currentOsVersion = Optional.of(currentOsVersion);
+ return this;
+ }
+
+ public NodeAttributes withCurrentFirmwareCheck(Instant currentFirmwareCheck) {
+ this.currentFirmwareCheck = Optional.of(currentFirmwareCheck);
+ return this;
+ }
+
+ public NodeAttributes withTrustStore(List<TrustStoreItem> trustStore) {
+ this.trustStore = List.copyOf(trustStore);
+ return this;
+ }
+
+ public NodeAttributes withWireguardPubkey(WireguardKey wireguardPubkey) {
+ this.wireguardPubkey = Optional.of(wireguardPubkey);
+ return this;
+ }
+
+ public NodeAttributes withReports(Map<String, JsonNode> nodeReports) {
+ this.reports = new TreeMap<>(nodeReports);
+ return this;
+ }
+
+ public NodeAttributes withReport(String reportId, JsonNode jsonNode) {
+ reports.put(reportId, jsonNode);
+ return this;
+ }
+
+ public NodeAttributes withReportRemoved(String reportId) {
+ reports.put(reportId, null);
+ return this;
+ }
+
+ public Optional<String> getHostId() {
+ return hostId;
+ }
+
+ public Optional<Long> getRestartGeneration() {
+ return restartGeneration;
+ }
+
+ public Optional<Long> getRebootGeneration() {
+ return rebootGeneration;
+ }
+
+ public Optional<DockerImage> getDockerImage() {
+ return dockerImage;
+ }
+
+ public Optional<Version> getVespaVersion() {
+ return vespaVersion;
+ }
+
+ public Optional<Version> getCurrentOsVersion() {
+ return currentOsVersion;
+ }
+
+ public Optional<Instant> getCurrentFirmwareCheck() {
+ return currentFirmwareCheck;
+ }
+
+ public List<TrustStoreItem> getTrustStore() {
+ return trustStore;
+ }
+
+ public Optional<WireguardKey> getWireguardPubkey() { return wireguardPubkey; }
+
+ public Map<String, JsonNode> getReports() {
+ return reports;
+ }
+
+ public <T extends BaseReport> Optional<T> getReport(String reportId, Class<T> classInstance) {
+ return Optional.ofNullable(reports.get(reportId)).map(jn -> BaseReport.fromJsonNode(jn, classInstance));
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(hostId, restartGeneration, rebootGeneration, dockerImage, vespaVersion, currentOsVersion,
+ currentFirmwareCheck, trustStore, wireguardPubkey, reports);
+ }
+
+ public boolean isEmpty() {
+ return equals(new NodeAttributes());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof NodeAttributes other)) {
+ return false;
+ }
+
+ return Objects.equals(hostId, other.hostId)
+ && Objects.equals(restartGeneration, other.restartGeneration)
+ && Objects.equals(rebootGeneration, other.rebootGeneration)
+ && Objects.equals(dockerImage, other.dockerImage)
+ && Objects.equals(vespaVersion, other.vespaVersion)
+ && Objects.equals(currentOsVersion, other.currentOsVersion)
+ && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck)
+ && Objects.equals(trustStore, other.trustStore)
+ && Objects.equals(wireguardPubkey, other.wireguardPubkey)
+ && Objects.equals(reports, other.reports);
+ }
+
+ @Override
+ public String toString() {
+ return Stream.of(hostId.map(id -> "hostId=" + id),
+ restartGeneration.map(gen -> "restartGeneration=" + gen),
+ rebootGeneration.map(gen -> "rebootGeneration=" + gen),
+ dockerImage.map(img -> "dockerImage=" + img.asString()),
+ vespaVersion.map(ver -> "vespaVersion=" + ver.toFullString()),
+ currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()),
+ currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at),
+ Optional.ofNullable(trustStore.isEmpty() ? null : "trustStore=" + trustStore),
+ Optional.ofNullable(wireguardPubkey.isEmpty() ? null : "wireguardPubkey=" + wireguardPubkey),
+ Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java
new file mode 100644
index 00000000000..c70eccfa0ea
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java
@@ -0,0 +1,115 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class NodeMembership {
+ private final ClusterType clusterType;
+ private final String clusterId;
+ private final String group;
+ private final int index;
+ private final boolean retired;
+
+ public NodeMembership(String clusterType, String clusterId, String group, int index, boolean retired) {
+ this.clusterType = new ClusterType(clusterType);
+ this.clusterId = clusterId;
+ this.group = group;
+ this.index = index;
+ this.retired = retired;
+ }
+
+ public ClusterType type() {
+ return clusterType;
+ }
+
+ public String clusterId() {
+ return clusterId;
+ }
+
+ public String group() {
+ return group;
+ }
+
+ public int index() { return index; }
+
+ public boolean isRetired() {
+ return retired;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ NodeMembership that = (NodeMembership) o;
+
+ if (index != that.index) return false;
+ if (retired != that.retired) return false;
+ if (!clusterType.equals(that.clusterType)) return false;
+ if (!clusterId.equals(that.clusterId)) return false;
+ return group.equals(that.group);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = clusterType.hashCode();
+ result = 31 * result + clusterId.hashCode();
+ result = 31 * result + group.hashCode();
+ result = 31 * result + index;
+ result = 31 * result + (retired ? 1 : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "Membership {" +
+ " clusterType = " + clusterType +
+ " clusterId = " + clusterId +
+ " group = " + group +
+ " index = " + index +
+ " retired = " + retired +
+ " }";
+ }
+
+ public static class ClusterType {
+ private final String type;
+
+ private ClusterType(String type) {
+ this.type = Objects.requireNonNull(type);
+ }
+
+ public boolean isAdmin() { return "admin".equals(type); }
+ public boolean isContent() { return "content".equals(type); }
+ public boolean isCombined() { return "combined".equals(type); }
+ public boolean isContainer() { return "container".equals(type); }
+ public boolean hasContainer() { return isContainer() || isCombined(); }
+ public boolean hasContent() { return isContent() || isCombined(); }
+
+ public String value() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return type;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ClusterType that = (ClusterType) o;
+ return type.equals(that.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return type.hashCode();
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java
new file mode 100644
index 00000000000..c45c2dd9578
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java
@@ -0,0 +1,103 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * API of node reports within node-admin.
+ *
+ * @author hakonhall
+ */
+public class NodeReports {
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ private final Map<String, JsonNode> reports = new TreeMap<>();
+
+ public NodeReports() { }
+
+ public NodeReports(NodeReports reports) {
+ this.reports.putAll(reports.reports);
+ }
+
+ private NodeReports(Map<String, JsonNode> reports) {
+ this.reports.putAll(Objects.requireNonNull(reports));
+ }
+
+ public static NodeReports fromMap(Map<String, JsonNode> reports) {
+ return new NodeReports(reports);
+ }
+
+ public void setReport(String reportId, JsonNode jsonNode) {
+ reports.put(reportId, jsonNode);
+ }
+
+ public boolean hasReport(String reportId) { return reports.containsKey(reportId); }
+
+ public <T> Optional<T> getReport(String reportId, Class<T> jacksonClass) {
+ return Optional.ofNullable(reports.get(reportId)).map(r -> uncheck(() -> mapper.treeToValue(r, jacksonClass)));
+ }
+
+ /** Gets all reports of the given types and deserialize with the given jacksonClass. */
+ public <T> TreeMap<String, T> getReports(Class<T> jacksonClass, BaseReport.Type... types) {
+ Set<BaseReport.Type> typeSet = Set.of(types);
+
+ return reports.entrySet().stream()
+ .filter(entry -> {
+ JsonNode reportType = entry.getValue().findValue(BaseReport.TYPE_FIELD);
+ if (reportType == null || !reportType.isTextual()) return false;
+ Optional<BaseReport.Type> type = BaseReport.Type.deserialize(reportType.asText());
+ return type.map(typeSet::contains).orElse(false);
+ })
+ .collect(Collectors.toMap(
+ entry -> entry.getKey(),
+ entry -> uncheck(() -> mapper.treeToValue(entry.getValue(), jacksonClass)),
+ (x,y) -> x, // resolves key collisions - cannot happen.
+ TreeMap::new
+ ));
+ }
+
+ public void removeReport(String reportId) {
+ if (reports.containsKey(reportId)) {
+ reports.put(reportId, null);
+ }
+ }
+
+ public Map<String, JsonNode> getRawMap() {
+ return new TreeMap<>(reports);
+ }
+
+ /** Apply the override to this. null value means removing report. */
+ public void updateFromRawMap(Map<String, JsonNode> override) {
+ override.forEach((reportId, jsonNode) -> {
+ if (jsonNode == null) {
+ reports.remove(reportId);
+ } else {
+ reports.put(reportId, jsonNode);
+ }
+ });
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ NodeReports that = (NodeReports) o;
+ return Objects.equals(reports, that.reports);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(reports);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java
new file mode 100644
index 00000000000..ac1f8ec059f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java
@@ -0,0 +1,38 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author stiankri
+ */
+public interface NodeRepository {
+
+ void addNodes(List<AddNode> nodes);
+
+ List<NodeSpec> getNodes(String baseHostName);
+
+ default NodeSpec getNode(String hostName) {
+ return getOptionalNode(hostName).orElseThrow(() -> new NoSuchNodeException(hostName + " not found in node-repo"));
+ }
+
+ Optional<NodeSpec> getOptionalNode(String hostName);
+
+ Map<String, Acl> getAcls(String hostname);
+
+ List<WireguardPeer> getExclavePeers();
+
+ List<WireguardPeer> getConfigserverPeers();
+
+ void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes);
+
+ void setNodeState(String hostName, NodeState nodeState);
+
+ default void reboot(String hostname) {
+ throw new UnsupportedOperationException("Rebooting not supported in " + getClass().getName());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java
new file mode 100644
index 00000000000..f46f0c9f446
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java
@@ -0,0 +1,10 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+
+public class NodeRepositoryException extends ConvergenceException {
+ public NodeRepositoryException(String message) {
+ super(message, null, true);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
new file mode 100644
index 00000000000..3700b57d169
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
@@ -0,0 +1,880 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudAccount;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.config.provision.WireguardKeyWithTimestamp;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
+
+/**
+ * @author stiankri
+ */
+public class NodeSpec {
+
+ private final String hostname;
+ private final String id;
+ private final NodeState state;
+ private final NodeType type;
+ private final CloudAccount cloudAccount;
+ private final String flavor;
+
+ private final Optional<DockerImage> wantedDockerImage;
+ private final Optional<DockerImage> currentDockerImage;
+
+ private final Optional<Version> wantedVespaVersion;
+ private final Optional<Version> currentVespaVersion;
+
+ private final Optional<Version> wantedOsVersion;
+ private final Optional<Version> currentOsVersion;
+
+ private final Optional<Long> wantedRestartGeneration;
+ private final Optional<Long> currentRestartGeneration;
+
+ private final long wantedRebootGeneration;
+ private final long currentRebootGeneration;
+
+ private final Optional<Instant> wantedFirmwareCheck;
+ private final Optional<Instant> currentFirmwareCheck;
+
+ private final Optional<String> modelName;
+
+ private final OrchestratorStatus orchestratorStatus;
+ private final Optional<ApplicationId> owner;
+ private final Optional<NodeMembership> membership;
+
+ private final NodeResources resources;
+ private final NodeResources realResources;
+ private final Set<String> ipAddresses;
+ private final Set<String> additionalIpAddresses;
+
+ private final NodeReports reports;
+ private final List<Event> events;
+
+ private final Optional<String> parentHostname;
+ private final Optional<URI> archiveUri;
+
+ private final Optional<ApplicationId> exclusiveTo;
+
+ private final List<TrustStoreItem> trustStore;
+
+ private final Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp;
+
+ private final boolean wantToRebuild;
+
+ public NodeSpec(
+ String hostname,
+ String id,
+ Optional<DockerImage> wantedDockerImage,
+ Optional<DockerImage> currentDockerImage,
+ NodeState state,
+ NodeType type,
+ CloudAccount cloudAccount,
+ String flavor,
+ Optional<Version> wantedVespaVersion,
+ Optional<Version> currentVespaVersion,
+ Optional<Version> wantedOsVersion,
+ Optional<Version> currentOsVersion,
+ OrchestratorStatus orchestratorStatus,
+ Optional<ApplicationId> owner,
+ Optional<NodeMembership> membership,
+ Optional<Long> wantedRestartGeneration,
+ Optional<Long> currentRestartGeneration,
+ long wantedRebootGeneration,
+ long currentRebootGeneration,
+ Optional<Instant> wantedFirmwareCheck,
+ Optional<Instant> currentFirmwareCheck,
+ Optional<String> modelName,
+ NodeResources resources,
+ NodeResources realResources,
+ Set<String> ipAddresses,
+ Set<String> additionalIpAddresses,
+ NodeReports reports,
+ List<Event> events,
+ Optional<String> parentHostname,
+ Optional<URI> archiveUri,
+ Optional<ApplicationId> exclusiveTo,
+ List<TrustStoreItem> trustStore,
+ Optional<WireguardKeyWithTimestamp> wireguardPubkey,
+ boolean wantToRebuild) {
+
+ if (state == NodeState.active) {
+ requireOptional(owner, "owner");
+ requireOptional(membership, "membership");
+ requireOptional(wantedVespaVersion, "wantedVespaVersion");
+ requireOptional(wantedDockerImage, "wantedDockerImage");
+ requireOptional(wantedRestartGeneration, "restartGeneration");
+ requireOptional(currentRestartGeneration, "currentRestartGeneration");
+ }
+
+ this.hostname = Objects.requireNonNull(hostname);
+ this.id = Objects.requireNonNull(id);
+ this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage);
+ this.currentDockerImage = Objects.requireNonNull(currentDockerImage);
+ this.state = Objects.requireNonNull(state);
+ this.type = Objects.requireNonNull(type);
+ this.cloudAccount = Objects.requireNonNull(cloudAccount);
+ this.flavor = Objects.requireNonNull(flavor);
+ this.modelName = Objects.requireNonNull(modelName);
+ this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion);
+ this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion);
+ this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion);
+ this.currentOsVersion = Objects.requireNonNull(currentOsVersion);
+ this.orchestratorStatus = Objects.requireNonNull(orchestratorStatus);
+ this.owner = Objects.requireNonNull(owner);
+ this.membership = Objects.requireNonNull(membership);
+ this.wantedRestartGeneration = wantedRestartGeneration;
+ this.currentRestartGeneration = currentRestartGeneration;
+ this.wantedRebootGeneration = wantedRebootGeneration;
+ this.currentRebootGeneration = currentRebootGeneration;
+ this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck);
+ this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck);
+ this.resources = Objects.requireNonNull(resources);
+ this.realResources = Objects.requireNonNull(realResources);
+ this.ipAddresses = Set.copyOf(ipAddresses);
+ this.additionalIpAddresses = Set.copyOf(additionalIpAddresses);
+ this.reports = Objects.requireNonNull(reports);
+ this.events = List.copyOf(events);
+ this.parentHostname = Objects.requireNonNull(parentHostname);
+ this.archiveUri = Objects.requireNonNull(archiveUri);
+ this.exclusiveTo = Objects.requireNonNull(exclusiveTo);
+ this.trustStore = Objects.requireNonNull(trustStore);
+ this.wireguardKeyWithTimestamp = Objects.requireNonNull(wireguardPubkey);
+ this.wantToRebuild = wantToRebuild;
+ }
+
+ public String hostname() {
+ return hostname;
+ }
+
+ /** Returns unique node ID */
+ public String id() {
+ return id;
+ }
+
+ public NodeState state() {
+ return state;
+ }
+
+ public NodeType type() {
+ return type;
+ }
+
+ public CloudAccount cloudAccount() {
+ return cloudAccount;
+ }
+
+ public String flavor() {
+ return flavor;
+ }
+
+ public Optional<DockerImage> wantedDockerImage() {
+ return wantedDockerImage;
+ }
+
+ public Optional<DockerImage> currentDockerImage() {
+ return currentDockerImage;
+ }
+
+ public Optional<Version> wantedVespaVersion() {
+ return wantedVespaVersion;
+ }
+
+ public Optional<Version> currentVespaVersion() {
+ return currentVespaVersion;
+ }
+
+ public Optional<Version> currentOsVersion() {
+ return currentOsVersion;
+ }
+
+ public Optional<Version> wantedOsVersion() {
+ return wantedOsVersion;
+ }
+
+ public Optional<Long> wantedRestartGeneration() {
+ return wantedRestartGeneration;
+ }
+
+ public Optional<Long> currentRestartGeneration() {
+ return currentRestartGeneration;
+ }
+
+ public long wantedRebootGeneration() {
+ return wantedRebootGeneration;
+ }
+
+ public long currentRebootGeneration() {
+ return currentRebootGeneration;
+ }
+
+ public Optional<Instant> wantedFirmwareCheck() {
+ return wantedFirmwareCheck;
+ }
+
+ public Optional<Instant> currentFirmwareCheck() {
+ return currentFirmwareCheck;
+ }
+
+ public Optional<String> modelName() {
+ return modelName;
+ }
+
+ public OrchestratorStatus orchestratorStatus() {
+ return orchestratorStatus;
+ }
+
+ public Optional<ApplicationId> owner() {
+ return owner;
+ }
+
+ public Optional<NodeMembership> membership() {
+ return membership;
+ }
+
+ public NodeResources resources() {
+ return resources;
+ }
+
+ public NodeResources realResources() {
+ return realResources;
+ }
+
+ public double vcpu() {
+ return realResources.vcpu();
+ }
+
+ public double memoryGb() {
+ return realResources.memoryGb();
+ }
+
+ public DiskSize diskSize() {
+ return DiskSize.of(realResources.diskGb(), DiskSize.Unit.GB);
+ }
+
+ public double diskGb() {
+ return realResources.diskGb();
+ }
+
+ public boolean isFastDisk() {
+ return realResources.diskSpeed() == fast;
+ }
+
+ public double bandwidthGbps() {
+ return realResources.bandwidthGbps();
+ }
+
+ public Set<String> ipAddresses() {
+ return ipAddresses;
+ }
+
+ public Set<String> additionalIpAddresses() {
+ return additionalIpAddresses;
+ }
+
+ public NodeReports reports() { return reports; }
+
+ public List<Event> events() {
+ return events;
+ }
+
+ public Optional<String> parentHostname() {
+ return parentHostname;
+ }
+
+ public Optional<URI> archiveUri() {
+ return archiveUri;
+ }
+
+ public Optional<ApplicationId> exclusiveTo() {
+ return exclusiveTo;
+ }
+
+ public List<TrustStoreItem> trustStore() {
+ return trustStore;
+ }
+
+ public Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp() { return wireguardKeyWithTimestamp; }
+
+ public boolean wantToRebuild() {
+ return wantToRebuild;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof NodeSpec that)) return false;
+
+ return Objects.equals(hostname, that.hostname) &&
+ Objects.equals(id, that.id) &&
+ Objects.equals(wantedDockerImage, that.wantedDockerImage) &&
+ Objects.equals(currentDockerImage, that.currentDockerImage) &&
+ Objects.equals(state, that.state) &&
+ Objects.equals(type, that.type) &&
+ Objects.equals(cloudAccount, that.cloudAccount) &&
+ Objects.equals(flavor, that.flavor) &&
+ Objects.equals(modelName, that.modelName) &&
+ Objects.equals(wantedVespaVersion, that.wantedVespaVersion) &&
+ Objects.equals(currentVespaVersion, that.currentVespaVersion) &&
+ Objects.equals(wantedOsVersion, that.wantedOsVersion) &&
+ Objects.equals(currentOsVersion, that.currentOsVersion) &&
+ Objects.equals(orchestratorStatus, that.orchestratorStatus) &&
+ Objects.equals(owner, that.owner) &&
+ Objects.equals(membership, that.membership) &&
+ Objects.equals(wantedRestartGeneration, that.wantedRestartGeneration) &&
+ Objects.equals(currentRestartGeneration, that.currentRestartGeneration) &&
+ Objects.equals(wantedRebootGeneration, that.wantedRebootGeneration) &&
+ Objects.equals(currentRebootGeneration, that.currentRebootGeneration) &&
+ Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) &&
+ Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) &&
+ Objects.equals(resources, that.resources) &&
+ Objects.equals(realResources, that.realResources) &&
+ Objects.equals(ipAddresses, that.ipAddresses) &&
+ Objects.equals(additionalIpAddresses, that.additionalIpAddresses) &&
+ Objects.equals(reports, that.reports) &&
+ Objects.equals(events, that.events) &&
+ Objects.equals(parentHostname, that.parentHostname) &&
+ Objects.equals(archiveUri, that.archiveUri) &&
+ Objects.equals(exclusiveTo, that.exclusiveTo) &&
+ Objects.equals(trustStore, that.trustStore) &&
+ Objects.equals(wireguardKeyWithTimestamp, that.wireguardKeyWithTimestamp) &&
+ Objects.equals(wantToRebuild, that.wantToRebuild);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ hostname,
+ id,
+ wantedDockerImage,
+ currentDockerImage,
+ state,
+ type,
+ cloudAccount,
+ flavor,
+ modelName,
+ wantedVespaVersion,
+ currentVespaVersion,
+ wantedOsVersion,
+ currentOsVersion,
+ orchestratorStatus,
+ owner,
+ membership,
+ wantedRestartGeneration,
+ currentRestartGeneration,
+ wantedRebootGeneration,
+ currentRebootGeneration,
+ wantedFirmwareCheck,
+ currentFirmwareCheck,
+ resources,
+ realResources,
+ ipAddresses,
+ additionalIpAddresses,
+ reports,
+ events,
+ parentHostname,
+ archiveUri,
+ exclusiveTo,
+ trustStore,
+ wireguardKeyWithTimestamp,
+ wantToRebuild);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + " {"
+ + " hostname=" + hostname
+ + " id=" + id
+ + " wantedDockerImage=" + wantedDockerImage
+ + " currentDockerImage=" + currentDockerImage
+ + " state=" + state
+ + " type=" + type
+ + " cloudAccount=" + cloudAccount
+ + " flavor=" + flavor
+ + " modelName=" + modelName
+ + " wantedVespaVersion=" + wantedVespaVersion
+ + " currentVespaVersion=" + currentVespaVersion
+ + " wantedOsVersion=" + wantedOsVersion
+ + " currentOsVersion=" + currentOsVersion
+ + " orchestratorStatus=" + orchestratorStatus
+ + " owner=" + owner
+ + " membership=" + membership
+ + " wantedRestartGeneration=" + wantedRestartGeneration
+ + " currentRestartGeneration=" + currentRestartGeneration
+ + " wantedRebootGeneration=" + wantedRebootGeneration
+ + " currentRebootGeneration=" + currentRebootGeneration
+ + " wantedFirmwareCheck=" + wantedFirmwareCheck
+ + " currentFirmwareCheck=" + currentFirmwareCheck
+ + " resources=" + resources
+ + " realResources=" + realResources
+ + " ipAddresses=" + ipAddresses
+ + " additionalIpAddresses=" + additionalIpAddresses
+ + " reports=" + reports
+ + " events=" + events
+ + " parentHostname=" + parentHostname
+ + " archiveUri=" + archiveUri
+ + " exclusiveTo=" + exclusiveTo
+ + " trustStore=" + trustStore
+ + " wireguardPubkey=" + wireguardKeyWithTimestamp
+ + " wantToRebuild=" + wantToRebuild
+ + " }";
+ }
+
+ public static class Builder {
+ private String hostname;
+ private String id;
+ private NodeState state;
+ private NodeType type;
+ private CloudAccount cloudAccount = CloudAccount.empty;
+ private String flavor;
+ private Optional<DockerImage> wantedDockerImage = Optional.empty();
+ private Optional<DockerImage> currentDockerImage = Optional.empty();
+ private Optional<Version> wantedVespaVersion = Optional.empty();
+ private Optional<Version> currentVespaVersion = Optional.empty();
+ private Optional<Version> wantedOsVersion = Optional.empty();
+ private Optional<Version> currentOsVersion = Optional.empty();
+ private OrchestratorStatus orchestratorStatus = OrchestratorStatus.NO_REMARKS;
+ private Optional<ApplicationId> owner = Optional.empty();
+ private Optional<NodeMembership> membership = Optional.empty();
+ private Optional<Long> wantedRestartGeneration = Optional.empty();
+ private Optional<Long> currentRestartGeneration = Optional.empty();
+ private long wantedRebootGeneration;
+ private long currentRebootGeneration;
+ private Optional<Instant> wantedFirmwareCheck = Optional.empty();
+ private Optional<Instant> currentFirmwareCheck = Optional.empty();
+ private Optional<String> modelName = Optional.empty();
+ private NodeResources resources;
+ private NodeResources realResources;
+ private Set<String> ipAddresses = Set.of();
+ private Set<String> additionalIpAddresses = Set.of();
+ private NodeReports reports = new NodeReports();
+ private List<Event> events = List.of();
+ private Optional<String> parentHostname = Optional.empty();
+ private Optional<URI> archiveUri = Optional.empty();
+ private Optional<ApplicationId> exclusiveTo = Optional.empty();
+ private List<TrustStoreItem> trustStore = List.of();
+ private Optional<WireguardKeyWithTimestamp> wireguardPubkey = Optional.empty();
+ private boolean wantToRebuild = false;
+
+ public Builder() {}
+
+ public Builder(NodeSpec node) {
+ hostname(node.hostname);
+ id(node.id);
+ state(node.state);
+ type(node.type);
+ flavor(node.flavor);
+ resources(node.resources);
+ realResources(node.realResources);
+ ipAddresses(node.ipAddresses);
+ additionalIpAddresses(node.additionalIpAddresses);
+ wantedRebootGeneration(node.wantedRebootGeneration);
+ currentRebootGeneration(node.currentRebootGeneration);
+ orchestratorStatus(node.orchestratorStatus);
+ reports(new NodeReports(node.reports));
+ events(node.events);
+ node.wantedDockerImage.ifPresent(this::wantedDockerImage);
+ node.currentDockerImage.ifPresent(this::currentDockerImage);
+ node.wantedVespaVersion.ifPresent(this::wantedVespaVersion);
+ node.currentVespaVersion.ifPresent(this::currentVespaVersion);
+ node.wantedOsVersion.ifPresent(this::wantedOsVersion);
+ node.currentOsVersion.ifPresent(this::currentOsVersion);
+ node.owner.ifPresent(this::owner);
+ node.membership.ifPresent(this::membership);
+ node.wantedRestartGeneration.ifPresent(this::wantedRestartGeneration);
+ node.currentRestartGeneration.ifPresent(this::currentRestartGeneration);
+ node.wantedFirmwareCheck.ifPresent(this::wantedFirmwareCheck);
+ node.currentFirmwareCheck.ifPresent(this::currentFirmwareCheck);
+ node.parentHostname.ifPresent(this::parentHostname);
+ node.archiveUri.ifPresent(this::archiveUri);
+ node.exclusiveTo.ifPresent(this::exclusiveTo);
+ trustStore(node.trustStore);
+ node.wireguardKeyWithTimestamp.ifPresent(this::wireguardKeyWithTimestamp);
+ wantToRebuild(node.wantToRebuild);
+ }
+
+ public Builder hostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ public Builder id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder wantedDockerImage(DockerImage wantedDockerImage) {
+ this.wantedDockerImage = Optional.of(wantedDockerImage);
+ return this;
+ }
+
+ public Builder currentDockerImage(DockerImage currentDockerImage) {
+ this.currentDockerImage = Optional.of(currentDockerImage);
+ return this;
+ }
+
+ public Builder state(NodeState state) {
+ this.state = state;
+ return this;
+ }
+
+ public Builder type(NodeType nodeType) {
+ this.type = nodeType;
+ return this;
+ }
+
+ public Builder cloudAccount(CloudAccount cloudAccount) {
+ this.cloudAccount = cloudAccount;
+ return this;
+ }
+
+ public Builder flavor(String flavor) {
+ this.flavor = flavor;
+ return this;
+ }
+
+ public Builder wantedVespaVersion(Version wantedVespaVersion) {
+ this.wantedVespaVersion = Optional.of(wantedVespaVersion);
+ return this;
+ }
+
+ public Builder currentVespaVersion(Version vespaVersion) {
+ this.currentVespaVersion = Optional.of(vespaVersion);
+ return this;
+ }
+
+ public Builder wantedOsVersion(Version wantedOsVersion) {
+ this.wantedOsVersion = Optional.of(wantedOsVersion);
+ return this;
+ }
+
+ public Builder currentOsVersion(Version currentOsVersion) {
+ this.currentOsVersion = Optional.of(currentOsVersion);
+ return this;
+ }
+
+ public Builder orchestratorStatus(OrchestratorStatus orchestratorStatus) {
+ this.orchestratorStatus = orchestratorStatus;
+ return this;
+ }
+
+ public Builder owner(ApplicationId owner) {
+ this.owner = Optional.of(owner);
+ return this;
+ }
+
+ public Builder membership(NodeMembership membership) {
+ this.membership = Optional.of(membership);
+ return this;
+ }
+
+ public Builder wantedRestartGeneration(long wantedRestartGeneration) {
+ this.wantedRestartGeneration = Optional.of(wantedRestartGeneration);
+ return this;
+ }
+
+ public Builder currentRestartGeneration(long currentRestartGeneration) {
+ this.currentRestartGeneration = Optional.of(currentRestartGeneration);
+ return this;
+ }
+
+ public Builder wantedRebootGeneration(long wantedRebootGeneration) {
+ this.wantedRebootGeneration = wantedRebootGeneration;
+ return this;
+ }
+
+ public Builder currentRebootGeneration(long currentRebootGeneration) {
+ this.currentRebootGeneration = currentRebootGeneration;
+ return this;
+ }
+
+ public Builder wantedFirmwareCheck(Instant wantedFirmwareCheck) {
+ this.wantedFirmwareCheck = Optional.of(wantedFirmwareCheck);
+ return this;
+ }
+
+ public Builder currentFirmwareCheck(Instant currentFirmwareCheck) {
+ this.currentFirmwareCheck = Optional.of(currentFirmwareCheck);
+ return this;
+ }
+
+ public Builder resources(NodeResources resources) {
+ this.resources = resources;
+ return this;
+ }
+
+ public Builder realResources(NodeResources realResources) {
+ this.realResources = realResources;
+ return this;
+ }
+
+ public Builder vcpu(double vcpu) {
+ return realResources(realResources.withVcpu(vcpu));
+ }
+
+ public Builder memoryGb(double memoryGb) {
+ return realResources(realResources.withMemoryGb(memoryGb));
+ }
+
+ public Builder diskGb(double diskGb) {
+ return realResources(realResources.withDiskGb(diskGb));
+ }
+
+ public Builder fastDisk(boolean fastDisk) {
+ return realResources(realResources.with(fastDisk ? fast : slow));
+ }
+
+ public Builder bandwidthGbps(double bandwidthGbps) {
+ return realResources(realResources.withBandwidthGbps(bandwidthGbps));
+ }
+
+ public Builder ipAddresses(Set<String> ipAddresses) {
+ this.ipAddresses = ipAddresses;
+ return this;
+ }
+
+ public Builder additionalIpAddresses(Set<String> additionalIpAddresses) {
+ this.additionalIpAddresses = additionalIpAddresses;
+ return this;
+ }
+
+ public Builder reports(NodeReports reports) {
+ this.reports = reports;
+ return this;
+ }
+
+ public Builder report(String reportId, JsonNode report) {
+ this.reports.setReport(reportId, report);
+ return this;
+ }
+
+ public Builder removeReport(String reportId) {
+ reports.removeReport(reportId);
+ return this;
+ }
+
+ public Builder events(List<Event> events) {
+ this.events = events;
+ return this;
+ }
+
+ public Builder parentHostname(String parentHostname) {
+ this.parentHostname = Optional.of(parentHostname);
+ return this;
+ }
+
+ public Builder archiveUri(URI archiveUri) {
+ this.archiveUri = Optional.of(archiveUri);
+ return this;
+ }
+
+ public Builder exclusiveTo(ApplicationId applicationId) {
+ this.exclusiveTo = Optional.of(applicationId);
+ return this;
+ }
+
+ public Builder trustStore(List<TrustStoreItem> trustStore) {
+ this.trustStore = List.copyOf(trustStore);
+ return this;
+ }
+
+ public Builder wireguardPubkey(WireguardKey wireguardPubkey) {
+ this.wireguardPubkey = Optional.of(new WireguardKeyWithTimestamp(wireguardPubkey, Instant.EPOCH));
+ return this;
+ }
+
+ public Builder wireguardKeyWithTimestamp(WireguardKeyWithTimestamp wireguardPubKey) {
+ this.wireguardPubkey = Optional.of(wireguardPubKey);
+ return this;
+ }
+
+ public Builder wantToRebuild(boolean wantToRebuild) {
+ this.wantToRebuild = wantToRebuild;
+ return this;
+ }
+
+ public Builder updateFromNodeAttributes(NodeAttributes attributes) {
+ attributes.getHostId().ifPresent(this::id);
+ attributes.getDockerImage().ifPresent(this::currentDockerImage);
+ attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion);
+ attributes.getRebootGeneration().ifPresent(this::currentRebootGeneration);
+ attributes.getRestartGeneration().ifPresent(this::currentRestartGeneration);
+ // Always replace entire trust store
+ trustStore(attributes.getTrustStore());
+ attributes.getWireguardPubkey().ifPresent(this::wireguardPubkey);
+ this.reports.updateFromRawMap(attributes.getReports());
+
+ return this;
+ }
+
+ public String hostname() {
+ return hostname;
+ }
+
+ public Optional<DockerImage> wantedDockerImage() {
+ return wantedDockerImage;
+ }
+
+ public Optional<DockerImage> currentDockerImage() {
+ return currentDockerImage;
+ }
+
+ public NodeState state() {
+ return state;
+ }
+
+ public NodeType type() {
+ return type;
+ }
+
+ public CloudAccount cloudAccount() {
+ return cloudAccount;
+ }
+
+ public String flavor() {
+ return flavor;
+ }
+
+ public Optional<Version> wantedVespaVersion() {
+ return wantedVespaVersion;
+ }
+
+ public Optional<Version> currentVespaVersion() {
+ return currentVespaVersion;
+ }
+
+ public Optional<Version> wantedOsVersion() {
+ return wantedOsVersion;
+ }
+
+ public Optional<Version> currentOsVersion() {
+ return currentOsVersion;
+ }
+
+ public OrchestratorStatus orchestratorStatus() {
+ return orchestratorStatus;
+ }
+
+ public Optional<ApplicationId> owner() {
+ return owner;
+ }
+
+ public Optional<NodeMembership> membership() {
+ return membership;
+ }
+
+ public Optional<Long> wantedRestartGeneration() {
+ return wantedRestartGeneration;
+ }
+
+ public Optional<Long> currentRestartGeneration() {
+ return currentRestartGeneration;
+ }
+
+ public long wantedRebootGeneration() {
+ return wantedRebootGeneration;
+ }
+
+ public long currentRebootGeneration() {
+ return currentRebootGeneration;
+ }
+
+ public NodeResources resources() {
+ return resources;
+ }
+
+ public NodeResources realResources() {
+ return realResources;
+ }
+
+ public Set<String> ipAddresses() {
+ return ipAddresses;
+ }
+
+ public Set<String> additionalIpAddresses() {
+ return additionalIpAddresses;
+ }
+
+ public NodeReports reports() {
+ return reports;
+ }
+
+ public List<Event> events() {
+ return events;
+ }
+
+ public Optional<String> parentHostname() {
+ return parentHostname;
+ }
+
+ public Optional<URI> archiveUri() {
+ return archiveUri;
+ }
+
+ public NodeSpec build() {
+ return new NodeSpec(hostname, id, wantedDockerImage, currentDockerImage, state, type, cloudAccount, flavor,
+ wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, orchestratorStatus,
+ owner, membership,
+ wantedRestartGeneration, currentRestartGeneration,
+ wantedRebootGeneration, currentRebootGeneration,
+ wantedFirmwareCheck, currentFirmwareCheck, modelName,
+ resources, realResources, ipAddresses, additionalIpAddresses,
+ reports, events, parentHostname, archiveUri, exclusiveTo, trustStore,
+ wireguardPubkey, wantToRebuild);
+ }
+
+
+ public static Builder testSpec(String hostname) {
+ return testSpec(hostname, NodeState.active);
+ }
+
+ /**
+ * Creates a NodeSpec.Builder that has the given hostname, in a given state, and some
+ * reasonable values for the remaining required NodeSpec fields.
+ */
+ public static Builder testSpec(String hostname, NodeState state) {
+ Builder builder = new Builder()
+ .id(hostname)
+ .hostname(hostname)
+ .state(state)
+ .type(NodeType.tenant)
+ .flavor("d-2-8-50")
+ .resources(new NodeResources(2, 8, 50, 10))
+ .realResources(new NodeResources(2, 8, 50, 10))
+ .events(List.of(new Event("operator", "rebooted", Instant.EPOCH)));
+
+ // Set the required allocated fields
+ if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) {
+ builder .owner(ApplicationId.defaultId())
+ .membership(new NodeMembership("container", "my-id", "group", 0, false))
+ .wantedVespaVersion(Version.fromString("7.1.1"))
+ .wantedDockerImage(DockerImage.fromString("docker.domain.tld/repo/image:7.1.1"))
+ .currentRestartGeneration(0)
+ .wantedRestartGeneration(0);
+ }
+
+ return builder;
+ }
+ }
+
+ private static void requireOptional(Optional<?> optional, String name) {
+ if (optional == null || optional.isEmpty())
+ throw new IllegalArgumentException(name + " must be set, was " + optional);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java
new file mode 100644
index 00000000000..8e66480c92a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+/**
+ * All the states a node can be in the node-repository.
+ *
+ * See com.yahoo.vespa.hosted.provision.NodeState
+ *
+ * @author freva
+ */
+public enum NodeState {
+ provisioned, ready, reserved, active, inactive, dirty, failed, parked, deprovisioned, breakfixed
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java
new file mode 100644
index 00000000000..d8532188c64
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java
@@ -0,0 +1,23 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import java.util.stream.Stream;
+
+public enum OrchestratorStatus {
+ NO_REMARKS, ALLOWED_TO_BE_DOWN, PERMANENTLY_DOWN, UNKNOWN;
+
+ public static OrchestratorStatus fromString(String statusString) {
+ return Stream.of(values())
+ .filter(status -> status.asString().equals(statusString))
+ .findFirst()
+ .orElse(UNKNOWN);
+ }
+
+ public String asString() {
+ return name();
+ }
+
+ public boolean isSuspended() {
+ return this != NO_REMARKS;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
new file mode 100644
index 00000000000..d340aa9fd3d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
@@ -0,0 +1,406 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.net.InetAddresses;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudAccount;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.config.provision.WireguardKeyWithTimestamp;
+import com.yahoo.config.provision.host.FlavorOverrides;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
+import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetNodesResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetWireguardResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress;
+import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author stiankri
+ * @author dybis
+ */
+public class RealNodeRepository implements NodeRepository {
+ private static final Logger logger = Logger.getLogger(RealNodeRepository.class.getName());
+
+ private final ConfigServerApi configServerApi;
+
+ public RealNodeRepository(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ }
+
+ @Override
+ public void addNodes(List<AddNode> nodes) {
+ List<NodeRepositoryNode> nodesToPost = nodes.stream()
+ .map(RealNodeRepository::nodeRepositoryNodeFromAddNode)
+ .toList();
+
+ configServerApi.post("/nodes/v2/node", nodesToPost, StandardConfigServerResponse.class)
+ .throwOnError("Failed to add nodes");
+ }
+
+ @Override
+ public List<NodeSpec> getNodes(String baseHostName) {
+ String path = "/nodes/v2/node/?recursive=true&parentHost=" + baseHostName;
+ final GetNodesResponse nodesForHost = configServerApi.get(path, GetNodesResponse.class);
+
+ return nodesForHost.nodes.stream()
+ .map(RealNodeRepository::createNodeSpec)
+ .toList();
+ }
+
+ @Override
+ public Optional<NodeSpec> getOptionalNode(String hostName) {
+ try {
+ NodeRepositoryNode nodeResponse = configServerApi.get("/nodes/v2/node/" + hostName,
+ NodeRepositoryNode.class);
+
+ return Optional.ofNullable(nodeResponse).map(RealNodeRepository::createNodeSpec);
+ } catch (HttpException.NotFoundException | HttpException.ForbiddenException e) {
+ // Return empty on 403 in addition to 404 as it likely means we're trying to access a node that
+ // has been deleted. When a node is deleted, the parent-child relationship no longer exists and
+ // authorization cannot be granted.
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Get all ACLs that belongs to a hostname. Usually this is a parent host and all
+ * ACLs for child nodes are returned.
+ */
+ @Override
+ public Map<String, Acl> getAcls(String hostName) {
+ String path = String.format("/nodes/v2/acl/%s?children=true", hostName);
+ GetAclResponse response = configServerApi.get(path, GetAclResponse.class);
+
+ // Group ports by container hostname that trusts them
+ Map<String, Set<Integer>> trustedPorts = response.trustedPorts.stream()
+ .collect(Collectors.groupingBy(
+ GetAclResponse.Port::getTrustedBy,
+ Collectors.mapping(port -> port.port, Collectors.toSet())));
+
+ // Group UDP ports by container hostname that trusts them
+ Map<String, Set<Integer>> trustedUdpPorts = response.trustedUdpPorts.stream()
+ .collect(Collectors.groupingBy(
+ GetAclResponse.Port::getTrustedBy,
+ Collectors.mapping(port -> port.port, Collectors.toSet())));
+
+ // Group node ip-addresses by container hostname that trusts them
+ Map<String, Set<Acl.Node>> trustedNodes = response.trustedNodes.stream()
+ .collect(Collectors.groupingBy(
+ GetAclResponse.Node::getTrustedBy,
+ Collectors.mapping(
+ node -> new Acl.Node(node.hostname, node.ipAddress, Set.copyOf(node.ports)),
+ Collectors.toSet())));
+
+ // Group trusted networks by container hostname that trusts them
+ Map<String, Set<String>> trustedNetworks = response.trustedNetworks.stream()
+ .collect(Collectors.groupingBy(GetAclResponse.Network::getTrustedBy,
+ Collectors.mapping(node -> node.network, Collectors.toSet())));
+
+
+ // For each hostname create an ACL
+ return Stream.of(trustedNodes.keySet(), trustedPorts.keySet(), trustedUdpPorts.keySet(), trustedNetworks.keySet())
+ .flatMap(Set::stream)
+ .distinct()
+ .collect(Collectors.toMap(
+ Function.identity(),
+ hostname -> new Acl(trustedPorts.get(hostname),
+ trustedUdpPorts.get(hostname),
+ trustedNodes.get(hostname),
+ trustedNetworks.get(hostname))));
+ }
+
+ @Override
+ public List<WireguardPeer> getExclavePeers() {
+ String path = "/nodes/v2/node/?recursive=true&enclave=true";
+ final GetNodesResponse response = configServerApi.get(path, GetNodesResponse.class);
+
+ return response.nodes.stream()
+ .mapMulti((NodeRepositoryNode node, Consumer<WireguardPeer> consumer) -> {
+ var keyWithTimestamp = createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp,
+ node.wireguardPubkey,
+ node.wireguardKeyTimestamp);
+ if (keyWithTimestamp == null) return;
+
+ List<VersionedIpAddress> ipAddresses = getIpAddresses(node);
+ if (ipAddresses.isEmpty()) return;
+
+ consumer.accept(new WireguardPeer(HostName.of(node.hostname), ipAddresses, keyWithTimestamp));
+ })
+ .sorted()
+ .toList();
+ }
+
+ private static List<VersionedIpAddress> getIpAddresses(NodeRepositoryNode node) {
+ return node.ipAddresses.stream()
+ .map(InetAddresses::forString)
+ .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress())
+ .map(VersionedIpAddress::from)
+ .toList();
+ }
+
+ @Override
+ public List<WireguardPeer> getConfigserverPeers() {
+ GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class);
+ return response.configservers.stream()
+ .map(RealNodeRepository::createConfigserverPeer)
+ .sorted(Comparator.comparing(WireguardPeer::hostname))
+ .toList();
+ }
+
+ @Override
+ public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) {
+ configServerApi.patch("/nodes/v2/node/" + hostName,
+ nodeRepositoryNodeFromNodeAttributes(nodeAttributes),
+ StandardConfigServerResponse.class)
+ .throwOnError("Failed to update node attributes");
+ }
+
+ @Override
+ public void setNodeState(String hostName, NodeState nodeState) {
+ String state = nodeState.name();
+ StandardConfigServerResponse response = configServerApi.put("/nodes/v2/state/" + state + "/" + hostName,
+ Optional.empty(), /* body */
+ StandardConfigServerResponse.class);
+ logger.info(response.message);
+ response.throwOnError("Failed to set node state");
+ }
+
+ @Override
+ public void reboot(String hostname) {
+ String uri = "/nodes/v2/command/reboot?hostname=" + hostname;
+ StandardConfigServerResponse response = configServerApi.post(uri, Optional.empty(), StandardConfigServerResponse.class);
+ logger.info(response.message);
+ response.throwOnError("Failed to reboot " + hostname);
+ }
+
+ private static NodeSpec createNodeSpec(NodeRepositoryNode node) {
+ Objects.requireNonNull(node.type, "Unknown node type");
+ NodeType nodeType = NodeType.valueOf(node.type);
+
+ Objects.requireNonNull(node.state, "Unknown node state");
+ NodeState nodeState = NodeState.valueOf(node.state);
+
+ Optional<NodeMembership> membership = Optional.ofNullable(node.membership)
+ .map(m -> new NodeMembership(m.clusterType, m.clusterId, m.group, m.index, m.retired));
+ NodeReports reports = NodeReports.fromMap(Optional.ofNullable(node.reports).orElseGet(Map::of));
+ List<Event> events = node.history.stream()
+ .map(event -> new Event(event.agent, event.event, Optional.ofNullable(event.at).map(Instant::ofEpochMilli).orElse(Instant.EPOCH)))
+ .toList();
+
+ List<TrustStoreItem> trustStore = Optional.ofNullable(node.trustStore).orElse(List.of()).stream()
+ .map(item -> new TrustStoreItem(item.fingerprint, Instant.ofEpochMilli(item.expiry)))
+ .toList();
+
+
+ return new NodeSpec(
+ node.hostname,
+ node.id,
+ Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString),
+ Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString),
+ nodeState,
+ nodeType,
+ Optional.ofNullable(node.cloudAccount).map(CloudAccount::from).orElse(CloudAccount.empty),
+ node.flavor,
+ Optional.ofNullable(node.wantedVespaVersion).map(Version::fromString),
+ Optional.ofNullable(node.vespaVersion).map(Version::fromString),
+ Optional.ofNullable(node.wantedOsVersion).map(Version::fromString),
+ Optional.ofNullable(node.currentOsVersion).map(Version::fromString),
+ Optional.ofNullable(node.orchestratorStatus).map(OrchestratorStatus::fromString).orElse(OrchestratorStatus.NO_REMARKS),
+ Optional.ofNullable(node.owner).map(o -> ApplicationId.from(o.tenant, o.application, o.instance)),
+ membership,
+ Optional.ofNullable(node.restartGeneration),
+ Optional.ofNullable(node.currentRestartGeneration),
+ node.rebootGeneration,
+ node.currentRebootGeneration,
+ Optional.ofNullable(node.wantedFirmwareCheck).map(Instant::ofEpochMilli),
+ Optional.ofNullable(node.currentFirmwareCheck).map(Instant::ofEpochMilli),
+ Optional.ofNullable(node.modelName),
+ nodeResources(node.resources),
+ nodeResources(node.realResources),
+ node.ipAddresses,
+ node.additionalIpAddresses,
+ reports,
+ events,
+ Optional.ofNullable(node.parentHostname),
+ Optional.ofNullable(node.archiveUri).map(URI::create),
+ Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm),
+ trustStore,
+ Optional.ofNullable(createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp,
+ node.wireguardPubkey,
+ node.wireguardKeyTimestamp)),
+ node.wantToRebuild);
+ }
+
+ private static NodeResources nodeResources(NodeRepositoryNode.NodeResources nodeResources) {
+ return new NodeResources(
+ nodeResources.vcpu,
+ nodeResources.memoryGb,
+ nodeResources.diskGb,
+ nodeResources.bandwidthGbps,
+ diskSpeedFromString(nodeResources.diskSpeed),
+ storageTypeFromString(nodeResources.storageType),
+ architectureFromString(nodeResources.architecture),
+ gpuResourcesFrom(nodeResources));
+ }
+
+ private static NodeResources.GpuResources gpuResourcesFrom(NodeRepositoryNode.NodeResources nodeResources) {
+ if (nodeResources.gpuCount == null || nodeResources.gpuMemoryGb == null) return NodeResources.GpuResources.zero();
+ return new NodeResources.GpuResources(nodeResources.gpuCount, nodeResources.gpuMemoryGb);
+ }
+
+ private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) {
+ if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault();
+ return switch (diskSpeed) {
+ case "fast" -> NodeResources.DiskSpeed.fast;
+ case "slow" -> NodeResources.DiskSpeed.slow;
+ case "any" -> NodeResources.DiskSpeed.any;
+ default -> throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'");
+ };
+ }
+
+ private static NodeResources.StorageType storageTypeFromString(String storageType) {
+ if (storageType == null) return NodeResources.StorageType.getDefault();
+ return switch (storageType) {
+ case "remote" -> NodeResources.StorageType.remote;
+ case "local" -> NodeResources.StorageType.local;
+ case "any" -> NodeResources.StorageType.any;
+ default -> throw new IllegalArgumentException("Unknown storage type '" + storageType + "'");
+ };
+ }
+
+ private static NodeResources.Architecture architectureFromString(String architecture) {
+ if (architecture == null) return NodeResources.Architecture.getDefault();
+ return switch (architecture) {
+ case "arm64" -> NodeResources.Architecture.arm64;
+ case "x86_64" -> NodeResources.Architecture.x86_64;
+ case "any" -> NodeResources.Architecture.any;
+ default -> throw new IllegalArgumentException("Unknown architecture '" + architecture + "'");
+ };
+ }
+
+ private static String toString(NodeResources.DiskSpeed diskSpeed) {
+ return switch (diskSpeed) {
+ case fast -> "fast";
+ case slow -> "slow";
+ case any -> "any";
+ };
+ }
+
+ private static String toString(NodeResources.StorageType storageType) {
+ return switch (storageType) {
+ case remote -> "remote";
+ case local -> "local";
+ case any -> "any";
+ };
+ }
+
+ private static String toString(NodeResources.Architecture architecture) {
+ return switch (architecture) {
+ case arm64 -> "arm64";
+ case x86_64 -> "x86_64";
+ case any -> "any";
+ };
+ }
+
+ private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) {
+ NodeRepositoryNode node = new NodeRepositoryNode();
+ node.id = addNode.id;
+ node.hostname = addNode.hostname;
+ node.parentHostname = addNode.parentHostname.orElse(null);
+ addNode.nodeFlavor.ifPresent(f -> node.flavor = f);
+ addNode.flavorOverrides.flatMap(FlavorOverrides::diskGb).ifPresent(d -> {
+ node.resources = new NodeRepositoryNode.NodeResources();
+ node.resources.diskGb = d;
+ });
+ addNode.nodeResources.ifPresent(resources -> {
+ node.resources = new NodeRepositoryNode.NodeResources();
+ node.resources.vcpu = resources.vcpu();
+ node.resources.memoryGb = resources.memoryGb();
+ node.resources.diskGb = resources.diskGb();
+ node.resources.bandwidthGbps = resources.bandwidthGbps();
+ node.resources.diskSpeed = toString(resources.diskSpeed());
+ node.resources.storageType = toString(resources.storageType());
+ node.resources.architecture = toString(resources.architecture());
+ if (!resources.gpuResources().isZero()) {
+ node.resources.gpuCount = resources.gpuResources().count();
+ node.resources.gpuMemoryGb = resources.gpuResources().memoryGb();
+ }
+ });
+ node.type = addNode.nodeType.name();
+ node.ipAddresses = addNode.ipAddresses;
+ node.additionalIpAddresses = addNode.additionalIpAddresses;
+ return node;
+ }
+
+ public static NodeRepositoryNode nodeRepositoryNodeFromNodeAttributes(NodeAttributes nodeAttributes) {
+ NodeRepositoryNode node = new NodeRepositoryNode();
+ node.id = nodeAttributes.getHostId().orElse(null);
+ node.currentDockerImage = nodeAttributes.getDockerImage().map(DockerImage::asString).orElse(null);
+ node.currentRestartGeneration = nodeAttributes.getRestartGeneration().orElse(null);
+ node.currentRebootGeneration = nodeAttributes.getRebootGeneration().orElse(null);
+ node.vespaVersion = nodeAttributes.getVespaVersion().map(Version::toFullString).orElse(null);
+ node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null);
+ node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null);
+ node.trustStore = nodeAttributes.getTrustStore().stream()
+ .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli()))
+ .toList();
+ // This is used for patching, and timestamp must only be set on the server side, hence sending EPOCH.
+ node.wireguardKeyWithTimestamp = nodeAttributes.getWireguardPubkey()
+ .map(key -> new NodeRepositoryNode.WireguardKeyWithTimestamp(key.value(), 0L))
+ .orElse(null);
+ Map<String, JsonNode> reports = nodeAttributes.getReports();
+ node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports);
+
+ // TODO wg: remove when all nodes are using new key+timestamp format
+ node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null);
+ return node;
+ }
+
+ private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) {
+ return new WireguardPeer(HostName.of(configServer.hostname),
+ configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(),
+ createWireguardKeyWithTimestamp(configServer.wireguardKeyWithTimestamp,
+ configServer.wireguardPubkey,
+ configServer.wireguardKeyTimestamp));
+ }
+
+ private static WireguardKeyWithTimestamp createWireguardKeyWithTimestamp(NodeRepositoryNode.WireguardKeyWithTimestamp wirguardJson,
+ String oldKeyJson, Long oldTimestampJson) {
+ if (wirguardJson != null && wirguardJson.key != null && ! wirguardJson.key.isEmpty()) {
+ return new WireguardKeyWithTimestamp(WireguardKey.from(wirguardJson.key),
+ Instant.ofEpochMilli(wirguardJson.timestamp));
+ // TODO wg: remove when all nodes are using new key+timestamp format
+ } else if (oldKeyJson != null) {
+ var timestamp = oldTimestampJson != null ? oldTimestampJson : 0L;
+ return new WireguardKeyWithTimestamp(WireguardKey.from(oldKeyJson),
+ Instant.ofEpochMilli(timestamp));
+ // TODO END
+ } else return null;
+
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java
new file mode 100644
index 00000000000..dfec70288e3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java
@@ -0,0 +1,48 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * @author mortent
+ */
+public class TrustStoreItem {
+ private final String fingerprint;
+ private final Instant expiry;
+
+ public TrustStoreItem(String fingerprint, Instant expiry) {
+ this.fingerprint = fingerprint;
+ this.expiry = expiry;
+ }
+
+ public String fingerprint() {
+ return fingerprint;
+ }
+
+ public Instant expiry() {
+ return expiry;
+ }
+
+ @Override
+ public String toString() {
+ return "TrustStoreItem{" +
+ "fingerprint='" + fingerprint + '\'' +
+ ", expiry=" + expiry +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TrustStoreItem that = (TrustStoreItem) o;
+ return Objects.equals(fingerprint, that.fingerprint) && Objects.equals(expiry, that.expiry);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fingerprint, expiry);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java
new file mode 100644
index 00000000000..d20f31e256e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java
@@ -0,0 +1,114 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * This class represents a response from the /nodes/v2/acl/ API.
+ *
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GetAclResponse {
+
+ @JsonProperty("trustedNodes")
+ public final List<Node> trustedNodes;
+
+ @JsonProperty("trustedNetworks")
+ public final List<Network> trustedNetworks;
+
+ @JsonProperty("trustedPorts")
+ public final List<Port> trustedPorts;
+
+ @JsonProperty("trustedUdpPorts")
+ public final List<Port> trustedUdpPorts;
+
+ @JsonCreator
+ public GetAclResponse(@JsonProperty("trustedNodes") List<Node> trustedNodes,
+ @JsonProperty("trustedNetworks") List<Network> trustedNetworks,
+ @JsonProperty("trustedPorts") List<Port> trustedPorts,
+ @JsonProperty("trustedUdpPorts") List<Port> trustedUdpPorts) {
+ this.trustedNodes = trustedNodes == null ? List.of() : List.copyOf(trustedNodes);
+ this.trustedNetworks = trustedNetworks == null ? List.of() : List.copyOf(trustedNetworks);
+ this.trustedPorts = trustedPorts == null ? List.of() : List.copyOf(trustedPorts);
+ this.trustedUdpPorts = trustedUdpPorts == null ? List.of() : List.copyOf(trustedUdpPorts);
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Node {
+
+ @JsonProperty("hostname")
+ public final String hostname;
+
+ @JsonProperty("type")
+ public final String nodeType;
+
+ @JsonProperty("ipAddress")
+ public final String ipAddress;
+
+ @JsonProperty("ports")
+ public final List<Integer> ports;
+
+ @JsonProperty("trustedBy")
+ public final String trustedBy;
+
+ @JsonCreator
+ public Node(@JsonProperty("hostname") String hostname, @JsonProperty("type") String nodeType,
+ @JsonProperty("ipAddress") String ipAddress, @JsonProperty("ports") List<Integer> ports,
+ @JsonProperty("trustedBy") String trustedBy) {
+ this.hostname = hostname;
+ this.nodeType = nodeType;
+ this.ipAddress = ipAddress;
+ this.ports = ports == null ? List.of() : List.copyOf(ports);
+ this.trustedBy = trustedBy;
+ }
+
+ public String getTrustedBy() {
+ return trustedBy;
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Network {
+
+ @JsonProperty("network")
+ public final String network;
+
+ @JsonProperty("trustedBy")
+ public final String trustedBy;
+
+ @JsonCreator
+ public Network(@JsonProperty("network") String network, @JsonProperty("trustedBy") String trustedBy) {
+ this.network = network;
+ this.trustedBy = trustedBy;
+ }
+
+ public String getTrustedBy() {
+ return trustedBy;
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Port {
+
+ @JsonProperty("port")
+ public final Integer port;
+
+ @JsonProperty("trustedBy")
+ public final String trustedBy;
+
+ @JsonCreator
+ public Port(@JsonProperty("port") Integer port, @JsonProperty("trustedBy") String trustedBy) {
+ this.port = port;
+ this.trustedBy = trustedBy;
+ }
+
+ public String getTrustedBy() {
+ return trustedBy;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java
new file mode 100644
index 00000000000..b744c935247
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java
@@ -0,0 +1,25 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This class represents a response from the /nodes/v2/node/ API. It is designed to be
+ * usable by any module, by not depending itself on any module-specific classes.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GetNodesResponse {
+
+ public final List<NodeRepositoryNode> nodes;
+
+ @JsonCreator
+ public GetNodesResponse(@JsonProperty("nodes") List<NodeRepositoryNode> nodes) {
+ this.nodes = Collections.unmodifiableList(nodes);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java
new file mode 100644
index 00000000000..572323d733b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java
@@ -0,0 +1,50 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * A response from the /nodes/v2/wireguard api.
+ *
+ * @author gjoranv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class GetWireguardResponse {
+
+ public final List<Configserver> configservers;
+
+ @JsonCreator
+ public GetWireguardResponse(@JsonProperty("configservers") List<Configserver> configservers) {
+ this.configservers = configservers;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Configserver {
+
+ @JsonProperty("hostname")
+ public String hostname;
+
+ @JsonProperty("ipAddresses")
+ public List<String> ipAddresses;
+
+ @JsonProperty("wireguard")
+ public NodeRepositoryNode.WireguardKeyWithTimestamp wireguardKeyWithTimestamp;
+
+
+ // TODO wg: remove when all nodes use new key+timestamp format
+ @JsonProperty("wireguardPubkey")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public String wireguardPubkey;
+ @JsonProperty("wireguardKeyTimestamp")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public Long wireguardKeyTimestamp;
+
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
new file mode 100644
index 00000000000..c377d521648
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
@@ -0,0 +1,279 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class NodeRepositoryNode {
+
+ @JsonProperty("state")
+ public String state;
+ @JsonProperty("hostname")
+ public String hostname;
+ @JsonProperty("ipAddresses")
+ public Set<String> ipAddresses;
+ @JsonProperty("additionalIpAddresses")
+ public Set<String> additionalIpAddresses;
+ @JsonProperty("id")
+ public String id;
+ @JsonProperty("cloudAccount")
+ public String cloudAccount;
+ @JsonProperty("flavor")
+ public String flavor;
+ @JsonProperty("resources")
+ public NodeResources resources;
+ @JsonProperty("realResources")
+ public NodeResources realResources;
+ @JsonProperty("membership")
+ public Membership membership;
+ @JsonProperty("owner")
+ public Owner owner;
+ @JsonProperty("restartGeneration")
+ public Long restartGeneration;
+ @JsonProperty("rebootGeneration")
+ public Long rebootGeneration;
+ @JsonProperty("currentRestartGeneration")
+ public Long currentRestartGeneration;
+ @JsonProperty("currentRebootGeneration")
+ public Long currentRebootGeneration;
+ @JsonProperty("vespaVersion")
+ public String vespaVersion;
+ @JsonProperty("wantedVespaVersion")
+ public String wantedVespaVersion;
+ @JsonProperty("currentOsVersion")
+ public String currentOsVersion;
+ @JsonProperty("wantedOsVersion")
+ public String wantedOsVersion;
+ @JsonProperty("currentFirmwareCheck")
+ public Long currentFirmwareCheck;
+ @JsonProperty("wantedFirmwareCheck")
+ public Long wantedFirmwareCheck;
+ @JsonProperty("modelName")
+ public String modelName;
+ @JsonProperty("failCount")
+ public Integer failCount;
+ @JsonProperty("environment")
+ public String environment;
+ @JsonProperty("reservedTo")
+ public String reservedTo;
+ @JsonProperty("type")
+ public String type;
+ @JsonProperty("wantedDockerImage")
+ public String wantedDockerImage;
+ @JsonProperty("currentDockerImage")
+ public String currentDockerImage;
+ @JsonProperty("parentHostname")
+ public String parentHostname;
+ @JsonProperty("wantToRetire")
+ public Boolean wantToRetire;
+ @JsonProperty("wantToDeprovision")
+ public Boolean wantToDeprovision;
+ @JsonProperty("wantToRebuild")
+ public Boolean wantToRebuild;
+ @JsonProperty("orchestratorStatus")
+ public String orchestratorStatus;
+ @JsonProperty("archiveUri")
+ public String archiveUri;
+ @JsonProperty("exclusiveTo")
+ public String exclusiveTo;
+ @JsonProperty("history")
+ public List<Event> history;
+ @JsonProperty("trustStore")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public List<TrustStoreItem> trustStore;
+ @JsonProperty("wireguard")
+ public WireguardKeyWithTimestamp wireguardKeyWithTimestamp;
+
+ // TODO wg: remove separate key and timestamp when all nodes use new keyWithTimestamp
+ @JsonProperty("wireguardPubkey")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public String wireguardPubkey;
+ @JsonProperty("wireguardKeyTimestamp")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public Long wireguardKeyTimestamp;
+
+ @JsonProperty("reports")
+ public Map<String, JsonNode> reports = null;
+
+ @Override
+ public String toString() {
+ return "NodeRepositoryNode{" +
+ "state='" + state + '\'' +
+ ", hostname='" + hostname + '\'' +
+ ", ipAddresses=" + ipAddresses +
+ ", additionalIpAddresses=" + additionalIpAddresses +
+ ", id='" + id + '\'' +
+ ", flavor='" + flavor + '\'' +
+ ", resources=" + resources +
+ ", realResources=" + realResources +
+ ", membership=" + membership +
+ ", owner=" + owner +
+ ", restartGeneration=" + restartGeneration +
+ ", rebootGeneration=" + rebootGeneration +
+ ", currentRestartGeneration=" + currentRestartGeneration +
+ ", currentRebootGeneration=" + currentRebootGeneration +
+ ", vespaVersion='" + vespaVersion + '\'' +
+ ", wantedVespaVersion='" + wantedVespaVersion + '\'' +
+ ", currentOsVersion='" + currentOsVersion + '\'' +
+ ", wantedOsVersion='" + wantedOsVersion + '\'' +
+ ", currentFirmwareCheck=" + currentFirmwareCheck +
+ ", wantedFirmwareCheck=" + wantedFirmwareCheck +
+ ", modelName='" + modelName + '\'' +
+ ", failCount=" + failCount +
+ ", environment='" + environment + '\'' +
+ ", reservedTo='" + reservedTo + '\'' +
+ ", type='" + type + '\'' +
+ ", wantedDockerImage='" + wantedDockerImage + '\'' +
+ ", currentDockerImage='" + currentDockerImage + '\'' +
+ ", parentHostname='" + parentHostname + '\'' +
+ ", wantToRetire=" + wantToRetire +
+ ", wantToDeprovision=" + wantToDeprovision +
+ ", wantToRebuild=" + wantToRebuild +
+ ", orchestratorStatus='" + orchestratorStatus + '\'' +
+ ", archiveUri='" + archiveUri + '\'' +
+ ", exclusiveTo='" + exclusiveTo + '\'' +
+ ", history=" + history +
+ ", trustStore=" + trustStore +
+ ", wireguard=" + wireguardKeyTimestamp +
+ ", reports=" + reports +
+ '}';
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class WireguardKeyWithTimestamp {
+ @JsonProperty("key")
+ public String key;
+ @JsonProperty("timestamp")
+ public long timestamp;
+
+ public WireguardKeyWithTimestamp(@JsonProperty("key") String key, @JsonProperty("timestamp") long timestamp) {
+ this.key = key;
+ this.timestamp = timestamp;
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Owner {
+ @JsonProperty("tenant")
+ public String tenant;
+ @JsonProperty("application")
+ public String application;
+ @JsonProperty("instance")
+ public String instance;
+
+ public String toString() {
+ return "Owner {" +
+ " tenant = " + tenant +
+ " application = " + application +
+ " instance = " + instance +
+ " }";
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Membership {
+ @JsonProperty("clustertype")
+ public String clusterType;
+ @JsonProperty("clusterid")
+ public String clusterId;
+ @JsonProperty("group")
+ public String group;
+ @JsonProperty("index")
+ public int index;
+ @JsonProperty("retired")
+ public boolean retired;
+
+ @Override
+ public String toString() {
+ return "Membership {" +
+ " clusterType = " + clusterType +
+ " clusterId = " + clusterId +
+ " group = " + group +
+ " index = " + index +
+ " retired = " + retired +
+ " }";
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class NodeResources {
+ @JsonProperty
+ public Double vcpu;
+ @JsonProperty
+ public Double memoryGb;
+ @JsonProperty
+ public Double diskGb;
+ @JsonProperty
+ public Double bandwidthGbps;
+ @JsonProperty
+ public String diskSpeed;
+ @JsonProperty
+ public String storageType;
+ @JsonProperty
+ public String architecture;
+ @JsonProperty
+ public Integer gpuCount;
+ @JsonProperty
+ public Double gpuMemoryGb;
+
+ @Override
+ public String toString() {
+ return "NodeResources{" +
+ "vcpu=" + vcpu +
+ ", memoryGb=" + memoryGb +
+ ", diskGb=" + diskGb +
+ ", bandwidthGbps=" + bandwidthGbps +
+ ", diskSpeed='" + diskSpeed + '\'' +
+ ", storageType='" + storageType + '\'' +
+ ", architecture='" + architecture + '\'' +
+ ", gpuCount=" + gpuCount +
+ ", gpuMemoryGb=" + gpuMemoryGb +
+ '}';
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class Event {
+ @JsonProperty
+ public String event;
+ @JsonProperty
+ public String agent;
+ @JsonProperty
+ public Long at;
+
+ @Override
+ public String toString() {
+ return "Event{" +
+ "agent=" + agent +
+ ", event=" + event +
+ ", at=" + at +
+ '}';
+ }
+ }
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class TrustStoreItem {
+ @JsonProperty ("fingerprint")
+ public String fingerprint;
+ @JsonProperty ("expiry")
+ public long expiry;
+
+ public TrustStoreItem(@JsonProperty("fingerprint") String fingerprint, @JsonProperty("expiry") long expiry) {
+ this.fingerprint = fingerprint;
+ this.expiry = expiry;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java
new file mode 100644
index 00000000000..bf83a1a4bdf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java
new file mode 100644
index 00000000000..ccc1f469e1e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java
@@ -0,0 +1,147 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.stream.Stream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * The most basic form of a node repository report on a node.
+ *
+ * <p>This class can be used directly for simple reports, or can be used as a base class for richer reports.
+ *
+ * <p><strong>Subclass requirements</strong>
+ *
+ * <ol>
+ * <li>A subclass must be a Jackson class that can be mapped to {@link JsonNode} with {@link #toJsonNode()},
+ * and from {@link JsonNode} with {@link #fromJsonNode(JsonNode, Class)}.</li>
+ * <li>A subclass must override {@link #updates(BaseReport)} and make sure to return true if
+ * {@code super.updates(current)}.</li>
+ * </ol>
+ *
+ * @author hakonhall
+ */
+// @Immutable
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class BaseReport {
+ /** The time the report was created, in milliseconds since Epoch. */
+ public static final String CREATED_FIELD = "createdMillis";
+ /** The description of the error (implies wanting to fail out node). */
+ public static final String DESCRIPTION_FIELD = "description";
+ /** The type of report, see {@link Type} enum. */
+ public static final String TYPE_FIELD = "type";
+
+ protected static final ObjectMapper mapper = new ObjectMapper();
+
+ private final OptionalLong createdMillis;
+ private final Optional<String> description;
+ private final Type type;
+
+ public enum Type {
+ /** The default type if none given, or not recognized. */
+ UNSPECIFIED,
+ /** A program to be executed once. */
+ ONCE,
+ /** The host has a soft failure and should be parked for manual inspection. */
+ SOFT_FAIL,
+ /** The host has a hard failure and should be given back to siteops. */
+ HARD_FAIL;
+
+ public static Optional<Type> deserialize(String typeString) {
+ return Stream.of(Type.values()).filter(type -> type.name().equalsIgnoreCase(typeString)).findAny();
+ }
+
+ public String serialize() { return name(); }
+ }
+
+ @JsonCreator
+ public BaseReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull,
+ @JsonProperty(DESCRIPTION_FIELD) String descriptionOrNull,
+ @JsonProperty(TYPE_FIELD) Type typeOrNull) {
+ this.createdMillis = createdMillisOrNull == null ? OptionalLong.empty() : OptionalLong.of(createdMillisOrNull);
+ this.description = Optional.ofNullable(descriptionOrNull);
+ this.type = typeOrNull == null ? Type.UNSPECIFIED : typeOrNull;
+ }
+
+ public BaseReport(Long createdMillisOrNull, String descriptionOrNull) {
+ this(createdMillisOrNull, descriptionOrNull, Type.UNSPECIFIED);
+ }
+
+ @JsonGetter(CREATED_FIELD)
+ public final Long getCreatedMillisOrNull() {
+ return createdMillis.isPresent() ? createdMillis.getAsLong() : null;
+ }
+
+ @JsonGetter(DESCRIPTION_FIELD)
+ public final String getDescriptionOrNull() {
+ return description.orElse(null);
+ }
+
+ /** null is returned on UNSPECIFIED to avoid noisy reports. */
+ @JsonGetter(TYPE_FIELD)
+ public final Type getTypeOrNull() {
+ return type == Type.UNSPECIFIED ? null : type;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ * Assume {@code this} is a freshly made report, and {@code current} is the report in the node repository:
+ * Return true iff the node repository should be updated.
+ *
+ * <p>The createdMillis field is ignored in this method (unless it is earlier than {@code current}'s?).
+ */
+ public boolean updates(BaseReport current) {
+ if (this == current) return false;
+ if (this.getClass() != current.getClass()) return true;
+ return !Objects.equals(description, current.description) ||
+ !Objects.equals(type, current.type);
+ }
+
+ /** A variant of {@link #updates(BaseReport)} handling possibly absent reports, whether new or old. */
+ public static <TNEW extends BaseReport, TOLD extends BaseReport>
+ boolean updates2(Optional<TNEW> newReport, Optional<TOLD> oldReport) {
+ if (newReport.isPresent() ^ oldReport.isPresent()) return true;
+ return newReport.map(r -> r.updates(oldReport.get())).orElse(false);
+ }
+
+ public static BaseReport fromJsonNode(JsonNode jsonNode) {
+ return fromJsonNode(jsonNode, BaseReport.class);
+ }
+
+ public static <R extends BaseReport> R fromJsonNode(JsonNode jsonNode, Class<R> jacksonClass) {
+ return uncheck(() -> mapper.treeToValue(jsonNode, jacksonClass));
+ }
+
+ public static BaseReport fromJson(String json) {
+ return fromJson(json, BaseReport.class);
+ }
+
+ public static <R extends BaseReport> R fromJson(String json, Class<R> jacksonClass) {
+ return uncheck(() -> mapper.readValue(json, jacksonClass));
+ }
+
+ /** Returns {@code this} as a {@link JsonNode}. */
+ public JsonNode toJsonNode() {
+ return uncheck(() -> mapper.valueToTree(this));
+ }
+
+ /** Returns {@code this} as a compact JSON string. */
+ public String toJson() {
+ return uncheck(() -> mapper.writeValueAsString(this));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java
new file mode 100644
index 00000000000..2bc8bea013a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java
@@ -0,0 +1,55 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports;
+
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DropDocumentsReport extends BaseReport {
+ private static final String REPORT_ID = "dropDocuments";
+ private static final String DROPPED_AT_FIELD = "droppedAt";
+ private static final String READIED_AT_FIELD = "readiedAt";
+ private static final String STARTED_AT_FIELD = "startedAt";
+
+ private final Long droppedAt;
+ private final Long readiedAt;
+ private final Long startedAt;
+
+ public DropDocumentsReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull,
+ @JsonProperty(DROPPED_AT_FIELD) Long droppedAtOrNull,
+ @JsonProperty(READIED_AT_FIELD) Long readiedAtOrNull,
+ @JsonProperty(STARTED_AT_FIELD) Long startedAtOrNull) {
+ super(createdMillisOrNull, null);
+ this.droppedAt = droppedAtOrNull;
+ this.readiedAt = readiedAtOrNull;
+ this.startedAt = startedAtOrNull;
+ }
+
+ @JsonGetter(DROPPED_AT_FIELD)
+ public Long droppedAt() { return droppedAt; }
+
+ @JsonGetter(READIED_AT_FIELD)
+ public Long readiedAt() { return readiedAt; }
+
+ @JsonGetter(STARTED_AT_FIELD)
+ public Long startedAt() { return startedAt; }
+
+ public DropDocumentsReport withDroppedAt(long droppedAt) {
+ return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt);
+ }
+
+ public DropDocumentsReport withStartedAt(long startedAt) {
+ return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt);
+ }
+
+ public static String reportId() {
+ return REPORT_ID;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java
new file mode 100644
index 00000000000..cd8a1383966
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java
new file mode 100644
index 00000000000..f16f2ca9be3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java
@@ -0,0 +1,36 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+import java.util.List;
+
+/**
+ * Abstraction for communicating with Orchestrator.
+ *
+ * @author bakksjo
+ */
+public interface Orchestrator {
+
+ /**
+ * Suspends a host.
+ *
+ * @throws OrchestratorException if suspend was denied
+ * @throws OrchestratorNotFoundException if host is unknown to the orchestrator
+ */
+ void suspend(String hostName);
+
+ /**
+ * Resumes a host.
+ *
+ * @throws OrchestratorException if resume was denied
+ * @throws OrchestratorNotFoundException if host is unknown to the orchestrator
+ */
+ void resume(String hostName);
+
+ /**
+ * Suspends a list of nodes on a parent.
+ *
+ * @throws OrchestratorException if batch suspend was denied
+ */
+ void suspend(String parentHostName, List<String> hostNames);
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java
new file mode 100644
index 00000000000..5c5c1183ea6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java
@@ -0,0 +1,16 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+
+@SuppressWarnings("serial")
+public class OrchestratorException extends ConvergenceException {
+ /** Creates a transient convergence exception. */
+ public OrchestratorException(String message) {
+ this(message, false);
+ }
+
+ protected OrchestratorException(String message, boolean isError) {
+ super(message, null, isError);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java
new file mode 100644
index 00000000000..614a79719ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java
@@ -0,0 +1,143 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException;
+import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
+import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * @author stiankri
+ * @author bakksjo
+ * @author dybis
+ */
+public class OrchestratorImpl implements Orchestrator {
+ private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName());
+
+ // The server-side Orchestrator has an internal timeout of 10s.
+ //
+ // Note: A 409 has been observed to be returned after 33s in a case possibly involving
+ // zk leader election (which is unfortunate as it is difficult to differentiate between
+ // transient timeouts (do not allow suspend on timeout) and the config server being
+ // permanently down (allow suspend)). For now we'd like to investigate such long
+ // requests so keep the timeout low(er).
+ private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(15);
+
+ // TODO: Find a way to avoid duplicating this (present in orchestrator's services.xml also).
+ private static final String ORCHESTRATOR_PATH_PREFIX = "/orchestrator";
+ static final String ORCHESTRATOR_PATH_PREFIX_HOST_API
+ = ORCHESTRATOR_PATH_PREFIX + "/v1/hosts";
+ static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API
+ = ORCHESTRATOR_PATH_PREFIX + "/v1/suspensions/hosts";
+
+ private final ConfigServerApi configServerApi;
+
+ public OrchestratorImpl(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ }
+
+ @Override
+ public void suspend(final String hostName) {
+ UpdateHostResponse response;
+ try {
+ var params = new ConfigServerApi
+ .Params<UpdateHostResponse>()
+ .setConnectionTimeout(CONNECTION_TIMEOUT)
+ .setRetryPolicy(createRetryPolicyForSuspend());
+ response = configServerApi.put(getSuspendPath(hostName), Optional.empty(), UpdateHostResponse.class, params);
+ } catch (HttpException.NotFoundException n) {
+ throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found");
+ } catch (HttpException e) {
+ throw new OrchestratorException("Failed to suspend " + hostName + ": " + e);
+ } catch (ConnectionException e) {
+ throw ConvergenceException.ofTransient("Failed to suspend " + hostName + ": " + e.getMessage());
+ } catch (RuntimeException e) {
+ throw new RuntimeException("Got error on suspend", e);
+ }
+
+ Optional.ofNullable(response.reason()).ifPresent(reason -> {
+ throw new OrchestratorException(reason.message());
+ });
+ }
+
+ private static ConfigServerApi.RetryPolicy<UpdateHostResponse> createRetryPolicyForSuspend() {
+ return new ConfigServerApi.RetryPolicy<>() {
+ @Override
+ public boolean tryNextConfigServer(URI configServerEndpoint, UpdateHostResponse response) {
+ HostStateChangeDenialReason reason = response.reason();
+ if (reason == null) {
+ return false;
+ }
+
+ // The config server has likely just bootstrapped, so try the next.
+ if ("unknown-service-status".equals(reason.constraintName())) {
+ // Warn for now and until this feature has proven to work well
+ logger.warning("Config server at [" + configServerEndpoint +
+ "] failed with transient error (will try next): " +
+ reason.message());
+
+ return true;
+ }
+
+ return false;
+ }
+ };
+ }
+
+ @Override
+ public void suspend(String parentHostName, List<String> hostNames) {
+ final BatchOperationResult batchOperationResult;
+ try {
+ var params = new ConfigServerApi.Params<BatchOperationResult>().setConnectionTimeout(CONNECTION_TIMEOUT);
+ String hostnames = String.join("&hostname=", hostNames);
+ String url = String.format("%s/%s?hostname=%s", ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API,
+ parentHostName, hostnames);
+ batchOperationResult = configServerApi.put(url, Optional.empty(), BatchOperationResult.class, params);
+ } catch (HttpException e) {
+ throw new OrchestratorException("Failed to batch suspend for " + parentHostName + ": " + e);
+ } catch (ConnectionException e) {
+ throw ConvergenceException.ofTransient("Failed to batch suspend for " + parentHostName + ": " + e.getMessage());
+ } catch (RuntimeException e) {
+ throw new RuntimeException("Got error on batch suspend for " + parentHostName + ", with nodes " + hostNames, e);
+ }
+
+ batchOperationResult.getFailureReason().ifPresent(reason -> {
+ throw new OrchestratorException(reason);
+ });
+ }
+
+ @Override
+ public void resume(final String hostName) {
+ UpdateHostResponse response;
+ try {
+ String path = getSuspendPath(hostName);
+ response = configServerApi.delete(path, UpdateHostResponse.class);
+ } catch (HttpException.NotFoundException n) {
+ throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found");
+ } catch (HttpException e) {
+ throw new OrchestratorException("Failed to resume " + hostName + ": " + e);
+ } catch (ConnectionException e) {
+ throw ConvergenceException.ofTransient("Failed to resume " + hostName + ": " + e.getMessage());
+ } catch (RuntimeException e) {
+ throw new RuntimeException("Got error on resume", e);
+ }
+
+ Optional.ofNullable(response.reason()).ifPresent(reason -> {
+ throw new OrchestratorException(reason.message());
+ });
+ }
+
+ private String getSuspendPath(String hostName) {
+ return ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended";
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java
new file mode 100644
index 00000000000..8025eb8df93
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java
@@ -0,0 +1,9 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+@SuppressWarnings("serial")
+public class OrchestratorNotFoundException extends OrchestratorException {
+ public OrchestratorNotFoundException(String message) {
+ super(message, true);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java
new file mode 100644
index 00000000000..6c89fbce90b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java
new file mode 100644
index 00000000000..af925db8b4e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java
new file mode 100644
index 00000000000..a82a82e56b0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java
@@ -0,0 +1,32 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+/**
+ * The healthiness of a remote Vespa server based on REST API
+ *
+ * @author hakon
+ */
+public enum HealthCode {
+ DOWN("down"),
+ INITIALIZING("initializing"),
+ UP("up");
+
+ private final String code;
+
+ HealthCode(String code) {
+ this.code = code;
+ }
+
+ public static HealthCode fromString(String code) {
+ return HealthCode.valueOf(code.toUpperCase());
+ }
+
+ public String asString() {
+ return code;
+ }
+
+ @Override
+ public String toString() {
+ return asString();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java
new file mode 100644
index 00000000000..0887637d5a1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java
@@ -0,0 +1,12 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+/**
+ * The /state/v1 REST API of the config server
+ *
+ * @author hakon
+ */
+public interface State {
+ /** Issue GET on /state/v1/health */
+ HealthCode getHealth();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java
new file mode 100644
index 00000000000..2471069cb4a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java
@@ -0,0 +1,29 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException;
+import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse;
+
+/**
+ * @author hakon
+ */
+public class StateImpl implements State {
+ private final ConfigServerApi configServerApi;
+
+ public StateImpl(ConfigServerApi configServerApi) {
+ this.configServerApi = configServerApi;
+ }
+
+ @Override
+ public HealthCode getHealth() {
+ try {
+ HealthResponse response = configServerApi.get("/state/v1/health", HealthResponse.class);
+ return HealthCode.fromString(response.status.code);
+ } catch (ConnectionException | HttpException e) {
+ return HealthCode.DOWN;
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java
new file mode 100644
index 00000000000..d0b94324941
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java
@@ -0,0 +1,36 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Response from /state/v1/health
+ *
+ * @author hakon
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class HealthResponse {
+ @JsonProperty("status")
+ public Status status = new Status();
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Status {
+ @JsonProperty("code")
+ public String code = "down";
+
+ @Override
+ public String toString() {
+ return "Status{" +
+ "code='" + code + '\'' +
+ '}';
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "HealthResponse{" +
+ "status=" + status +
+ '}';
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java
new file mode 100644
index 00000000000..fd237ec6cb4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java
new file mode 100644
index 00000000000..f6f9ebd79e9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java
@@ -0,0 +1,85 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A Podman container.
+ *
+ * @author mpolden
+ */
+public class Container extends PartialContainer {
+
+ private final String hostname;
+ private final ContainerResources resources;
+ private final int conmonPid;
+ private final List<Network> networks;
+
+ public Container(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image,
+ Map<String, String> labels, int pid, int conmonPid, String hostname,
+ ContainerResources resources, List<Network> networks, boolean managed) {
+ super(id, name, createdAt, state, imageId, image, labels, pid, managed);
+ this.hostname = Objects.requireNonNull(hostname);
+ this.resources = Objects.requireNonNull(resources);
+ this.conmonPid = conmonPid;
+ this.networks = List.copyOf(Objects.requireNonNull(networks));
+ }
+
+ /** The hostname of this, if any */
+ public String hostname() {
+ return hostname;
+ }
+
+ /** Resource limits for this*/
+ public ContainerResources resources() {
+ return resources;
+ }
+
+ /** Pid of the conmon process for this container */
+ public int conmonPid() {
+ return conmonPid;
+ }
+
+ /** The networks used by this */
+ public List<Network> networks() {
+ return networks;
+ }
+
+ @Override
+ public String toString() {
+ return "Container{" +
+ "hostname='" + hostname + '\'' +
+ ", resources=" + resources +
+ ", conmonPid=" + conmonPid +
+ ", networks=" + networks +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) return false;
+ Container that = (Container) o;
+ return conmonPid == that.conmonPid && hostname.equals(that.hostname) && resources.equals(that.resources) && networks.equals(that.networks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks);
+ }
+
+ /** The network of a container */
+ public record Network(String name, String ipv4Address) {
+ public Network {
+ Objects.requireNonNull(name);
+ Objects.requireNonNull(ipv4Address);
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java
new file mode 100644
index 00000000000..26c3ba2a45b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java
@@ -0,0 +1,70 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.image.Image;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Interface for a container engine, such as Docker or Podman.
+ *
+ * @author mpolden
+ */
+public interface ContainerEngine {
+
+ /**
+ * Create a new container
+ * @return ContainerData that can be used to write files inside container
+ */
+ ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources);
+
+ /** Start a created container */
+ void startContainer(NodeAgentContext context);
+
+ /** Update an existing container with new resources */
+ void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources);
+
+ /** Remove given container. The container will be stopped if necessary */
+ void removeContainer(TaskContext context, PartialContainer container);
+
+ /** Get container for given context */
+ Optional<Container> getContainer(NodeAgentContext context);
+
+ /** Returns all containers known by this */
+ List<PartialContainer> listContainers(TaskContext context);
+
+ /** Returns the network interface used by container in given context */
+ String networkInterface(NodeAgentContext context);
+
+ /** Execute command inside container as given user. Ignores non-zero exit code */
+ CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command);
+
+ /** Execute command inside the container's network namespace. Throws on non-zero exit code */
+ CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command);
+
+ default CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command) {
+ return executeInNetworkNamespace(context, new CommandLine.Options(), command);
+ }
+
+ /** Download given image */
+ void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials);
+
+ /** Returns whether given image is already downloaded */
+ boolean hasImage(TaskContext context, DockerImage image);
+
+ /** Remove image by id */
+ void removeImage(TaskContext context, String id);
+
+ /** Returns images available in this */
+ List<Image> listImages(TaskContext context);
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java
new file mode 100644
index 00000000000..5a800efcbd0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java
@@ -0,0 +1,36 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+//
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import java.util.Objects;
+
+/**
+ * The ID of a container.
+ *
+ * @author hakon
+ */
+public class ContainerId {
+ private final String id;
+
+ public ContainerId(String id) {
+ this.id = Objects.requireNonNull(id, "id cannot be null");
+ }
+
+ @Override
+ public String toString() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ContainerId that = (ContainerId) o;
+ return id.equals(that.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java
new file mode 100644
index 00000000000..c504e38575c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java
@@ -0,0 +1,59 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import java.util.Objects;
+import java.util.regex.Pattern;
+
+/**
+ * Type-safe value wrapper for docker container names.
+ *
+ * @author bakksjo
+ */
+public class ContainerName implements Comparable<ContainerName> {
+
+ private static final Pattern LEGAL_CONTAINER_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9_-]+$");
+ private final String name;
+
+ public ContainerName(final String name) {
+ this.name = Objects.requireNonNull(name);
+ if (! LEGAL_CONTAINER_NAME_PATTERN.matcher(name).matches()) {
+ throw new IllegalArgumentException("Illegal container name: " + name + ". Must match " +
+ LEGAL_CONTAINER_NAME_PATTERN.pattern());
+ }
+ }
+
+ public String asString() {
+ return name;
+ }
+
+ public static ContainerName fromHostname(final String hostName) {
+ return new ContainerName(hostName.split("\\.", 2)[0]);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ContainerName other)) {
+ return false;
+ }
+
+ return Objects.equals(name, other.name);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + " {"
+ + " name=" + name
+ + " }";
+ }
+
+ @Override
+ public int compareTo(ContainerName o) {
+ return name.compareTo(o.name);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java
new file mode 100644
index 00000000000..a737b049e11
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java
@@ -0,0 +1,29 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import java.util.Objects;
+
+/**
+ * Container network modes supported by node-admin.
+ *
+ * @author hakon
+ */
+public enum ContainerNetworkMode {
+
+ /** Network Prefix-Translated networking. */
+ NPT("vespa-bridge"),
+
+ /** A host running a single container in the host network namespace. */
+ HOST_NETWORK("host");
+
+ private final String networkName;
+
+ ContainerNetworkMode(String networkName) {
+ this.networkName = Objects.requireNonNull(networkName);
+ }
+
+ public String networkName() {
+ return networkName;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java
new file mode 100644
index 00000000000..ce26f8e69e7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java
@@ -0,0 +1,153 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImageDownloader;
+import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.nio.file.FileSystem;
+import java.time.Duration;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * High-level interface for container operations. Code managing containers should use this and not
+ * {@link ContainerEngine} directly.
+ *
+ * @author hakonhall
+ * @author mpolden
+ */
+public class ContainerOperations {
+
+ private final ContainerEngine containerEngine;
+ private final ContainerImageDownloader imageDownloader;
+ private final ContainerImagePruner imagePruner;
+ private final ContainerStatsCollector containerStatsCollector;
+
+ public ContainerOperations(ContainerEngine containerEngine, Cgroup cgroup, FileSystem fileSystem, Timer timer) {
+ this.containerEngine = Objects.requireNonNull(containerEngine);
+ this.imageDownloader = new ContainerImageDownloader(containerEngine, timer);
+ this.imagePruner = new ContainerImagePruner(containerEngine, timer);
+ this.containerStatsCollector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem);
+ }
+
+ public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) {
+ return containerEngine.createContainer(context, containerResources);
+ }
+
+ public void startContainer(NodeAgentContext context) {
+ containerEngine.startContainer(context);
+ }
+
+ public void removeContainer(NodeAgentContext context, Container container) {
+ containerEngine.removeContainer(context, container);
+ }
+
+ public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
+ containerEngine.updateContainer(context, containerId, containerResources);
+ }
+
+ public Optional<Container> getContainer(NodeAgentContext context) {
+ return containerEngine.getContainer(context);
+ }
+
+ /** Pull image asynchronously. Returns true if image is still downloading and false if download is complete */
+ public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentialsProvider credentialsProvider) {
+ return !imageDownloader.get(context, dockerImage, credentialsProvider);
+ }
+
+ /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */
+ public CommandResult executeCommandInContainer(NodeAgentContext context, UnixUser user, String... command) {
+ return executeCommandInContainer(context, user, CommandLine.DEFAULT_TIMEOUT, command);
+ }
+
+ /** Execute command inside container identified by given context. Does NOT throw on non-zero exit code */
+ public CommandResult executeCommandInContainer(NodeAgentContext context, UnixUser user, Duration timeout, String... command) {
+ return containerEngine.execute(context, user, timeout, command);
+ }
+
+ /** Execute command in inside containers network namespace, identified by given context. Throws on non-zero exit code */
+ public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) {
+ return executeCommandInNetworkNamespace(context, new CommandLine.Options(), command);
+ }
+
+ public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) {
+ return containerEngine.executeInNetworkNamespace(context, options, command);
+ }
+
+ /** Resume node. Resuming a node means that it is ready to receive traffic */
+ public String resumeNode(NodeAgentContext context) {
+ return executeNodeCtlInContainer(context, "resume");
+ }
+
+ /**
+ * Suspend node and return output. Suspending a node means the node should be taken temporarily offline,
+ * such that maintenance of the node can be done (upgrading, rebooting, etc).
+ */
+ public String suspendNode(NodeAgentContext context) {
+ return executeNodeCtlInContainer(context, "suspend");
+ }
+
+ /** Restart Vespa inside container. Same as running suspend, stop, start and resume */
+ public String restartVespa(NodeAgentContext context) {
+ return executeNodeCtlInContainer(context, "restart-vespa");
+ }
+
+ /** Start Vespa inside container */
+ public String startServices(NodeAgentContext context) {
+ return executeNodeCtlInContainer(context, "start");
+ }
+
+ /** Stop Vespa inside container */
+ public String stopServices(NodeAgentContext context) {
+ return executeNodeCtlInContainer(context, "stop");
+ }
+
+ /** Get container statistics */
+ public Optional<ContainerStats> getContainerStats(NodeAgentContext context) {
+ String iface = containerEngine.networkInterface(context);
+ return getContainer(context).flatMap(container -> containerStatsCollector.collect(context, container.id(), container.pid(), iface));
+ }
+
+ /** Returns true if no containers managed by node-admin are running */
+ public boolean noManagedContainersRunning(TaskContext context) {
+ return containerEngine.listContainers(context).stream()
+ .filter(PartialContainer::managed)
+ .noneMatch(container -> container.state() == Container.State.running);
+ }
+
+ /**
+ * Stop and remove all managed containers except the given ones
+ *
+ * @return true if any containers were removed
+ */
+ public boolean retainManagedContainers(TaskContext context, Set<ContainerName> containerNames) {
+ return containerEngine.listContainers(context).stream()
+ .filter(PartialContainer::managed)
+ .filter(container -> !containerNames.contains(container.name()))
+ .peek(container -> containerEngine.removeContainer(context, container))
+ .count() > 0;
+ }
+
+ /** Deletes the local images that are currently not in use by any container and not recently used. */
+ public boolean deleteUnusedContainerImages(TaskContext context, List<DockerImage> excludes, Duration minImageAgeToDelete) {
+ List<String> excludedRefs = excludes.stream().map(DockerImage::asString).toList();
+ return imagePruner.removeUnusedImages(context, excludedRefs, minImageAgeToDelete);
+ }
+
+ private String executeNodeCtlInContainer(NodeAgentContext context, String program) {
+ String[] command = new String[] {context.paths().underVespaHome("bin/vespa-nodectl").pathInContainer(), program};
+ return executeCommandInContainer(context, context.users().vespa(), command).getOutput();
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java
new file mode 100644
index 00000000000..05398e90053
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java
@@ -0,0 +1,135 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class ContainerResources {
+
+ public static final ContainerResources UNLIMITED = ContainerResources.from(0, 0, 0);
+ public static final int CPU_PERIOD_US = 100_000; // 100 ms
+
+ /**
+ * Hard limit on container's CPU usage: Implemented using Completely Fair Scheduler (CFS) by allocating a given
+ * time within a given period, Container's processes are not bound to any specific CPU, which may create significant
+ * performance degradation as processes are scheduled on another CPU after exhausting the quota.
+ */
+ private final double cpus;
+
+ /**
+ * Soft limit on container's CPU usage: When plenty of CPU cycles are available, all containers use as much
+ * CPU as they need. It prioritizes container CPU resources for the available CPU cycles.
+ * It does not guarantee or reserve any specific CPU access.
+ */
+ private final int cpuShares;
+
+ /** The maximum amount, in bytes, of memory the container can use. */
+ private final long memoryBytes;
+
+ public ContainerResources(double cpus, int cpuShares, long memoryBytes) {
+ this.cpus = cpus;
+ this.cpuShares = cpuShares;
+ this.memoryBytes = memoryBytes;
+
+ if (cpus < 0)
+ throw new IllegalArgumentException("CPUs must be a positive number or 0 for unlimited, was " + cpus);
+ if (cpuShares != 0 && (cpuShares < 2 || cpuShares > 262_144))
+ throw new IllegalArgumentException("CPU shares must be a positive integer in [2, 262144] or 0 for unlimited, was " + cpuShares);
+ if (memoryBytes < 0)
+ throw new IllegalArgumentException("memoryBytes must be a positive integer or 0 for unlimited, was " + memoryBytes);
+ }
+
+ /**
+ * Create container resources from required fields.
+ *
+ * @param maxVcpu the amount of vcpu that allocation policies should allocate exclusively to this container.
+ * This is a hard upper limit. To allow an unlimited amount use 0.
+ * @param minVcpu the minimal amount of vcpu dedicated to this container.
+ * To avoid dedicating any cpu at all, use 0.
+ * @param memoryGb the amount of memory that allocation policies should allocate to this container.
+ * This is a hard upper limit. To allow the container to allocate an unlimited amount use 0.
+ * @return the container resources encapsulating the parameters
+ */
+ public static ContainerResources from(double maxVcpu, double minVcpu, double memoryGb) {
+ return new ContainerResources(maxVcpu,
+ (int) Math.round(32 * minVcpu),
+ (long) ((1L << 30) * memoryGb));
+ }
+
+ public double cpus() {
+ return cpus;
+ }
+
+ /** Returns the CFS CPU quota per {@link #cpuPeriod()}, or -1 if disabled. */
+ public int cpuQuota() {
+ return cpus > 0 ? (int) (cpus * CPU_PERIOD_US) : -1;
+ }
+
+ /** Duration (in µs) of a single period used as the basis for process scheduling */
+ public int cpuPeriod() {
+ return CPU_PERIOD_US;
+ }
+
+ public int cpuShares() {
+ return cpuShares;
+ }
+
+ public long memoryBytes() {
+ return memoryBytes;
+ }
+
+ /** Returns true iff the memory component(s) of between <code>this</code> and <code>other</code> are equal */
+ public boolean equalsMemory(ContainerResources other) {
+ return memoryBytes == other.memoryBytes;
+ }
+
+ /** Returns true iff the CPU component(s) of between <code>this</code> and <code>other</code> are equal */
+ public boolean equalsCpu(ContainerResources other) {
+ return Math.abs(other.cpus - cpus) < 0.0001 &&
+ // When using CGroups V2, CPU shares (range [2, 262144]) is mapped to CPU weight (range [1, 10000]),
+ // because there are ~26.2 shares/weight, we must allow for small deviation in cpuShares
+ // when comparing ContainerResources created from NodeResources vs one created from reading the
+ // CGroups weight file
+ Math.abs(cpuShares - other.cpuShares) < 28;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ContainerResources that = (ContainerResources) o;
+ return equalsMemory(that) && equalsCpu(that);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cpus, cpuShares, memoryBytes);
+ }
+
+
+ /** Returns only the memory component(s) of {@link #toString()} */
+ public String toStringMemory() {
+ return (memoryBytes > 0 ? memoryBytes + "B" : "unlimited") + " memory";
+ }
+
+ /** Returns only the CPU component(s) of {@link #toString()} */
+ public String toStringCpu() {
+ return (cpus > 0 ? String.format("%.2f", cpus) : "unlimited") +" CPUs, " +
+ (cpuShares > 0 ? cpuShares : "unlimited") + " CPU Shares";
+ }
+
+ @Override
+ public String toString() {
+ return toStringCpu() + ", " + toStringMemory();
+ }
+
+ public ContainerResources withMemoryBytes(long memoryBytes) {
+ return new ContainerResources(cpus, cpuShares, memoryBytes);
+ }
+
+ public ContainerResources withUnlimitedCpus() {
+ return new ContainerResources(0, 0, memoryBytes);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java
new file mode 100644
index 00000000000..9c1b8db144c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java
@@ -0,0 +1,94 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import ai.vespa.validation.Validation;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * CPU, GPU, memory and network statistics collected from a container.
+ *
+ * @author freva
+ */
+public record ContainerStats(Map<String, NetworkStats> networks,
+ MemoryStats memoryStats,
+ CpuStats cpuStats,
+ List<GpuStats> gpuStats) {
+
+ public ContainerStats(Map<String, NetworkStats> networks, MemoryStats memoryStats, CpuStats cpuStats, List<GpuStats> gpuStats) {
+ this.networks = Collections.unmodifiableMap(new LinkedHashMap<>(Objects.requireNonNull(networks)));
+ this.memoryStats = Objects.requireNonNull(memoryStats);
+ this.cpuStats = Objects.requireNonNull(cpuStats);
+ this.gpuStats = List.copyOf(Objects.requireNonNull(gpuStats));
+ }
+
+ /**
+ * Statistics for network usage
+ *
+ * @param rxBytes received bytes
+ * @param rxDropped received bytes, which were dropped
+ * @param rxErrors received errors
+ * @param txBytes transmitted bytes
+ * @param txDropped transmitted bytes, which were dropped
+ * @param txErrors transmission errors
+ */
+ public record NetworkStats(long rxBytes, long rxDropped, long rxErrors, long txBytes, long txDropped, long txErrors) {}
+
+ /**
+ * Statistics for memory usage
+ *
+ * @param cache memory used by cache in bytes
+ * @param usage memory usage in bytes
+ * @param limit memory limit in bytes
+ * @param sock network transmission buffers in bytes
+ * @param slab in-kernel data structures in bytes
+ * @param slabReclaimable part of "slab" that might be reclaimed in bytes
+ * @param anon anonymous mappings in bytes
+ */
+ public record MemoryStats(long cache, long usage, long limit, long sock, long slab, long slabReclaimable, long anon) {
+ public MemoryStats(long cache, long usage, long limit) { this(cache, usage, limit, 0, 0, 0, 0); }
+ }
+
+ /**
+ * Statistics for CPU usage
+ *
+ * @param onlineCpus CPU cores
+ * @param systemCpuUsage Total CPU time (in µs) spent executing all the processes on this host
+ * @param totalUsage Total CPU time (in µs) spent running all the processes in this container
+ * @param usageInKernelMode Total CPU time (in µs) spent in kernel mode while executing processes in this container
+ * @param throttledTime Total CPU time (in µs) processes in this container were throttled for
+ * @param throttlingActivePeriods Number of periods with throttling enabled for this container
+ * @param throttledPeriods Number of periods this container hit the throttling limit
+ */
+ public record CpuStats(int onlineCpus,
+ long systemCpuUsage,
+ long totalUsage,
+ long usageInKernelMode,
+ long throttledTime,
+ long throttlingActivePeriods,
+ long throttledPeriods) {}
+
+ /**
+ * GPU statistics
+ *
+ * @param deviceNumber GPU device number
+ * @param loadPercentage Load/utilization in %
+ * @param memoryTotalBytes Total memory, in bytes
+ * @param memoryUsedBytes Memory used, in bytes
+ */
+ public record GpuStats(int deviceNumber, int loadPercentage, long memoryTotalBytes, long memoryUsedBytes) {
+
+ public GpuStats {
+ Validation.requireAtLeast(deviceNumber, "deviceNumber", 0);
+ Validation.requireAtLeast(loadPercentage, "loadPercentage", 0);
+ Validation.requireAtLeast(memoryTotalBytes, "memoryTotalBytes", 0L);
+ Validation.requireAtLeast(memoryUsedBytes, "memoryUsedBytes", 0L);
+ }
+
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java
new file mode 100644
index 00000000000..aa6f8d8f5f6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java
@@ -0,0 +1,168 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup;
+import com.yahoo.vespa.hosted.node.admin.cgroup.CpuController;
+import com.yahoo.vespa.hosted.node.admin.cgroup.Size;
+import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * Collects CPU, GPU, memory and network statistics for a container.
+ *
+ * Uses same approach as runc: https://github.com/opencontainers/runc/tree/master/libcontainer/cgroups/fs
+ *
+ * @author mpolden
+ */
+class ContainerStatsCollector {
+
+ private final ContainerEngine containerEngine;
+ private final FileSystem fileSystem;
+ private final Cgroup rootCgroup;
+ private final int onlineCpus;
+
+ ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem) {
+ this(containerEngine, rootCgroup, fileSystem, Runtime.getRuntime().availableProcessors());
+ }
+
+ ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem, int onlineCpus) {
+ this.containerEngine = Objects.requireNonNull(containerEngine);
+ this.fileSystem = Objects.requireNonNull(fileSystem);
+ this.rootCgroup = Objects.requireNonNull(rootCgroup);
+ this.onlineCpus = onlineCpus;
+ }
+
+ /** Collect statistics for given container ID and PID */
+ public Optional<ContainerStats> collect(NodeAgentContext context, ContainerId containerId, int pid, String iface) {
+ try {
+ ContainerStats.CpuStats cpuStats = collectCpuStats(containerId);
+ ContainerStats.MemoryStats memoryStats = collectMemoryStats(containerId);
+ Map<String, ContainerStats.NetworkStats> networkStats = Map.of(iface, collectNetworkStats(iface, pid));
+ List<ContainerStats.GpuStats> gpuStats = collectGpuStats(context);
+ return Optional.of(new ContainerStats(networkStats, memoryStats, cpuStats, gpuStats));
+ } catch (NoSuchFileException ignored) {
+ return Optional.empty(); // Container disappeared while we collected stats
+ } catch (UncheckedIOException e) {
+ if (e.getCause() != null && e.getCause() instanceof NoSuchFileException)
+ return Optional.empty();
+ throw e;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private List<ContainerStats.GpuStats> collectGpuStats(NodeAgentContext context) {
+ boolean hasGpu = Files.exists(fileSystem.getPath("/dev/nvidia0"));
+ if (!hasGpu) {
+ return List.of();
+ }
+ Stream<String> lines = containerEngine.execute(context, UnixUser.ROOT, Duration.ofSeconds(5),
+ "nvidia-smi",
+ "--query-gpu=index,utilization.gpu,memory.total,memory.free",
+ "--format=csv,noheader,nounits")
+ .getOutputLinesStream();
+ return lines.map(ContainerStatsCollector::parseGpuStats).toList();
+ }
+
+ private static ContainerStats.GpuStats parseGpuStats(String s) {
+ String[] fields = fields(s, ",\\s*");
+ if (fields.length < 4) throw new IllegalArgumentException("Could not parse GPU stats from '" + s + "'");
+ int deviceNumber = Integer.parseInt(fields[0]);
+ int loadPercentage = Integer.parseInt(fields[1]);
+ long mega = 2 << 19;
+ long memoryTotalBytes = Long.parseLong(fields[2]) * mega;
+ long memoryFreeBytes = Long.parseLong(fields[3]) * mega;
+ long memoryUsedBytes = memoryTotalBytes - memoryFreeBytes;
+ return new ContainerStats.GpuStats(deviceNumber, loadPercentage, memoryTotalBytes, memoryUsedBytes);
+ }
+
+ private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException {
+ Map<CpuController.StatField, Long> cpuStats = rootCgroup.resolveContainer(containerId).cpu().readStats();
+ return new ContainerStats.CpuStats(onlineCpus,
+ systemCpuUsage(),
+ cpuStats.get(CpuController.StatField.TOTAL_USAGE_USEC),
+ cpuStats.get(CpuController.StatField.SYSTEM_USAGE_USEC),
+ cpuStats.get(CpuController.StatField.THROTTLED_TIME_USEC),
+ cpuStats.get(CpuController.StatField.TOTAL_PERIODS),
+ cpuStats.get(CpuController.StatField.THROTTLED_PERIODS));
+ }
+
+ private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException {
+ MemoryController memoryController = rootCgroup.resolveContainer(containerId).memory();
+ Size max = memoryController.readMax();
+ long memoryUsageInBytes = memoryController.readCurrent().value();
+ var stats = memoryController.readStat();
+ return new ContainerStats.MemoryStats(
+ stats.file().value(), memoryUsageInBytes, max.isMax() ? -1 : max.value(),
+ stats.sock().value(), stats.slab().value(), stats.slabReclaimable().value(), stats.anon().value());
+ }
+
+ private ContainerStats.NetworkStats collectNetworkStats(String iface, int containerPid) throws IOException {
+ for (var line : Files.readAllLines(netDevPath(containerPid))) {
+ String[] fields = fields(line.trim());
+ if (fields.length < 17 || !fields[0].equals(iface + ":")) continue;
+
+ long rxBytes = Long.parseLong(fields[1]);
+ long rxErrors = Long.parseLong(fields[3]);
+ long rxDropped = Long.parseLong(fields[4]);
+
+ long txBytes = Long.parseLong(fields[9]);
+ long txErrors = Long.parseLong(fields[11]);
+ long txDropped = Long.parseLong(fields[12]);
+
+ return new ContainerStats.NetworkStats(rxBytes, rxDropped, rxErrors, txBytes, txDropped, txErrors);
+ }
+ throw new IllegalArgumentException("No statistics found for interface " + iface);
+ }
+
+ /** Returns total CPU time in µs spent executing all the processes on this host */
+ private long systemCpuUsage() throws IOException {
+ long ticks = parseLong(Files.readAllLines(fileSystem.getPath("/proc/stat")), "cpu");
+ return userHzToMicroSeconds(ticks);
+ }
+
+ private long parseLong(List<String> lines, String fieldName) {
+ long value = 0;
+ for (var line : lines) {
+ String[] fields = fields(line);
+ if (fields.length < 2 || !fields[0].equals(fieldName)) continue;
+ for (int i = 1; i < fields.length; i++) {
+ value += Long.parseLong(fields[i]);
+ }
+ break;
+ }
+ return value;
+ }
+
+ private Path netDevPath(int containerPid) {
+ return fileSystem.getPath("/proc/" + containerPid + "/net/dev");
+ }
+
+ static long userHzToMicroSeconds(long ticks) {
+ // Ideally we would read this from _SC_CLK_TCK, but then we need JNI. However, in practice this is always 100 on x86 Linux
+ return ticks * 10_000;
+ }
+
+ private static String[] fields(String s) {
+ return fields(s, "\\s+");
+ }
+
+ private static String[] fields(String s, String regex) {
+ return s.trim().split(regex);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java
new file mode 100644
index 00000000000..c9310897df9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java
@@ -0,0 +1,139 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A partial container, containing only fields returned by a container list command such as 'podman ps'.
+ *
+ * @author mpolden
+ */
+public class PartialContainer {
+
+ private final ContainerId id;
+ private final ContainerName name;
+ private final Instant createdAt;
+ private final State state;
+ private final String imageId;
+ private final DockerImage image;
+ private final Map<String, String> labels;
+ private final int pid;
+ private final boolean managed;
+
+ public PartialContainer(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId,
+ DockerImage image, Map<String, String> labels, int pid, boolean managed) {
+ this.id = Objects.requireNonNull(id);
+ this.name = Objects.requireNonNull(name);
+ this.createdAt = Objects.requireNonNull(createdAt);
+ this.state = Objects.requireNonNull(state);
+ this.imageId = Objects.requireNonNull(imageId);
+ this.image = Objects.requireNonNull(image);
+ this.labels = Map.copyOf(Objects.requireNonNull(labels));
+ this.pid = pid;
+ this.managed = managed;
+ }
+
+ /** A unique identifier for this. Typically generated by the container engine */
+ public ContainerId id() {
+ return id;
+ }
+
+ /** The given name of this */
+ public ContainerName name() {
+ return name;
+ }
+
+ /** Timestamp when this container was created */
+ public Instant createdAt() {
+ return createdAt;
+ }
+
+ /** Current state of this */
+ public State state() {
+ return state;
+ }
+
+ /** A unique identifier for the image in use by this */
+ public String imageId() {
+ return imageId;
+ }
+
+ /** The image in use by this */
+ public DockerImage image() {
+ return image;
+ }
+
+ /** The labels set on this */
+ public Map<String, String> labels() {
+ return labels;
+ }
+
+ /** The PID of this */
+ public int pid() {
+ return pid;
+ }
+
+ /** Returns whether this container is managed by node-admin */
+ public boolean managed() {
+ return managed;
+ }
+
+ /** Returns the value of given label key */
+ public String label(String key) {
+ String labelValue = labels.get(key);
+ if (labelValue == null) throw new IllegalArgumentException("No such label '" + key + "'");
+ return labelValue;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PartialContainer that = (PartialContainer) o;
+ return pid == that.pid && managed == that.managed && id.equals(that.id) && name.equals(that.name) && createdAt.equals(that.createdAt) && state == that.state && imageId.equals(that.imageId) && image.equals(that.image) && labels.equals(that.labels);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, name, createdAt, state, imageId, image, labels, pid, managed);
+ }
+
+ /** The state of a container */
+ public enum State {
+
+ unknown,
+ configured,
+ created,
+ running,
+ stopped,
+ paused,
+ exited,
+ removing,
+ stopping;
+
+ public boolean isRunning() {
+ return this == running;
+ }
+
+ public static Container.State from(String state) {
+ switch (state) {
+ case "unknown": return unknown;
+ case "configured": return configured;
+ case "created": return created;
+ case "running": return running;
+ case "stopped": return stopped;
+ case "paused": return paused;
+ case "exited": return exited;
+ case "removing": return removing;
+ case "stopping": return stopping;
+ }
+ throw new IllegalArgumentException("Invalid state '" + state + "'");
+ }
+
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java
new file mode 100644
index 00000000000..7a5f46dab74
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java
@@ -0,0 +1,25 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import java.util.Objects;
+
+/**
+ * Credentials for a container registry server.
+ *
+ * @author mpolden
+ */
+public record RegistryCredentials(String username, String password) {
+
+ public static final RegistryCredentials none = new RegistryCredentials("", "");
+
+ public RegistryCredentials {
+ Objects.requireNonNull(username);
+ Objects.requireNonNull(password);
+ }
+
+ @Override
+ public String toString() {
+ return "registry credentials [username=" + username + ",password=<hidden>]";
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java
new file mode 100644
index 00000000000..8711227058a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+/**
+ * Interface for retrieving credentials for a container registry.
+ *
+ * @author mpolden
+ */
+public interface RegistryCredentialsProvider {
+
+ RegistryCredentials get();
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java
new file mode 100644
index 00000000000..ab2adc061fd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java
@@ -0,0 +1,66 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Download a container image asynchronously.
+ *
+ * @author mpolden
+ */
+public class ContainerImageDownloader {
+
+ private static final Logger LOG = Logger.getLogger(ContainerImageDownloader.class.getName());
+
+ private final ContainerEngine containerEngine;
+ private final Timer timer;
+
+ private final ExecutorService executorService = Executors.newSingleThreadExecutor(
+ new DaemonThreadFactory("container-image-downloader")); // Download one image at a time
+ private final Set<DockerImage> pendingDownloads = Collections.synchronizedSet(new HashSet<>());
+
+ public ContainerImageDownloader(ContainerEngine containerEngine, Timer timer) {
+ this.containerEngine = Objects.requireNonNull(containerEngine);
+ this.timer = Objects.requireNonNull(timer);
+ }
+
+ /**
+ * Download given container image.
+ *
+ * @return true if the image download has completed.
+ */
+ public boolean get(TaskContext context, DockerImage image, RegistryCredentialsProvider credentialsProvider) {
+ if (pendingDownloads.contains(image)) return false;
+ if (containerEngine.hasImage(context, image)) return true;
+ executorService.submit(() -> {
+ try {
+ Instant start = timer.currentTime();
+ containerEngine.pullImage(context, image, credentialsProvider.get());
+ LOG.log(Level.INFO, "Downloaded container image " + image + " in " + Duration.between(start, timer.currentTime()));
+ } catch (RuntimeException e) {
+ LOG.log(Level.SEVERE, "Failed to download container image " + image, e);
+ } finally {
+ pendingDownloads.remove(image);
+ }
+ });
+ pendingDownloads.add(image);
+ return false;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java
new file mode 100644
index 00000000000..51bf238fa67
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java
@@ -0,0 +1,164 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine;
+import com.yahoo.vespa.hosted.node.admin.container.PartialContainer;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This class removes container images that have not been recently used by any containers.
+ *
+ * <p>Definitions:
+ * <ul>
+ * <li>Every image has exactly 1 id</li>
+ * <li>Every image has between 0..n tags, see
+ * <a href="https://docs.docker.com/engine/reference/commandline/tag/">docker tag</a> for more</li>
+ * <li>Every image has 0..1 parent ids</li>
+ * </ul>
+ *
+ * <p>Limitations:
+ * <ol>
+ * <li>Image that has more than 1 tag cannot be deleted by ID</li>
+ * <li>Deleting a tag of an image with multiple tags will only remove the tag, the image with the
+ * remaining tags will remain</li>
+ * <li>Deleting the last tag of an image will delete the entire image.</li>
+ * <li>Image cannot be deleted if:
+ * <p>- It has 1 or more children
+ * <p>- A container uses it
+ * </li>
+ * </ol>
+ *
+ * @author freva
+ * @author mpolden
+ */
+public class ContainerImagePruner {
+
+ private static final Logger LOG = Logger.getLogger(ContainerImagePruner.class.getName());
+
+ private final Timer timer;
+ private final ContainerEngine containerEngine;
+
+ private final Map<String, Instant> lastTimeUsedByImageId = new ConcurrentHashMap<>();
+
+ public ContainerImagePruner(ContainerEngine containerEngine, Timer timer) {
+ this.containerEngine = Objects.requireNonNull(containerEngine);
+ this.timer = Objects.requireNonNull(timer);
+ }
+
+ /**
+ * Remove unused container images.
+ *
+ * Note: This method must be called frequently enough to see all containers to know which images are being used.
+ *
+ * @param excludedRefs List of image references (tag or id) to keep, regardless of their status
+ * @param minAge Minimum age of for image to be removed
+ * @return true if any image was remove
+ */
+ public boolean removeUnusedImages(TaskContext context, List<String> excludedRefs, Duration minAge) {
+ List<Image> images = containerEngine.listImages(context);
+ List<PartialContainer> containers = containerEngine.listContainers(context);
+
+ Map<String, Image> imageByImageId = images.stream().collect(Collectors.toMap(Image::id, Function.identity()));
+
+ // The set of images that we want to keep is:
+ // 1. The images that were recently used
+ // 2. The images that were explicitly excluded
+ Set<String> imagesToKeep = Stream
+ .concat(
+ updateRecentlyUsedImageIds(images, containers, minAge).stream(), // 1
+ referencesToImages(excludedRefs, images).stream()) // 2
+ .collect(Collectors.toSet());
+
+ // Now take all the images we have locally
+ List<Image> imagesToRemove = imageByImageId.keySet().stream()
+ // filter out images we want to keep
+ .filter(imageId -> !imagesToKeep.contains(imageId))
+ .map(imageByImageId::get)
+ .collect(Collectors.toCollection(ArrayList::new));
+
+ // We cannot delete an image that is referenced by other images as parent. Computing parent image is complicated, see
+ // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/layer_tree.go#L235:L299
+ // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/oci.go#L30:L97
+ // In practice, our images do not have any parents on prod machines, so we should be able to delete in any
+ // order. In case we ever do get a parent on a host somehow, we could get stuck if we always attempt to delete
+ // in wrong order, so shuffle first to ensure this eventually converges
+ Collections.shuffle(imagesToRemove);
+
+ imagesToRemove.forEach(image -> {
+ // Deleting an image by image ID with multiple tags will fail -> delete by tags instead
+ referencesOf(image).forEach(imageReference -> {
+ LOG.info("Deleting unused image " + imageReference);
+ containerEngine.removeImage(context, imageReference);
+ });
+ lastTimeUsedByImageId.remove(image.id());
+ });
+ return !imagesToRemove.isEmpty();
+ }
+
+ private Set<String> updateRecentlyUsedImageIds(List<Image> images, List<PartialContainer> containers, Duration minImageAgeToDelete) {
+ final Instant now = timer.currentTime();
+
+ // Add any already downloaded image to the list once
+ images.forEach(image -> lastTimeUsedByImageId.putIfAbsent(image.id(), now));
+
+ // Update last used time for all current containers
+ containers.forEach(container -> lastTimeUsedByImageId.put(container.imageId(), now));
+
+ // Return list of images that have been used within minImageAgeToDelete
+ return lastTimeUsedByImageId.entrySet().stream()
+ .filter(entry -> Duration.between(entry.getValue(), now).minus(minImageAgeToDelete).isNegative())
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Map given references (image tags or ids) to images.
+ *
+ * This only works if the given tag is actually present locally. This is fine, because if it isn't - we can't delete
+ * it, so no harm done.
+ */
+ private Set<String> referencesToImages(List<String> references, List<Image> images) {
+ Map<String, String> imageIdByImageTag = images.stream()
+ .flatMap(image -> referencesOf(image).stream()
+ .map(repoTag -> new Pair<>(repoTag, image.id())))
+ .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
+
+ return references.stream()
+ .map(ref -> imageIdByImageTag.getOrDefault(ref, ref))
+ .collect(Collectors.toUnmodifiableSet());
+ }
+
+ /**
+ * Returns list of references to given image, preferring image tag(s), if any exist.
+ *
+ * If image is untagged, its ID is returned instead.
+ */
+ private static List<String> referencesOf(Image image) {
+ if (image.names().isEmpty()) {
+ return List.of(image.id());
+ }
+ return image.names().stream()
+ .map(tag -> {
+ if ("<none>:<none>".equals(tag)) return image.id();
+ return tag;
+ })
+ .toList();
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java
new file mode 100644
index 00000000000..223304f058e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java
@@ -0,0 +1,50 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This represents a container image that exists locally.
+ *
+ * @author mpolden
+ */
+public class Image {
+
+ private final String id;
+ private final List<String> names;
+
+ public Image(String id, List<String> names) {
+ this.id = Objects.requireNonNull(id);
+ this.names = List.copyOf(Objects.requireNonNull(names));
+ }
+
+ /** The identifier of this image */
+ public String id() {
+ return id;
+ }
+
+ /** Names for this image, such as tags or digests */
+ public List<String> names() {
+ return names;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Image image = (Image) o;
+ return id.equals(image.id) && names.equals(image.names);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, names);
+ }
+
+ @Override
+ public String toString() {
+ return "image " + id;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java
new file mode 100644
index 00000000000..fa348209520
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author mpolden
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java
new file mode 100644
index 00000000000..e6d05e04965
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java
@@ -0,0 +1,28 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+/**
+ * @author freva
+ */
+public class Counter implements MetricValue {
+ private final Object lock = new Object();
+
+ private long value = 0;
+
+ public void increment() {
+ add(1L);
+ }
+
+ public void add(long n) {
+ synchronized (lock) {
+ value += n;
+ }
+ }
+
+ @Override
+ public Number getValue() {
+ synchronized (lock) {
+ return value;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java
new file mode 100644
index 00000000000..724432431cd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java
@@ -0,0 +1,76 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * @author freva
+ */
+public class DimensionMetrics {
+
+ private final String application;
+ private final Dimensions dimensions;
+ private final Map<String, Number> metrics;
+
+ DimensionMetrics(String application, Dimensions dimensions, Map<String, Number> metrics) {
+ this.application = Objects.requireNonNull(application);
+ this.dimensions = Objects.requireNonNull(dimensions);
+ this.metrics = metrics.entrySet().stream()
+ .filter(DimensionMetrics::metricIsFinite)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ public String getApplication() {
+ return application;
+ }
+
+ public Dimensions getDimensions() {
+ return dimensions;
+ }
+
+ public Map<String, Number> getMetrics() {
+ return metrics;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ DimensionMetrics that = (DimensionMetrics) o;
+ return application.equals(that.application) &&
+ dimensions.equals(that.dimensions) &&
+ metrics.equals(that.metrics);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(application, dimensions, metrics);
+ }
+
+ private static boolean metricIsFinite(Map.Entry<String, Number> metric) {
+ return ! (metric.getValue() instanceof Double) || Double.isFinite((double) metric.getValue());
+ }
+
+ public static class Builder {
+ private final String application;
+ private final Dimensions dimensions;
+ private final Map<String, Number> metrics = new HashMap<>();
+
+ public Builder(String application, Dimensions dimensions) {
+ this.application = application;
+ this.dimensions = dimensions;
+ }
+
+ public Builder withMetric(String metricName, Number metricValue) {
+ metrics.put(metricName, metricValue);
+ return this;
+ }
+
+ public DimensionMetrics build() {
+ return new DimensionMetrics(application, dimensions, metrics);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java
new file mode 100644
index 00000000000..0f9144b9ca1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java
@@ -0,0 +1,30 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author freva
+ */
+public record Dimensions(Map<String, String> dimensionsMap) {
+
+ public static final Dimensions NONE = new Dimensions(Map.of());
+
+ public Dimensions(Map<String, String> dimensionsMap) {
+ this.dimensionsMap = Map.copyOf(dimensionsMap);
+ }
+
+ public static class Builder {
+ private final Map<String, String> dimensionsMap = new HashMap<>();
+
+ public Dimensions.Builder add(String dimensionName, String dimensionValue) {
+ dimensionsMap.put(dimensionName, dimensionValue);
+ return this;
+ }
+
+ public Dimensions build() {
+ return new Dimensions(dimensionsMap);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java
new file mode 100644
index 00000000000..d97db8f0242
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java
@@ -0,0 +1,24 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+/**
+ * @author freva
+ */
+public class Gauge implements MetricValue {
+ private final Object lock = new Object();
+
+ private double value;
+
+ public void sample(double x) {
+ synchronized (lock) {
+ this.value = x;
+ }
+ }
+
+ @Override
+ public Number getValue() {
+ synchronized (lock) {
+ return value;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java
new file mode 100644
index 00000000000..da05464e0be
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java
@@ -0,0 +1,9 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+/**
+ * @author freva
+ */
+public interface MetricValue {
+ Number getValue();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java
new file mode 100644
index 00000000000..e144f3a91e3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java
@@ -0,0 +1,139 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+import com.yahoo.component.annotation.Inject;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Stores the latest metric for the given application, name, dimension triplet in memory
+ *
+ * @author freva
+ */
+public class Metrics {
+ // Application names used
+ public static final String APPLICATION_HOST = "vespa.host";
+ public static final String APPLICATION_NODE = "vespa.node";
+
+ private final Object monitor = new Object();
+ private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>();
+
+ @Inject
+ public Metrics() { }
+
+ /**
+ * Creates a counter metric under vespa.host application, with no dimensions and default dimension type
+ * See {@link #declareCounter(String, String, Dimensions, DimensionType)}
+ */
+ public Counter declareCounter(String name) {
+ return declareCounter(name, Dimensions.NONE);
+ }
+
+ /**
+ * Creates a counter metric under vespa.host application, with the given dimensions and default dimension type
+ * See {@link #declareCounter(String, String, Dimensions, DimensionType)}
+ */
+ public Counter declareCounter(String name, Dimensions dimensions) {
+ return declareCounter(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT);
+ }
+
+ /** Creates a counter metric. This method is idempotent. */
+ public Counter declareCounter(String application, String name, Dimensions dimensions, DimensionType type) {
+ synchronized (monitor) {
+ return (Counter) getOrCreateApplicationMetrics(application, type)
+ .computeIfAbsent(dimensions, d -> new HashMap<>())
+ .computeIfAbsent(name, n -> new Counter());
+ }
+ }
+
+ /**
+ * Creates a gauge metric under vespa.host application, with no dimensions and default dimension type
+ * See {@link #declareGauge(String, String, Dimensions, DimensionType)}
+ */
+ public Gauge declareGauge(String name) {
+ return declareGauge(name, Dimensions.NONE);
+ }
+
+ /**
+ * Creates a gauge metric under vespa.host application, with the given dimensions and default dimension type
+ * See {@link #declareGauge(String, String, Dimensions, DimensionType)}
+ */
+ public Gauge declareGauge(String name, Dimensions dimensions) {
+ return declareGauge(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT);
+ }
+
+ /** Creates a gauge metric. This method is idempotent */
+ public Gauge declareGauge(String application, String name, Dimensions dimensions, DimensionType type) {
+ synchronized (monitor) {
+ return (Gauge) getOrCreateApplicationMetrics(application, type)
+ .computeIfAbsent(dimensions, d -> new HashMap<>())
+ .computeIfAbsent(name, n -> new Gauge());
+ }
+ }
+
+ public List<DimensionMetrics> getDefaultMetrics() {
+ return getMetricsByType(DimensionType.DEFAULT);
+ }
+
+ public List<DimensionMetrics> getMetricsByType(DimensionType type) {
+ synchronized (monitor) {
+ List<DimensionMetrics> dimensionMetrics = new ArrayList<>();
+ metrics.getOrDefault(type, Map.of())
+ .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream()
+ .map(entry -> new DimensionMetrics(application, entry.getKey(),
+ entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue()))))
+ .forEach(dimensionMetrics::add));
+ return dimensionMetrics;
+ }
+ }
+
+ public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) {
+ synchronized (monitor) {
+ Optional.ofNullable(metrics.get(type))
+ .map(m -> m.get(name))
+ .map(ApplicationMetrics::metricsByDimensions)
+ .ifPresent(m -> m.remove(dimensionsToRemove));
+ }
+ }
+
+ public void deleteMetricByName(String application, String metricName, DimensionType type) {
+ synchronized (monitor) {
+ Optional.ofNullable(metrics.get(type))
+ .map(m -> m.get(application))
+ .map(ApplicationMetrics::metricsByDimensions)
+ .ifPresent(dims ->
+ dims.values().forEach(metrics -> metrics.remove(metricName))
+ );
+ }
+ }
+
+ Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) {
+ return metrics.computeIfAbsent(type, m -> new HashMap<>())
+ .computeIfAbsent(application, app -> new ApplicationMetrics())
+ .metricsByDimensions();
+ }
+
+ // "Application" is the monitoring application, not Vespa application
+ private static class ApplicationMetrics {
+ private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>();
+
+ Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() {
+ return metricsByDimensions;
+ }
+ }
+
+ // Used to distinguish whether metrics have been populated with all tag values
+ public enum DimensionType {
+ /** Default metrics get added default dimensions set in check config */
+ DEFAULT,
+
+ /** Pretagged metrics will only get the dimensions explicitly set when creating the counter/gauge */
+ PRETAGGED
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java
new file mode 100644
index 00000000000..e6ddfa2f4c8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java
new file mode 100644
index 00000000000..86f3c31ff39
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java
new file mode 100644
index 00000000000..332a225bda3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java
@@ -0,0 +1,16 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+
+/**
+ * Wireguard task for containers.
+ *
+ * @author gjoranv
+ */
+public interface ContainerWireguardTask {
+
+ void converge(NodeAgentContext context, ContainerId containerId);
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
new file mode 100644
index 00000000000..8bfb3f86aa7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -0,0 +1,196 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.Container;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
+import com.yahoo.vespa.hosted.node.admin.maintenance.disk.CoredumpCleanupRule;
+import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup;
+import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule;
+import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
+
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+
+/**
+ * @author freva
+ */
+public class StorageMaintainer {
+ private static final Logger logger = Logger.getLogger(StorageMaintainer.class.getName());
+ private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter
+ .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC);
+
+ private final Terminal terminal;
+ private final CoredumpHandler coredumpHandler;
+ private final DiskCleanup diskCleanup;
+ private final SyncClient syncClient;
+ private final Timer timer;
+ private final Path archiveContainerStoragePath;
+
+ // We cache disk usage to avoid doing expensive disk operations so often
+ private final Cache<ContainerName, DiskSize> diskUsage = CacheBuilder.newBuilder()
+ .maximumSize(100)
+ .expireAfterWrite(5, TimeUnit.MINUTES)
+ .build();
+
+ public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, DiskCleanup diskCleanup,
+ SyncClient syncClient, Timer timer, Path archiveContainerStoragePath) {
+ this.terminal = terminal;
+ this.coredumpHandler = coredumpHandler;
+ this.diskCleanup = diskCleanup;
+ this.syncClient = syncClient;
+ this.timer = timer;
+ this.archiveContainerStoragePath = archiveContainerStoragePath;
+ }
+
+ public boolean syncLogs(NodeAgentContext context, boolean throttle) {
+ Optional<URI> archiveUri = context.node().archiveUri();
+ if (archiveUri.isEmpty()) return false;
+ ApplicationId owner = context.node().owner().orElseThrow();
+
+ List<SyncFileInfo> syncFileInfos = FileFinder.files(context.paths().underVespaHome("logs/vespa"))
+ .maxDepth(2)
+ .stream()
+ .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime))
+ .flatMap(fa -> SyncFileInfo.forLogFile(archiveUri.get(), fa.path(), throttle, owner).stream())
+ .toList();
+
+ return syncClient.sync(context, syncFileInfos, throttle ? 1 : 100);
+ }
+
+ public Optional<DiskSize> diskUsageFor(NodeAgentContext context) {
+ try {
+ DiskSize cachedDiskUsage = diskUsage.getIfPresent(context.containerName());
+ if (cachedDiskUsage != null) return Optional.of(cachedDiskUsage);
+
+ DiskSize diskUsageBytes = getDiskUsed(context, context.paths().of("/").pathOnHost());
+ diskUsage.put(context.containerName(), diskUsageBytes);
+ return Optional.of(diskUsageBytes);
+ } catch (Exception e) {
+ context.log(logger, Level.WARNING, "Failed to get disk usage", e);
+ return Optional.empty();
+ }
+ }
+
+ DiskSize getDiskUsed(TaskContext context, Path pathOnHost) {
+ if (!Files.exists(pathOnHost)) return DiskSize.ZERO;
+
+ String output = terminal.newCommandLine(context)
+ .add("du", "-xsk", pathOnHost.toString())
+ .setTimeout(Duration.ofSeconds(60))
+ .executeSilently()
+ .getOutput();
+
+ String[] results = output.split("\t");
+ if (results.length != 2) {
+ throw ConvergenceException.ofError("Result from disk usage command not as expected: " + output);
+ }
+
+ return DiskSize.of(Long.parseLong(results[0]), DiskSize.Unit.kiB);
+ }
+
+ public boolean cleanDiskIfFull(NodeAgentContext context) {
+ if (context.isDisabled(NodeAgentTask.DiskCleanup)) return false;
+
+ double totalBytes = context.node().diskSize().bytes();
+ // Delete enough bytes to get below 70% disk usage, but only if we are already using more than 80% disk
+ long bytesToRemove = diskUsageFor(context)
+ .map(diskUsage -> (long) (diskUsage.bytes() - 0.7 * totalBytes))
+ .filter(bytes -> bytes > totalBytes * 0.1)
+ .orElse(0L);
+
+ if (bytesToRemove > 0 && diskCleanup.cleanup(context, createCleanupRules(context), bytesToRemove)) {
+ diskUsage.invalidate(context.containerName());
+ return true;
+ }
+ return false;
+ }
+
+ private List<DiskCleanupRule> createCleanupRules(NodeAgentContext context) {
+ Instant start = timer.currentTime();
+ double oneMonthSeconds = Duration.ofDays(30).getSeconds();
+ Function<Instant, Double> monthNormalizer = instant -> Duration.between(instant, start).getSeconds() / oneMonthSeconds;
+ List<DiskCleanupRule> rules = new ArrayList<>();
+
+ rules.add(CoredumpCleanupRule.forContainer(context.paths().underVespaHome("var/crash")));
+
+ rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("var/tmp")).list(),
+ fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST));
+
+ if (context.node().membership().map(m -> m.type().hasContainer()).orElse(false)) {
+ rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/access")).list(),
+ fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST));
+ }
+ if (context.nodeType() == NodeType.tenant && context.node().membership().map(m -> m.type().isAdmin()).orElse(false))
+ rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/logarchive")).list(),
+ fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST));
+
+ if (context.nodeType() == NodeType.proxy)
+ rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/nginx")).list(),
+ fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.MEDIUM));
+
+ return rules;
+ }
+
+ /** Checks if container has any new coredumps, reports and archives them if so */
+ public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container, boolean throwIfCoreBeingWritten) {
+ if (context.isDisabled(NodeAgentTask.CoreDumps)) return;
+ coredumpHandler.converge(context, container.map(Container::image), throwIfCoreBeingWritten);
+ }
+
+ /**
+ * Prepares the container-storage for the next container by deleting/archiving all the data of the current container.
+ * Removes old files, reports coredumps and archives container data, runs when container enters state "dirty"
+ */
+ public void archiveNodeStorage(NodeAgentContext context) {
+ ContainerPath logsDirInContainer = context.paths().underVespaHome("logs");
+ Path containerLogsInArchiveDir = archiveContainerStoragePath
+ .resolve(context.containerName().asString() + "_" + DATE_TIME_FORMATTER.format(timer.currentTime()) + logsDirInContainer.pathInContainer());
+
+ // Files.move() does not support moving non-empty directories across providers, move using host paths
+ UnixPath containerLogsOnHost = new UnixPath(logsDirInContainer.pathOnHost());
+ if (containerLogsOnHost.exists()) {
+ new UnixPath(containerLogsInArchiveDir).createParents();
+ containerLogsOnHost.moveIfExists(containerLogsInArchiveDir);
+ }
+ new UnixPath(context.paths().of("/")).deleteRecursively();
+
+ // Operations on ContainerPath will fail if Container FS root doesn't exist, it is therefore important that
+ // it exists as long as NodeAgent is running. Normally the root is only created when NodeAgent is first
+ // started. Because non-tenant nodes are never removed from node-repo, we immediately re-create the new root
+ // after archiving the previous
+ if (context.nodeType() != NodeType.tenant)
+ context.paths().of("/").getFileSystem().createRoot();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java
new file mode 100644
index 00000000000..99715e6cad9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java
@@ -0,0 +1,138 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.google.common.net.InetAddresses;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * This class maintains the iptables (ipv4 and ipv6) for all running containers.
+ * The filter table is synced with ACLs fetched from the Node repository while the nat table
+ * is synched with the proper redirect rule.
+ * <p>
+ * If an ACL cannot be configured (e.g. iptables process execution fails) we attempted to flush the rules
+ * rendering the firewall open.
+ * <p>
+ * This class currently assumes control over the filter and nat table.
+ * <p>
+ * The configuration will be retried the next time the maintainer runs.
+ *
+ * @author mpolden
+ * @author smorgrav
+ */
+public class AclMaintainer {
+ private static final Logger logger = Logger.getLogger(AclMaintainer.class.getName());
+
+ private final ContainerOperations containerOperations;
+ private final IPAddresses ipAddresses;
+
+ public AclMaintainer(ContainerOperations containerOperations, IPAddresses ipAddresses) {
+ this.containerOperations = containerOperations;
+ this.ipAddresses = ipAddresses;
+ }
+
+ // ip(6)tables operate while having the xtables lock, run with synchronized to prevent multiple NodeAgents
+ // invoking ip(6)tables concurrently.
+ public synchronized void converge(NodeAgentContext context) {
+ if (context.isDisabled(NodeAgentTask.AclMaintainer)) return;
+
+ // Apply acl to the filter table
+ editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4));
+ editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6));
+
+ ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv4).ifPresent(addr -> applyRedirect(context, addr));
+ ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv6).ifPresent(addr -> applyRedirect(context, addr));
+ }
+
+ private void applyRedirect(NodeAgentContext context, InetAddress address) {
+ IPVersion ipVersion = IPVersion.get(address);
+ // Necessary to avoid the routing packets destined for the node's own public IP address
+ // via the bridge, which is illegal.
+ String redirectRule = "-A OUTPUT -d " + InetAddresses.toAddrString(address) + ipVersion.singleHostCidr() + " -j REDIRECT";
+ editLogOnError(context, ipVersion, "nat", NatTableLineEditor.from(redirectRule));
+ }
+
+ private boolean editFlushOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) {
+ return edit(context, table, ipVersion, lineEditor, true);
+ }
+
+ private boolean editLogOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) {
+ return edit(context, table, ipVersion, lineEditor, false);
+ }
+
+ private boolean edit(NodeAgentContext context, String table, IPVersion ipVersion, LineEditor lineEditor, boolean flush) {
+ Editor editor = new Editor(
+ ipVersion.iptablesCmd() + "-" + table,
+ listTable(context, table, ipVersion),
+ restoreTable(context, table, ipVersion, flush),
+ lineEditor);
+ return editor.edit(message -> context.log(logger, message));
+ }
+
+ private Supplier<List<String>> listTable(NodeAgentContext context, String table, IPVersion ipVersion) {
+ return () -> containerOperations
+ .executeCommandInNetworkNamespace(context, new CommandLine.Options().setSilent(true), ipVersion.iptablesCmd(), "-S", "-t", table)
+ .mapEachLine(String::trim);
+ }
+
+ private Consumer<List<String>> restoreTable(NodeAgentContext context, String table, IPVersion ipVersion, boolean flush) {
+ return list -> {
+ try (TemporaryIpTablesFileHandler fileHandler = new TemporaryIpTablesFileHandler(table)) {
+ String rules = String.join("\n", list);
+ String fileContent = "*" + table + "\n" + rules + "\nCOMMIT\n";
+ fileHandler.writeUtf8Content(fileContent);
+ containerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesRestore(), fileHandler.absolutePath());
+ } catch (Exception e) {
+ if (flush) {
+ context.log(logger, Level.SEVERE, "Exception occurred while syncing iptable " + table + ", attempting rollback", e);
+ try {
+ containerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-F", "-t", table);
+ } catch (Exception ne) {
+ context.log(logger, Level.SEVERE, "Rollback of table " + table + " failed, giving up", ne);
+ }
+ } else {
+ context.log(logger, Level.WARNING, "Unable to sync iptables for " + table, e);
+ }
+ }
+ };
+ }
+
+ private static class TemporaryIpTablesFileHandler implements AutoCloseable {
+ private final Path path;
+
+ private TemporaryIpTablesFileHandler(String table) {
+ this.path = uncheck(() -> Files.createTempFile("iptables-restore", "." + table));
+ }
+
+ private void writeUtf8Content(String content) throws IOException {
+ Files.writeString(path, content);
+ }
+
+ private String absolutePath() {
+ return path.toAbsolutePath().toString();
+ }
+
+ @Override
+ public void close() throws IOException {
+ Files.deleteIfExists(path);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java
new file mode 100644
index 00000000000..4b831745f27
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java
@@ -0,0 +1,61 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+
+import java.util.List;
+
+/**
+ * An editor that assumes all rules in the filter table are exactly as the wanted rules
+ *
+ * @author smorgrav
+ */
+class FilterTableLineEditor implements LineEditor {
+
+ private final List<String> wantedRules;
+ private int position = 0;
+
+ private FilterTableLineEditor(List<String> wantedRules) {
+ this.wantedRules = List.copyOf(wantedRules);
+ }
+
+ static FilterTableLineEditor from(Acl acl, IPVersion ipVersion) {
+ List<String> rules = acl.toRules(ipVersion);
+ return new FilterTableLineEditor(rules);
+ }
+
+ @Override
+ public LineEdit edit(String line) {
+ int index = indexOf(wantedRules, line, position);
+ // Unwanted rule, remove
+ if (index < 0) return LineEdit.remove();
+
+ // Wanted rule at the expected position, no diff
+ if (index == position) {
+ position++;
+ return LineEdit.none();
+ }
+
+ // Insert the rules between position and index before index
+ List<String> toInsert = wantedRules.subList(position, index);
+ position = ++index;
+ return LineEdit.insertBefore(toInsert);
+ }
+
+ @Override
+ public List<String> onComplete() {
+ return wantedRules.subList(position, wantedRules.size());
+ }
+
+ private static <T> int indexOf(List<T> list, T value, int startPos) {
+ for (int i = startPos; i < list.size(); i++) {
+ if (value.equals(list.get(i)))
+ return i;
+ }
+
+ return -1;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java
new file mode 100644
index 00000000000..9eff816d467
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java
@@ -0,0 +1,49 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor;
+
+import java.util.List;
+
+/**
+ * An editor that only cares about the REDIRECT statement
+ *
+ * @author smorgrav
+ */
+class NatTableLineEditor implements LineEditor {
+
+ private final String redirectRule;
+ private boolean redirectExists;
+
+ private NatTableLineEditor(String redirectRule) {
+ this.redirectRule = redirectRule;
+ }
+
+ static NatTableLineEditor from(String redirectRule) {
+ return new NatTableLineEditor(redirectRule);
+ }
+
+ @Override
+ public LineEdit edit(String line) {
+ if (line.endsWith("REDIRECT")) {
+ if (redirectExists) {
+ // Only allow one redirect rule
+ return LineEdit.remove();
+ } else {
+ redirectExists = true;
+ if (line.equals(redirectRule)) {
+ return LineEdit.none();
+ } else {
+ return LineEdit.replaceWith(redirectRule);
+ }
+ }
+ } else return LineEdit.none();
+ }
+
+ @Override
+ public List<String> onComplete() {
+ if (redirectExists) return List.of();
+ return List.of(redirectRule);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java
new file mode 100644
index 00000000000..f98a32ba488
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
new file mode 100644
index 00000000000..0028784eec8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -0,0 +1,132 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Takes in an uncompressed core dump and collects relevant metadata.
+ *
+ * @author freva
+ */
+public class CoreCollector {
+ private static final Logger logger = Logger.getLogger(CoreCollector.class.getName());
+
+ private static final Pattern JAVA_HEAP_DUMP_PATTERN = Pattern.compile("java_pid.*\\.hprof$");
+ private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("(?m)^Core was generated by `(?<path>.*?)'\\.");
+ private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'");
+ private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'");
+ static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-12/root/bin/gdb";
+
+ private final ContainerOperations container;
+
+ public CoreCollector(ContainerOperations container) {
+ this.container = container;
+ }
+
+ String readBinPathFallback(NodeAgentContext context, ContainerPath coredumpPath) {
+ String[] command = {GDB_PATH_RHEL8, "-n", "-batch", "-core", coredumpPath.pathInContainer()};
+ CommandResult result = container.executeCommandInContainer(context, context.users().root(), command);
+
+ Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput());
+ if (! matcher.find()) {
+ throw ConvergenceException.ofError(String.format("Failed to extract binary path from GDB, result: %s, command: %s",
+ asString(result), Arrays.toString(command)));
+ }
+ return matcher.group("path").split(" ")[0];
+ }
+
+ String readBinPath(NodeAgentContext context, ContainerPath coredumpPath) {
+ String[] command = {"file", coredumpPath.pathInContainer()};
+ try {
+ CommandResult result = container.executeCommandInContainer(context, context.users().root(), command);
+ if (result.getExitCode() != 0) {
+ throw ConvergenceException.ofError("file command failed with " + asString(result));
+ }
+
+ Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getOutput());
+ if (execfnMatcher.find()) {
+ return execfnMatcher.group("path").split(" ")[0];
+ }
+
+ Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getOutput());
+ if (fromMatcher.find()) {
+ return fromMatcher.group("path").split(" ")[0];
+ }
+ } catch (RuntimeException e) {
+ context.log(logger, Level.WARNING, String.format("Failed getting bin path, command: %s. " +
+ "Trying fallback instead", Arrays.toString(command)), e);
+ }
+
+ return readBinPathFallback(context, coredumpPath);
+ }
+
+ List<String> readBacktrace(NodeAgentContext context, ContainerPath coredumpPath, String binPath, boolean allThreads) {
+ String threads = allThreads ? "thread apply all bt" : "bt";
+ String[] command = {GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", threads, "-batch", binPath, coredumpPath.pathInContainer()};
+
+ CommandResult result = container.executeCommandInContainer(context, context.users().root(), command);
+ if (result.getExitCode() != 0)
+ throw ConvergenceException.ofError("Failed to read backtrace " + asString(result) + ", Command: " + Arrays.toString(command));
+
+ return List.of(result.getOutput().split("\n"));
+ }
+
+ List<String> readJstack(NodeAgentContext context, ContainerPath coredumpPath, String binPath) {
+ String[] command = {"jhsdb", "jstack", "--exe", binPath, "--core", coredumpPath.pathInContainer()};
+
+ CommandResult result = container.executeCommandInContainer(context, context.users().root(), command);
+ if (result.getExitCode() != 0)
+ throw ConvergenceException.ofError("Failed to read jstack " + asString(result) + ", Command: " + Arrays.toString(command));
+
+ return List.of(result.getOutput().split("\n"));
+ }
+
+ CoreDumpMetadata collect(NodeAgentContext context, ContainerPath coredumpPath) {
+ var metadata = new CoreDumpMetadata()
+ .setCreated(new UnixPath(coredumpPath).getLastModifiedTime());
+
+ if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) {
+ metadata.setType(CoreDumpMetadata.Type.JVM_HEAP)
+ .setBinPath("java")
+ .setBacktrace(List.of("Heap dump, no backtrace available"));
+ return metadata;
+ }
+
+ try {
+ String binPath = context.paths().underVespaHome("").resolve(readBinPath(context, coredumpPath)).pathInContainer();
+ metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath);
+
+ if (Path.of(binPath).getFileName().toString().equals("java")) {
+ metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath));
+ } else {
+ metadata.setBacktrace(readBacktrace(context, coredumpPath, binPath, false));
+ metadata.setBacktraceAllThreads(readBacktrace(context, coredumpPath, binPath, true));
+ }
+ } catch (ConvergenceException e) {
+ context.log(logger, Level.WARNING, "Failed to extract backtrace: " + e.getMessage());
+ } catch (RuntimeException e) {
+ context.log(logger, Level.WARNING, "Failed to extract backtrace", e);
+ }
+ return metadata;
+ }
+
+ private String asString(CommandResult result) {
+ return "exit status " + result.getExitCode() + ", output '" + result.getOutput() + "'";
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
new file mode 100644
index 00000000000..a3386a3032f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
@@ -0,0 +1,338 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.security.KeyId;
+import com.yahoo.security.SecretSharedKey;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileDeleter;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileMover;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.MakeDirectory;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.IntStream;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Finds coredumps, collects metadata and reports them
+ *
+ * @author freva
+ */
+public class CoredumpHandler {
+
+ public static final String COREDUMP_FILENAME_PREFIX = "dump_";
+
+ private static final Logger logger = Logger.getLogger(CoredumpHandler.class.getName());
+ private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log");
+ private static final String PROCESSING_DIRECTORY_NAME = "processing";
+ private static final String METADATA2_FILE_NAME = "metadata2.json";
+ private static final String COMPRESSED_EXTENSION = ".zst";
+ private static final String ENCRYPTED_EXTENSION = ".enc";
+
+ private final CoreCollector coreCollector;
+ private final Cores cores;
+ private final String crashPatchInContainer;
+ private final Path doneCoredumpsPath;
+ private final Metrics metrics;
+ private final Timer timer;
+ private final Supplier<String> coredumpIdSupplier;
+ private final SecretSharedKeySupplier secretSharedKeySupplier;
+ private final StringFlag coreEncryptionPublicKeyIdFlag;
+
+ /**
+ * @param crashPathInContainer path inside the container where core dump are dumped
+ * @param doneCoredumpsPath path on host where processed core dumps are stored
+ */
+ public CoredumpHandler(CoreCollector coreCollector, Cores cores,
+ String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Timer timer,
+ SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) {
+ this(coreCollector, cores, crashPathInContainer, doneCoredumpsPath,
+ metrics, timer, () -> UUID.randomUUID().toString(), secretSharedKeySupplier,
+ flagSource);
+ }
+
+ CoredumpHandler(CoreCollector coreCollector, Cores cores,
+ String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics,
+ Timer timer, Supplier<String> coredumpIdSupplier,
+ SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) {
+ this.coreCollector = coreCollector;
+ this.cores = cores;
+ this.crashPatchInContainer = crashPathInContainer;
+ this.doneCoredumpsPath = doneCoredumpsPath;
+ this.metrics = metrics;
+ this.timer = timer;
+ this.coredumpIdSupplier = coredumpIdSupplier;
+ this.secretSharedKeySupplier = secretSharedKeySupplier;
+ this.coreEncryptionPublicKeyIdFlag = Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.bindTo(flagSource);
+ }
+
+
+ public void converge(NodeAgentContext context, Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) {
+ ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa());
+ ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME);
+
+ updateMetrics(context, containerCrashPath);
+
+ if (throwIfCoreBeingWritten) {
+ List<String> pendingCores = FileFinder.files(containerCrashPath)
+ .match(fileAttributes -> !isReadyForProcessing(fileAttributes))
+ .maxDepth(1).stream()
+ .map(FileFinder.FileAttributes::filename)
+ .toList();
+ if (!pendingCores.isEmpty())
+ throw ConvergenceException.ofError(String.format("Cannot process %s coredumps: Still being written",
+ pendingCores.size() < 5 ? pendingCores : pendingCores.size()));
+ }
+
+ // Check if we have already started to process a core dump or we can enqueue a new core one
+ getCoredumpToProcess(context, containerCrashPath, containerProcessingPath)
+ .ifPresent(path -> processAndReportSingleCoreDump(context, path, dockerImage));
+ }
+
+ /** @return path to directory inside processing directory that contains a core dump file to process */
+ Optional<ContainerPath> getCoredumpToProcess(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) {
+ return FileFinder.directories(containerProcessingPath).stream()
+ .map(FileFinder.FileAttributes::path)
+ .findAny()
+ .map(ContainerPath.class::cast)
+ .or(() -> enqueueCoredump(context, containerCrashPath, containerProcessingPath));
+ }
+
+ /**
+ * Moves a coredump and related hs_err file(s) to a new directory under the processing/ directory.
+ * Limit to only processing one coredump at the time, starting with the oldest.
+ *
+ * Assumption: hs_err files are much smaller than core files and are written (last modified time)
+ * before the core file.
+ *
+ * @return path to directory inside processing directory which contains the enqueued core dump file
+ */
+ Optional<ContainerPath> enqueueCoredump(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) {
+ Predicate<String> isCoreDump = filename -> !HS_ERR_PATTERN.matcher(filename).matches();
+
+ List<Path> toProcess = FileFinder.files(containerCrashPath)
+ .match(attributes -> {
+ if (isReadyForProcessing(attributes)) {
+ return true;
+ } else {
+ if (isCoreDump.test(attributes.filename()))
+ context.log(logger, attributes.path() + " is still being written");
+ return false;
+ }
+ })
+ .maxDepth(1)
+ .stream()
+ .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime))
+ .map(FileFinder.FileAttributes::path)
+ .toList();
+
+ int coredumpIndex = IntStream.range(0, toProcess.size())
+ .filter(i -> isCoreDump.test(toProcess.get(i).getFileName().toString()))
+ .findFirst()
+ .orElse(-1);
+
+ // Either there are no files in crash directory, or all the files are hs_err files.
+ if (coredumpIndex == -1) return Optional.empty();
+
+ ContainerPath enqueuedDir = containerProcessingPath.resolve(coredumpIdSupplier.get());
+ new MakeDirectory(enqueuedDir).createParents().converge(context);
+ IntStream.range(0, coredumpIndex + 1)
+ .forEach(i -> {
+ Path path = toProcess.get(i);
+ String prefix = i == coredumpIndex ? COREDUMP_FILENAME_PREFIX : "";
+ new FileMover(path, enqueuedDir.resolve(prefix + path.getFileName())).converge(context);
+ });
+ return Optional.of(enqueuedDir);
+ }
+
+ private String corePublicKeyFlagValue(NodeAgentContext context) {
+ return coreEncryptionPublicKeyIdFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value();
+ }
+
+ static OutputStream wrapWithEncryption(OutputStream wrappedStream, SecretSharedKey sharedCoreKey) {
+ return sharedCoreKey.makeEncryptionCipher().wrapOutputStream(wrappedStream);
+ }
+
+ /**
+ * Compresses and, if a key is provided, encrypts core file (and deletes the uncompressed core), then moves
+ * the entire core dump processing directory to {@link #doneCoredumpsPath} for archive
+ */
+ private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, SecretSharedKey sharedCoreKey) {
+ ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory);
+ String extension = COMPRESSED_EXTENSION + ENCRYPTED_EXTENSION;
+ ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + extension);
+
+ try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile));
+ OutputStream fos = wrapWithEncryption(Files.newOutputStream(compressedCoreFile), sharedCoreKey)) {
+ zcis.transferTo(fos);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ new FileDeleter(coreFile).converge(context);
+
+ Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString());
+ new MakeDirectory(newCoredumpDirectory).createParents().converge(context);
+ // Files.move() does not support moving non-empty directories across providers, move using host paths
+ new FileMover(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString()))
+ .converge(context);
+ }
+
+ ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) {
+ return (ContainerPath) FileFinder.files(coredumpProccessingDirectory)
+ .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(COMPRESSED_EXTENSION).negate())
+ .and(nameEndsWith(ENCRYPTED_EXTENSION).negate()))
+ .maxDepth(1)
+ .stream()
+ .map(FileFinder.FileAttributes::path)
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException(
+ "No coredump file found in processing directory " + coredumpProccessingDirectory));
+ }
+
+ void updateMetrics(NodeAgentContext context, ContainerPath containerCrashPath) {
+ Dimensions dimensions = generateDimensions(context);
+
+ // Unprocessed coredumps
+ int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPath)
+ .match(nameStartsWith(".").negate())
+ .match(nameMatches(HS_ERR_PATTERN).negate())
+ .match(nameEndsWith(COMPRESSED_EXTENSION).negate())
+ .match(nameEndsWith(ENCRYPTED_EXTENSION).negate())
+ .match(nameStartsWith("metadata").negate())
+ .list().size();
+
+ metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.enqueued", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfUnprocessedCoredumps);
+
+ // Processed coredumps
+ Path processedCoredumpsPath = doneCoredumpsPath.resolve(context.containerName().asString());
+ int numberOfProcessedCoredumps = FileFinder.directories(processedCoredumpsPath)
+ .maxDepth(1)
+ .list().size();
+
+ metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.processed", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfProcessedCoredumps);
+ }
+
+ private Dimensions generateDimensions(NodeAgentContext context) {
+ NodeSpec node = context.node();
+ Dimensions.Builder dimensionsBuilder = new Dimensions.Builder()
+ .add("host", node.hostname())
+ .add("flavor", node.flavor())
+ .add("state", node.state().toString())
+ .add("zone", context.zone().getId().value());
+
+ node.owner().ifPresent(owner ->
+ dimensionsBuilder
+ .add("tenantName", owner.tenant().value())
+ .add("applicationName", owner.application().value())
+ .add("instanceName", owner.instance().value())
+ .add("app", String.join(".", owner.application().value(), owner.instance().value()))
+ .add("applicationId", owner.toFullString())
+ );
+
+ node.membership().ifPresent(membership ->
+ dimensionsBuilder
+ .add("clustertype", membership.type().value())
+ .add("clusterid", membership.clusterId())
+ );
+
+ node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent));
+ dimensionsBuilder.add("system", context.zone().getSystemName().value());
+
+ return dimensionsBuilder.build();
+ }
+
+ private boolean isReadyForProcessing(FileFinder.FileAttributes fileAttributes) {
+ // Wait at least a minute until we start processing a core/heap dump to ensure that
+ // kernel/JVM has finished writing it
+ return timer.currentTime().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime());
+ }
+
+ void processAndReportSingleCoreDump(NodeAgentContext context, ContainerPath coreDumpDirectory,
+ Optional<DockerImage> dockerImage) {
+ CoreDumpMetadata metadata = gatherMetadata(context, coreDumpDirectory);
+ dockerImage.ifPresent(metadata::setDockerImage);
+ dockerImage.flatMap(DockerImage::tag).ifPresent(metadata::setVespaVersion);
+ dockerImage.ifPresent(metadata::setDockerImage);
+ SecretSharedKey sharedCoreKey = Optional.of(corePublicKeyFlagValue(context))
+ .filter(k -> !k.isEmpty())
+ .map(KeyId::ofString)
+ .flatMap(secretSharedKeySupplier::create)
+ .orElseThrow(() -> ConvergenceException.ofError("No core dump encryption key provided"));
+ metadata.setDecryptionToken(sharedCoreKey.sealedSharedKey().toTokenString());
+
+ String coreDumpId = coreDumpDirectory.getFileName().toString();
+ cores.report(context.hostname(), coreDumpId, metadata);
+ context.log(logger, "Core dump reported: " + coreDumpId);
+ finishProcessing(context, coreDumpDirectory, sharedCoreKey);
+ }
+
+ CoreDumpMetadata gatherMetadata(NodeAgentContext context, ContainerPath coreDumpDirectory) {
+ ContainerPath metadataPath = coreDumpDirectory.resolve(METADATA2_FILE_NAME);
+ Optional<ReportCoreDumpRequest> request = ReportCoreDumpRequest.load(metadataPath);
+ if (request.isPresent()) {
+ var metadata = new CoreDumpMetadata();
+ request.get().populateMetadata(metadata, doneCoredumpsPath.getFileSystem());
+ return metadata;
+ }
+
+ ContainerPath coreDumpFile = findCoredumpFileInProcessingDirectory(coreDumpDirectory);
+ CoreDumpMetadata metadata = coreCollector.collect(context, coreDumpFile);
+ metadata.setCpuMicrocodeVersion(getMicrocodeVersion())
+ .setKernelVersion(System.getProperty("os.version"))
+ .setCoreDumpPath(doneCoredumpsPath.resolve(context.containerName().asString())
+ .resolve(coreDumpDirectory.getFileName().toString())
+ .resolve(coreDumpFile.getFileName().toString()));
+
+ ReportCoreDumpRequest requestInstance = new ReportCoreDumpRequest();
+ requestInstance.fillFrom(metadata);
+ requestInstance.save(metadataPath);
+ context.log(logger, "Wrote " + metadataPath.pathOnHost());
+ return metadata;
+ }
+
+ private String getMicrocodeVersion() {
+ String output = uncheck(() -> Files.readAllLines(doneCoredumpsPath.getFileSystem().getPath("/proc/cpuinfo")).stream()
+ .filter(line -> line.startsWith("microcode"))
+ .findFirst()
+ .orElse("microcode : UNKNOWN"));
+
+ String[] results = output.split(":");
+ if (results.length != 2) {
+ throw ConvergenceException.ofError("Result from detect microcode command not as expected: " + output);
+ }
+
+ return results[1].trim();
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java
new file mode 100644
index 00000000000..e5291c837a2
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java
@@ -0,0 +1,17 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.security.KeyId;
+import com.yahoo.security.SecretSharedKey;
+
+import java.util.Optional;
+
+/**
+ * @author vekterli
+ */
+@FunctionalInterface
+public interface SecretSharedKeySupplier {
+
+ Optional<SecretSharedKey> create(KeyId publicKeyId);
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java
new file mode 100644
index 00000000000..0b6b3d18b01
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java
new file mode 100644
index 00000000000..50cd16f5617
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java
@@ -0,0 +1,106 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoField;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith;
+
+/**
+ * @author freva
+ */
+public class CoredumpCleanupRule {
+
+ private static final Comparator<FileAttributes> CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR = Comparator
+ .comparing((FileAttributes fa) -> !fa.filename().contains("vespa-"))
+ .thenComparing(FileAttributes::lastModifiedTime);
+
+ public static DiskCleanupRule forContainer(Path containerCrashPath) {
+ return new ContainerCoredumpCleanupRule(containerCrashPath);
+ }
+
+ public static DiskCleanupRule forHost(Path processedCoredumpsPath) {
+ return new HostCoredumpCleanupRule(processedCoredumpsPath);
+ }
+
+ /** Assigns MEDIUM priority to the oldest, unprocessed coredump and HIGHEST for the remaining */
+ private static class ContainerCoredumpCleanupRule implements DiskCleanupRule {
+ private final Path containerCrashPath;
+
+ private ContainerCoredumpCleanupRule(Path containerCrashPath) {
+ this.containerCrashPath = containerCrashPath;
+ }
+
+ @Override
+ public Collection<PrioritizedFileAttributes> prioritize() {
+ List<FileAttributes> fileAttributes = FileFinder.files(containerCrashPath)
+ .maxDepth(1).stream()
+ .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR)
+ .toList();
+
+ return mapFirstAndRemaining(fileAttributes, Priority.MEDIUM, Priority.HIGHEST).toList();
+ }
+ }
+
+ /** Assigns MEDIUM priority to the first coredump of the day for each container, HIGH for the remaining */
+ private static class HostCoredumpCleanupRule implements DiskCleanupRule {
+ private final Path processedCoredumpsPath;
+
+ private HostCoredumpCleanupRule(Path processedCoredumpsPath) {
+ this.processedCoredumpsPath = processedCoredumpsPath;
+ }
+
+ @Override
+ public Collection<PrioritizedFileAttributes> prioritize() {
+ Map<String, List<FileAttributes>> fileAttributesByContainerDay = FileFinder.files(processedCoredumpsPath)
+ .match(nameStartsWith(CoredumpHandler.COREDUMP_FILENAME_PREFIX))
+ .stream()
+ .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR)
+ .collect(Collectors.groupingBy(
+ // Group FileAttributes by string [container-name]_[day of year], e.g. zt00534-v6-2_234
+ fa -> containerNameFromProcessedCoredumpPath(fa.path()) + "_" + dayOfYear(fa.lastModifiedTime()),
+ Collectors.collectingAndThen(
+ Collectors.toCollection(ArrayList::new),
+ l -> { l.sort(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR); return l; } )));
+
+ return fileAttributesByContainerDay.values().stream()
+ .flatMap(fa -> mapFirstAndRemaining(fa, Priority.MEDIUM, Priority.HIGH))
+ .toList();
+ }
+ }
+
+ /**
+ * Maps list of FileAttributes into list of PrioritizedFileAttributes where the first FileAttribute is given
+ * {@code first} priority, while the remaining FileAttributes are given {@code remaining} priority */
+ private static Stream<PrioritizedFileAttributes> mapFirstAndRemaining(List<FileAttributes> fileAttributes, Priority first, Priority remaining) {
+ return IntStream.range(0, fileAttributes.size())
+ .mapToObj(i -> new PrioritizedFileAttributes(fileAttributes.get(i), i == 0 ? first : remaining));
+ }
+
+ /** Extracts container-name from path under processed-coredumps or empty string */
+ private static String containerNameFromProcessedCoredumpPath(Path path) {
+ if (path.getNameCount() < 3) return ""; // Path is too short
+ return path.getName(path.getNameCount() - 3).toString();
+ }
+
+ /** Returns day number of the year (1-365 (or 366 for leap years)) */
+ private static int dayOfYear(Instant instant) {
+ return instant.atOffset(ZoneOffset.UTC).get(ChronoField.DAY_OF_YEAR);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java
new file mode 100644
index 00000000000..54cf9324909
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java
@@ -0,0 +1,59 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author freva
+ */
+public class DiskCleanup {
+
+ private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName());
+ private static final Comparator<PrioritizedFileAttributes> PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator
+ .comparing(PrioritizedFileAttributes::priority)
+ .thenComparingLong(f -> f.fileAttributes().size())
+ .reversed();
+
+ public boolean cleanup(TaskContext context, List<DiskCleanupRule> rules, long bytesToRemove) {
+ if (bytesToRemove <= 0) return false;
+
+ long[] btr = new long[] { bytesToRemove };
+ List<Path> deletedPaths = new ArrayList<>();
+ try {
+ rules.stream()
+ .flatMap(rule -> rule.prioritize().stream())
+ .sorted(PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR)
+ .takeWhile(fa -> btr[0] > 0)
+ .forEach(pfa -> {
+ if (uncheck(() -> Files.deleteIfExists(pfa.fileAttributes().path()))) {
+ btr[0] -= pfa.fileAttributes().size();
+ deletedPaths.add(pfa.fileAttributes().path());
+ }
+ });
+
+ } finally {
+ String wantedDeleteSize = DiskSize.of(bytesToRemove).asString();
+ String deletedSize = DiskSize.of(bytesToRemove - btr[0]).asString();
+ if (deletedPaths.size() > 20) {
+ context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize);
+ } else if (deletedPaths.size() > 0) {
+ context.log(logger, "Deleted %s because disk was getting full from: %s", deletedSize, deletedPaths);
+ } else {
+ context.log(logger, "Wanted to delete %s, but failed to find any files to delete", wantedDeleteSize);
+ }
+ }
+
+ return !deletedPaths.isEmpty();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java
new file mode 100644
index 00000000000..88b89f1f201
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java
@@ -0,0 +1,20 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+
+import java.util.Collection;
+
+/**
+ * @author freva
+ */
+public interface DiskCleanupRule {
+
+ Collection<PrioritizedFileAttributes> prioritize();
+
+ enum Priority {
+ LOWEST, LOW, MEDIUM, HIGH, HIGHEST
+ }
+
+ record PrioritizedFileAttributes(FileFinder.FileAttributes fileAttributes, Priority priority) { }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java
new file mode 100644
index 00000000000..961d978bfcf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java
@@ -0,0 +1,48 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+
+/**
+ * Prioritizes files by first scoring them with the given scoring function and then mapping the scores to a
+ * priority within the given range.
+ * The priority room is evenly split between given lowest and highest priority for range [0, 1.0). Scores below 0
+ * are assigned lowest, while scores at or higher than 1 are assigned highest priority.
+ *
+ * Typical use-case is for log files. The scoring function calculates the file age and normalizes it by dividing it
+ * by expected max age of log files. The oldest logs will then by prioritized by highest given priority.
+ *
+ * @author freva
+ */
+public class LinearCleanupRule implements DiskCleanupRule {
+ private final Supplier<List<FileAttributes>> lister;
+ private final Function<FileAttributes, Priority> prioritizer;
+
+ public LinearCleanupRule(Supplier<List<FileAttributes>> lister,
+ Function<FileAttributes, Double> scorer, Priority lowest, Priority highest) {
+ if (lowest.ordinal() > highest.ordinal())
+ throw new IllegalArgumentException("Lowest priority: " + lowest + " is higher than highest priority: " + highest);
+
+ this.lister = lister;
+
+ Priority[] values = Priority.values();
+ int range = highest.ordinal() - lowest.ordinal() + 1;
+ this.prioritizer = fa -> {
+ int ordinal = (int) (lowest.ordinal() + scorer.apply(fa) * range);
+ return values[Math.max(lowest.ordinal(), Math.min(highest.ordinal(), ordinal))];
+ };
+ }
+
+ @Override
+ public Collection<PrioritizedFileAttributes> prioritize() {
+ return lister.get().stream()
+ .map(fa -> new PrioritizedFileAttributes(fa, prioritizer.apply(fa)))
+ .toList();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java
new file mode 100644
index 00000000000..6b5f60a66c7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
new file mode 100644
index 00000000000..f46950aa448
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -0,0 +1,433 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.identity;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.security.SslContextBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzRole;
+import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
+import com.yahoo.vespa.athenz.client.zts.InstanceIdentity;
+import com.yahoo.vespa.athenz.client.zts.ZtsClient;
+import com.yahoo.vespa.athenz.client.zts.ZtsClientException;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient;
+import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
+import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator;
+import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient;
+import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
+import com.yahoo.vespa.athenz.utils.SiaUtils;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.NODE;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.TENANT;
+
+/**
+ * A maintainer that is responsible for providing and refreshing Athenz credentials for a container.
+ *
+ * @author bjorncs
+ */
+public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
+
+ private static final Logger logger = Logger.getLogger(AthenzCredentialsMaintainer.class.getName());
+
+ private static final Duration EXPIRY_MARGIN = Duration.ofDays(1);
+ private static final Duration REFRESH_PERIOD = Duration.ofDays(1);
+ private static final Duration REFRESH_BACKOFF = Duration.ofHours(1); // Backoff when refresh fails to ensure ZTS is not DDoS'ed.
+
+ private static final String CONTAINER_SIA_DIRECTORY = "/var/lib/sia";
+ private static final String LEGACY_SIA_DIRECTORY = "/opt/vespa/var/vespa/sia";
+
+ private final Path ztsTrustStorePath;
+ private final Timer timer;
+ private final String certificateDnsSuffix;
+ private final ServiceIdentityProvider hostIdentityProvider;
+ private final IdentityDocumentClient identityDocumentClient;
+
+ // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts
+ private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>();
+
+ public AthenzCredentialsMaintainer(Path ztsTrustStorePath,
+ ConfigServerInfo configServerInfo,
+ String certificateDnsSuffix,
+ ServiceIdentityProvider hostIdentityProvider,
+ Timer timer) {
+ this.ztsTrustStorePath = ztsTrustStorePath;
+ this.certificateDnsSuffix = certificateDnsSuffix;
+ this.hostIdentityProvider = hostIdentityProvider;
+ this.identityDocumentClient = new DefaultIdentityDocumentClient(
+ configServerInfo.getLoadBalancerEndpoint(),
+ hostIdentityProvider,
+ new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity())));
+ this.timer = timer;
+ }
+
+ public boolean converge(NodeAgentContext context) {
+ var modified = false;
+ modified |= maintain(context, NODE);
+
+ if (context.zone().getSystemName().isPublic())
+ return modified;
+
+ modified |= maintain(context, TENANT);
+ return modified;
+ }
+
+ private boolean maintain(NodeAgentContext context, IdentityType identityType) {
+ if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false;
+
+ try {
+ var modified = false;
+ context.log(logger, Level.FINE, "Checking certificate");
+ ContainerPath siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa());
+ ContainerPath identityDocumentFile = siaDirectory.resolve(identityType.getIdentityDocument());
+ Optional<AthenzIdentity> optionalAthenzIdentity = getAthenzIdentity(context, identityType, identityDocumentFile);
+ if (optionalAthenzIdentity.isEmpty())
+ return false;
+ AthenzIdentity athenzIdentity = optionalAthenzIdentity.get();
+ ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity);
+ ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity);
+ if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile) || !Files.exists(identityDocumentFile)) {
+ context.log(logger, "Certificate/private key/identity document file does not exist");
+ Files.createDirectories(privateKeyFile.getParent());
+ Files.createDirectories(certificateFile.getParent());
+ Files.createDirectories(identityDocumentFile.getParent());
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity);
+ modified = true;
+ }
+
+ X509Certificate certificate = readCertificateFromFile(certificateFile);
+ Instant now = timer.currentTime();
+ Instant expiry = certificate.getNotAfter().toInstant();
+ var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile);
+ if (refreshIdentityDocument(doc, context)) {
+ context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion());
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity);
+ modified = true;
+ } else if (isCertificateExpired(expiry, now)) {
+ context.log(logger, "Certificate has expired (expiry=%s)", expiry.toString());
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity);
+ modified = true;
+ }
+
+ Duration age = Duration.between(certificate.getNotBefore().toInstant(), now);
+ if (shouldRefreshCredentials(age)) {
+ context.log(logger, "Certificate is ready to be refreshed (age=%s)", age.toString());
+ if (shouldThrottleRefreshAttempts(context.containerName(), now)) {
+ context.log(logger, Level.WARNING, String.format(
+ "Skipping refresh attempt as last refresh was on %s (less than %s ago)",
+ lastRefreshAttempt.get(context.containerName()).toString(), REFRESH_BACKOFF.toString()));
+ } else {
+ lastRefreshAttempt.put(context.containerName(), now);
+ refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc.identityDocument(), identityType, athenzIdentity);
+ modified = true;
+ }
+ }
+
+ if (identityType == TENANT) {
+ modified |= maintainRoleCertificates(context, siaDirectory, privateKeyFile, certificateFile, athenzIdentity, doc.identityDocument());
+ copyCredsToLegacyPath(context, privateKeyFile, certificateFile);
+ }
+ return modified;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private boolean maintainRoleCertificates(NodeAgentContext context,
+ ContainerPath siaDirectory,
+ ContainerPath privateKeyFile,
+ ContainerPath certificateFile,
+ AthenzIdentity identity,
+ IdentityDocument identityDocument) {
+ var modified = false;
+
+ for (var role : getRoleList(context)) {
+ try {
+ var roleCertificatePath = siaDirectory.resolve("certs")
+ .resolve(String.format("%s.cert.pem", role));
+ var roleKeyPath = siaDirectory.resolve("keys")
+ .resolve(String.format("%s.key.pem", role));
+ if (Files.notExists(roleCertificatePath)) {
+ writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role);
+ modified = true;
+ } else if (shouldRefreshCertificate(context, roleCertificatePath)) {
+ writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role);
+ modified = true;
+ }
+ } catch (IOException e) {
+ context.log(logger, Level.WARNING, "Failed to maintain role certificate " + role, e);
+ }
+ }
+ return modified;
+ }
+
+ private boolean shouldRefreshCertificate(NodeAgentContext context, ContainerPath certificatePath) throws IOException {
+ var certificate = readCertificateFromFile(certificatePath);
+ var now = timer.currentTime();
+ var shouldRefresh = now.isAfter(certificate.getNotAfter().toInstant()) ||
+ now.isAfter(certificate.getNotBefore().toInstant().plus(REFRESH_PERIOD));
+ return !shouldThrottleRefreshAttempts(context.containerName(), now) &&
+ shouldRefresh;
+ }
+
+ private void writeRoleCredentials(NodeAgentContext context,
+ ContainerPath privateKeyFile,
+ ContainerPath certificateFile,
+ ContainerPath roleCertificatePath,
+ ContainerPath roleKeyPath,
+ AthenzIdentity identity,
+ IdentityDocument identityDocument,
+ String role) throws IOException {
+ HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true;
+ var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
+ var athenzRole = AthenzRole.fromResourceNameString(role);
+
+ try (ZtsClient ztsClient = ztsClient(identityDocument.ztsUrl(), privateKeyFile, certificateFile, ztsHostNameVerifier)) {
+ var csrGenerator = new CsrGenerator(certificateDnsSuffix, identityDocument.providerService().getFullName());
+ var csr = csrGenerator.generateRoleCsr(
+ identity, athenzRole, identityDocument.providerUniqueId(), identityDocument.clusterType(), keyPair);
+ var roleCertificate = ztsClient.getRoleCertificate(athenzRole, csr);
+ writePrivateKeyAndCertificate(roleKeyPath, keyPair.getPrivate(), roleCertificatePath, roleCertificate);
+ context.log(logger, "Role certificate successfully retrieved written to file " + roleCertificatePath.pathInContainer());
+ }
+ }
+
+ private boolean refreshIdentityDocument(SignedIdentityDocument signedIdentityDocument, NodeAgentContext context) {
+ int expectedVersion = documentVersion(context);
+ return signedIdentityDocument.outdated() || signedIdentityDocument.documentVersion() != expectedVersion;
+ }
+
+ public void clearCredentials(NodeAgentContext context) {
+ FileFinder.files(context.paths().of(CONTAINER_SIA_DIRECTORY))
+ .deleteRecursively(context);
+ lastRefreshAttempt.remove(context.containerName());
+ }
+
+ @Override
+ public Duration certificateLifetime(NodeAgentContext context) {
+ ContainerPath containerSiaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY);
+ ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity());
+ try {
+ X509Certificate certificate = readCertificateFromFile(certificateFile);
+ Instant now = timer.currentTime();
+ Instant expiry = certificate.getNotAfter().toInstant();
+ return Duration.between(now, expiry);
+ } catch (IOException e) {
+ context.log(logger, Level.SEVERE, "Unable to read certificate at " + certificateFile, e);
+ return Duration.ZERO;
+ }
+ }
+
+ @Override
+ public String name() {
+ return "node-certificate";
+ }
+
+ private boolean shouldRefreshCredentials(Duration age) {
+ return age.compareTo(REFRESH_PERIOD) >= 0;
+ }
+
+ private boolean shouldThrottleRefreshAttempts(ContainerName containerName, Instant now) {
+ return REFRESH_BACKOFF.compareTo(
+ Duration.between(
+ lastRefreshAttempt.getOrDefault(containerName, Instant.EPOCH),
+ now)) > 0;
+ }
+
+ private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType, AthenzIdentity identity) {
+ KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
+ SignedIdentityDocument signedDoc = signedIdentityDocument(context, identityType);
+ IdentityDocument doc = signedDoc.identityDocument();
+ CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName());
+ Pkcs10Csr csr = csrGenerator.generateInstanceCsr(
+ identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair);
+
+ // Allow all zts hosts while removing SIS
+ HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true;
+ try (ZtsClient ztsClient = ztsClient(doc.ztsUrl(), hostIdentityProvider.privateKeyPath(), hostIdentityProvider.certificatePath(), ztsHostNameVerifier)) {
+ InstanceIdentity instanceIdentity =
+ ztsClient.registerInstance(
+ doc.providerService(),
+ identity,
+ EntityBindingsMapper.toAttestationData(signedDoc),
+ csr);
+ EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedDoc);
+ writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate());
+ context.log(logger, "Instance successfully registered and credentials written to file");
+ }
+ }
+
+ private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile,
+ ContainerPath identityDocumentFile, IdentityDocument doc, IdentityType identityType, AthenzIdentity identity) {
+ try {
+ // Do not rotate private key on every refresh.
+ // TODO: rotate key pair only on Vespa upgrade or similar
+ PrivateKey privateKey = readPrivateKeyFromFile(privateKeyFile);
+ KeyPair keyPair = KeyUtils.toKeyPair(privateKey);
+ CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName());
+ Pkcs10Csr csr = csrGenerator.generateInstanceCsr(
+ identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair);
+
+ // Allow all zts hosts while removing SIS
+ HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true;
+ try (ZtsClient ztsClient = ztsClient(doc.ztsUrl(), privateKeyFile, certificateFile, ztsHostNameVerifier)) {
+ InstanceIdentity instanceIdentity =
+ ztsClient.refreshInstance(
+ doc.providerService(),
+ identity,
+ doc.providerUniqueId().asDottedString(),
+ csr);
+ writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate());
+ context.log(logger, "Instance successfully refreshed and credentials written to file");
+ } catch (ZtsClientException e) {
+ if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) {
+ context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e);
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, identity);
+ } else {
+ throw e;
+ }
+ }
+ } catch (Exception e) {
+ context.log(logger, Level.SEVERE, "Certificate refresh failed: " + e.getMessage(), e);
+ }
+ }
+
+
+ private static void writePrivateKeyAndCertificate(ContainerPath privateKeyFile,
+ PrivateKey privateKey,
+ ContainerPath certificateFile,
+ X509Certificate certificate) {
+ writeFile(privateKeyFile, KeyUtils.toPem(privateKey));
+ writeFile(certificateFile, X509CertificateUtils.toPem(certificate));
+ }
+
+ private static void writeFile(ContainerPath path, String utf8Content) {
+ new UnixPath(path.resolveSibling(path.getFileName() + ".tmp"))
+ .writeUtf8File(utf8Content, "r--------")
+ .atomicMove(path);
+ }
+
+ private static X509Certificate readCertificateFromFile(ContainerPath certificateFile) throws IOException {
+ String pemEncodedCertificate = new String(Files.readAllBytes(certificateFile));
+ return X509CertificateUtils.fromPem(pemEncodedCertificate);
+ }
+
+ private static PrivateKey readPrivateKeyFromFile(ContainerPath privateKeyFile) throws IOException {
+ String pemEncodedKey = new String(Files.readAllBytes(privateKeyFile));
+ return KeyUtils.fromPemEncodedPrivateKey(pemEncodedKey);
+ }
+
+ private static boolean isCertificateExpired(Instant expiry, Instant now) {
+ return now.isAfter(expiry.minus(EXPIRY_MARGIN));
+ }
+
+ private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) {
+ return switch (identityType) {
+ case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value(), documentVersion(context));
+ case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)).get();
+ };
+ }
+
+ private Optional<AthenzIdentity> getAthenzIdentity(NodeAgentContext context, IdentityType identityType, ContainerPath identityDocumentFile) {
+ return switch (identityType) {
+ case NODE -> Optional.of(context.identity());
+ case TENANT -> getTenantIdentity(context, identityDocumentFile);
+ };
+ }
+
+ private Optional<AthenzIdentity> getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) {
+ if (Files.exists(identityDocumentFile)) {
+ return Optional.of(EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).identityDocument().serviceIdentity());
+ } else {
+ return identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context))
+ .map(doc -> doc.identityDocument().serviceIdentity());
+ }
+ }
+
+ private void copyCredsToLegacyPath(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile) throws IOException {
+ var legacySiaDirectory = context.paths().of(LEGACY_SIA_DIRECTORY, context.users().vespa());
+ var keysDirectory = legacySiaDirectory.resolve("keys");
+ var certsDirectory = legacySiaDirectory.resolve("certs");
+ Files.createDirectories(keysDirectory);
+ Files.createDirectories(certsDirectory);
+ writeFile(certsDirectory.resolve(certificateFile.getFileName()), Files.readString(certificateFile));
+ writeFile(keysDirectory.resolve(privateKeyFile.getFileName()), Files.readString(privateKeyFile));
+ }
+
+ /** Get the document version to ask for */
+ private int documentVersion(NodeAgentContext context) {
+ return SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION;
+ }
+
+ private ZtsClient ztsClient(URI ztsEndpoint, Path privateKeyFile, Path certificateFile, HostnameVerifier hostnameVerifier) {
+ SSLContext sslContext = new SslContextBuilder()
+ .withKeyStore(privateKeyFile, certificateFile)
+ .withTrustStore(ztsTrustStorePath)
+ .build();
+ return new DefaultZtsClient.Builder(ztsEndpoint)
+ .withSslContext(sslContext)
+ .withHostnameVerifier(hostnameVerifier)
+ .build();
+ }
+
+ private List<String> getRoleList(NodeAgentContext context) {
+ try {
+ return identityDocumentClient.getNodeRoles(context.hostname().value());
+ } catch (Exception e) {
+ context.log(logger, Level.WARNING, "Failed to retrieve role list", e);
+ return List.of();
+ }
+ }
+
+ enum IdentityType {
+ NODE("vespa-node-identity-document.json"),
+ TENANT("vespa-tenant-identity-document.json");
+
+ private final String identityDocument;
+ IdentityType(String identityDocument) {
+ this.identityDocument = identityDocument;
+ }
+
+ public String getIdentityDocument() {
+ return identityDocument;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java
new file mode 100644
index 00000000000..0e387ac2731
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java
@@ -0,0 +1,29 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.identity;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+
+import java.time.Duration;
+
+/**
+ * A maintainer that is responsible for providing and refreshing credentials for a container.
+ *
+ * @author freva
+ */
+public interface CredentialsMaintainer {
+
+ /**
+ * Creates/refreshes credentials for the given NodeAgentContext. Called for every NodeAgent tick.
+ * @return false if already converged, i.e. was a no-op.
+ */
+ boolean converge(NodeAgentContext context);
+
+ /** Remove any existing credentials. This method is called just before container data is archived. */
+ void clearCredentials(NodeAgentContext context);
+
+ /** Get time until the certificate expires. Invoked each time metrics are collected. */
+ Duration certificateLifetime(NodeAgentContext context);
+
+ /** Name used when reporting metrics */
+ String name();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java
new file mode 100644
index 00000000000..a48f4f45aa8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.identity;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java
new file mode 100644
index 00000000000..2ef78aa9c54
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java
new file mode 100644
index 00000000000..ee8da84e9cb
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java
@@ -0,0 +1,55 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.Optional;
+
+/**
+ * An artifact file produced by a {@link ArtifactProducer}.
+ *
+ * @author bjorncs
+ */
+class Artifact {
+
+ enum Classification {
+ CONFIDENTIAL("confidential"),
+ INTERNAL("internal");
+
+ private final String value;
+ Classification(String value) { this.value = value; }
+ public String value() { return value; }
+ }
+
+ private final Classification classification;
+ private final ContainerPath file;
+ private final boolean compressOnUpload;
+
+ private Artifact(Builder builder) {
+ if (builder.file == null) {
+ throw new IllegalArgumentException("No file specified");
+ }
+ this.file = builder.file;
+ this.classification = builder.classification;
+ this.compressOnUpload = Boolean.TRUE.equals(builder.compressOnUpload);
+ }
+
+ static Builder newBuilder() { return new Builder(); }
+
+ Optional<Classification> classification() { return Optional.ofNullable(classification); }
+ ContainerPath file() { return file; }
+ boolean compressOnUpload() { return compressOnUpload; }
+
+ static class Builder {
+ private Classification classification;
+ private ContainerPath file;
+ private Boolean compressOnUpload;
+
+ private Builder() {}
+
+ Builder classification(Classification c) { this.classification = c; return this; }
+ Builder file(ContainerPath f) { this.file = f; return this; }
+ Builder compressOnUpload() { this.compressOnUpload = true; return this; }
+ Artifact build() { return new Artifact(this); }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java
new file mode 100644
index 00000000000..87ab1ef8bf5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java
@@ -0,0 +1,37 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+
+import java.util.List;
+import java.util.OptionalDouble;
+
+/**
+ * Produces service dump artifacts.
+ *
+ * @author bjorncs
+ */
+interface ArtifactProducer {
+
+ String artifactName();
+ String description();
+ List<Artifact> produceArtifacts(Context ctx);
+
+ interface Context {
+ String serviceId();
+ int servicePid();
+ CommandResult executeCommandInNode(List<String> command, boolean logOutput);
+ ContainerPath outputContainerPath();
+ ContainerPath containerPathUnderVespaHome(String relativePath);
+ Options options();
+
+ interface Options {
+ OptionalDouble duration();
+ boolean callGraphRecording();
+ boolean sendProfilingSignal();
+ }
+ }
+
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java
new file mode 100644
index 00000000000..939bebc5fac
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java
@@ -0,0 +1,109 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.yolean.concurrent.Sleeper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * @author bjorncs
+ */
+class ArtifactProducers {
+
+ private final Map<String, ArtifactProducer> producers;
+ private final Map<String, List<ArtifactProducer>> aliases;
+
+ private ArtifactProducers(Set<ArtifactProducer> producers,
+ Map<String, List<Class<? extends ArtifactProducer>>> aliases) {
+ var producerMap = producers.stream()
+ .collect(Collectors.toMap(ArtifactProducer::artifactName, Function.identity()));
+ Map<String, List<ArtifactProducer>> aliasMap = new HashMap<>();
+ aliases.forEach((alias, mapping) -> {
+ List<ArtifactProducer> concreteMapping = mapping.stream()
+ .map(type -> producers.stream()
+ .filter(p -> p.getClass().equals(type))
+ .findAny()
+ .orElseThrow(() -> new IllegalArgumentException("No producer of type " + type)))
+ .toList();
+ if (producerMap.containsKey(alias)) {
+ throw new IllegalStateException("Alias name '" + alias + "' conflicts with producer");
+ }
+ aliasMap.put(alias, concreteMapping);
+ });
+ this.producers = producerMap;
+ this.aliases = aliasMap;
+ }
+
+ static ArtifactProducers createDefault(Sleeper sleeper) {
+ var producers = Set.of(
+ new PerfReporter(),
+ new JvmDumper.JavaFlightRecorder(sleeper),
+ new JvmDumper.HeapDump(),
+ new JvmDumper.Jmap(),
+ new JvmDumper.Jstat(),
+ new JvmDumper.Jstack(),
+ new PmapReporter(),
+ new VespaLogDumper(sleeper),
+ new ZooKeeperSnapshotDumper(),
+ new ConfigDumper());
+ var aliases =
+ Map.of(
+ "jvm-dump",
+ List.of(
+ JvmDumper.HeapDump.class, JvmDumper.Jmap.class, JvmDumper.Jstat.class,
+ JvmDumper.Jstack.class, VespaLogDumper.class)
+ );
+ return new ArtifactProducers(producers, aliases);
+ }
+
+ static ArtifactProducers createCustom(Set<ArtifactProducer> producers,
+ Map<String, List<Class<? extends ArtifactProducer>>> aliases) {
+ return new ArtifactProducers(producers, aliases);
+ }
+
+ List<ArtifactProducer> resolve(List<String> requestedArtifacts) {
+ List<ArtifactProducer> resolved = new ArrayList<>();
+ for (String artifact : requestedArtifacts) {
+ if (aliases.containsKey(artifact)) {
+ aliases.get(artifact).stream()
+ .filter(p -> !resolved.contains(p))
+ .forEach(resolved::add);
+ } else if (producers.containsKey(artifact)) {
+ ArtifactProducer producer = producers.get(artifact);
+ if (!resolved.contains(producer)) {
+ resolved.add(producer);
+ }
+ } else {
+ throw createInvalidArtifactException(artifact);
+ }
+ }
+ return resolved;
+ }
+
+ private IllegalArgumentException createInvalidArtifactException(String artifact) {
+ String producersString = producers.keySet().stream()
+ .map(a -> "'" + a + "'")
+ .sorted()
+ .collect(Collectors.joining(", ", "[", "]"));
+ String aliasesString = aliases.entrySet().stream()
+ .map(e -> String.format(
+ "'%s': %s",
+ e.getKey(),
+ e.getValue().stream()
+ .map(p -> "'" + p.artifactName() + "'")
+ .sorted()
+ .collect(Collectors.joining(", ", "[", "]")))
+ )
+ .collect(Collectors.joining(", ", "[", "]"));
+ String msg = String.format(
+ "Invalid artifact type '%s'. Valid types are %s and valid aliases are %s",
+ artifact, producersString, aliasesString);
+ return new IllegalArgumentException(msg);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java
new file mode 100644
index 00000000000..8eadabf07cf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java
@@ -0,0 +1,35 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL;
+
+/**
+ * Performs dump of config on a node.
+ *
+ * @author hmusum
+ */
+class ConfigDumper implements ArtifactProducer {
+ @Override public String artifactName() { return "config-dump"; }
+ @Override public String description() { return "Dumps config"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath dir = ctx.outputContainerPath().resolve("config");
+ ContainerPath configDump = ctx.outputContainerPath().resolve("config-dump.tar.zst");
+ List<String> cmd = List.of("bash", "-c",
+ String.format("mkdir -p %s; /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache %s; tar cvf %s.tar %s; zstd %s.tar -o %s",
+ dir.pathInContainer(),
+ dir.pathInContainer(),
+ dir.pathInContainer(),
+ dir.pathInContainer(),
+ dir.pathInContainer(),
+ configDump.pathInContainer()));
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(Artifact.newBuilder().classification(INTERNAL).file(configDump).build());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java
new file mode 100644
index 00000000000..360a212646f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java
@@ -0,0 +1,103 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.yolean.concurrent.Sleeper;
+
+import java.time.Duration;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL;
+
+/**
+ * @author bjorncs
+ */
+class JvmDumper {
+ private JvmDumper() {}
+
+ static class HeapDump implements ArtifactProducer {
+ @Override public String artifactName() { return "jvm-heap-dump"; }
+ @Override public String description() { return "JVM heap dump"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath heapDumpFile = ctx.outputContainerPath().resolve("jvm-heap-dump.hprof");
+ List<String> cmd = List.of(
+ "jmap", "-dump:live,format=b,file=" + heapDumpFile.pathInContainer(), Integer.toString(ctx.servicePid()));
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(
+ Artifact.newBuilder().classification(CONFIDENTIAL).file(heapDumpFile).compressOnUpload().build());
+ }
+ }
+
+ static class Jmap implements ArtifactProducer {
+ @Override public String artifactName() { return "jvm-jmap"; }
+ @Override public String description() { return "JVM jmap output"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath jmapReport = ctx.outputContainerPath().resolve("jvm-jmap.txt");
+ List<String> cmd = List.of("bash", "-c", "jhsdb jmap --heap --pid " + ctx.servicePid() + " > " + jmapReport.pathInContainer());
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(Artifact.newBuilder().classification(INTERNAL).file(jmapReport).build());
+ }
+ }
+
+ static class Jstat implements ArtifactProducer {
+ @Override public String artifactName() { return "jvm-jstat"; }
+ @Override public String description() { return "JVM jstat output"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath jstatReport = ctx.outputContainerPath().resolve("jvm-jstat.txt");
+ List<String> cmd = List.of("bash", "-c", "jstat -gcutil " + ctx.servicePid() + " > " + jstatReport.pathInContainer());
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(Artifact.newBuilder().classification(INTERNAL).file(jstatReport).build());
+ }
+ }
+
+ static class Jstack implements ArtifactProducer {
+ @Override public String artifactName() { return "jvm-jstack"; }
+ @Override public String description() { return "JVM jstack output"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath jstackReport = ctx.outputContainerPath().resolve("jvm-jstack.txt");
+ ctx.executeCommandInNode(List.of("bash", "-c", "jstack " + ctx.servicePid() + " > " + jstackReport.pathInContainer()), true);
+ return List.of(Artifact.newBuilder().classification(INTERNAL).file(jstackReport).build());
+ }
+ }
+
+ static class JavaFlightRecorder implements ArtifactProducer {
+ private final Sleeper sleeper;
+
+ JavaFlightRecorder(Sleeper sleeper) { this.sleeper = sleeper; }
+
+ @Override public String artifactName() { return "jvm-jfr"; }
+ @Override public String description() { return "Java Flight Recorder recording"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(ArtifactProducer.Context ctx) {
+ int seconds = (int) (ctx.options().duration().orElse(30.0));
+ ContainerPath outputFile = ctx.outputContainerPath().resolve("recording.jfr");
+ List<String> startCommand = List.of("jcmd", Integer.toString(ctx.servicePid()), "JFR.start", "name=host-admin",
+ "path-to-gc-roots=true", "settings=profile", "filename=" + outputFile.pathInContainer(), "duration=" + seconds + "s");
+ ctx.executeCommandInNode(startCommand, true);
+ sleeper.sleep(Duration.ofSeconds(seconds).plusSeconds(1));
+ int maxRetries = 10;
+ List<String> checkCommand = List.of("jcmd", Integer.toString(ctx.servicePid()), "JFR.check", "name=host-admin");
+ for (int i = 0; i < maxRetries; i++) {
+ boolean stillRunning = ctx.executeCommandInNode(checkCommand, true).getOutputLines().stream()
+ .anyMatch(l -> l.contains("name=host-admin") && l.contains("running"));
+ if (!stillRunning) {
+ Artifact a = Artifact.newBuilder()
+ .classification(CONFIDENTIAL).file(outputFile).compressOnUpload().build();
+ return List.of(a);
+ }
+ sleeper.sleep(Duration.ofSeconds(1));
+ }
+ throw new RuntimeException("Failed to wait for JFR dump to complete after " + maxRetries + " retries");
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java
new file mode 100644
index 00000000000..f4b4307b0d7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java
@@ -0,0 +1,40 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL;
+
+/**
+ * @author bjorncs
+ */
+class PerfReporter implements ArtifactProducer {
+
+ PerfReporter() {}
+
+ @Override public String artifactName() { return "perf-report"; }
+ @Override public String description() { return "Perf recording and report"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ int duration = (int)ctx.options().duration().orElse(30.0);
+ List<String> perfRecordCommand = new ArrayList<>(List.of("perf", "record"));
+ if (ctx.options().callGraphRecording()) {
+ perfRecordCommand.add("-g");
+ }
+ ContainerPath recordFile = ctx.outputContainerPath().resolve("perf-record.bin");
+ perfRecordCommand.addAll(
+ List.of("--output=" + recordFile.pathInContainer(),
+ "--pid=" + ctx.servicePid(), "sleep", Integer.toString(duration)));
+ ctx.executeCommandInNode(perfRecordCommand, true);
+ ContainerPath reportFile = ctx.outputContainerPath().resolve("perf-report.txt");
+ ctx.executeCommandInNode(List.of("bash", "-c", "perf report --input=" + recordFile.pathInContainer() + " > " + reportFile.pathInContainer()), true);
+ return List.of(
+ Artifact.newBuilder().classification(CONFIDENTIAL).file(recordFile).compressOnUpload().build(),
+ Artifact.newBuilder().classification(INTERNAL).file(reportFile).build());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java
new file mode 100644
index 00000000000..8f8feb57c27
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java
@@ -0,0 +1,24 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL;
+
+/**
+ * @author bjorncs
+ */
+class PmapReporter implements ArtifactProducer {
+ @Override public String artifactName() { return "pmap"; }
+ @Override public String description() { return "Pmap report"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath pmapReport = ctx.outputContainerPath().resolve("pmap.txt");
+ List<String> cmd = List.of("bash", "-c", "pmap -x " + ctx.servicePid() + " > " + pmapReport.pathInContainer());
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(Artifact.newBuilder().classification(INTERNAL).file(pmapReport).build());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java
new file mode 100644
index 00000000000..744eeefca07
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java
@@ -0,0 +1,143 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.List;
+
+/**
+ * JSON representation of Vespa service dump report.
+ *
+ * @author bjorncs
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+class ServiceDumpReport extends BaseReport {
+
+ public static final String REPORT_ID = "serviceDump";
+
+ private static final String STARTED_AT_FIELD = "startedAt";
+ private static final String COMPLETED_AT_FIELD = "completedAt";
+ private static final String FAILED_AT_FIELD = "failedAt";
+ private static final String LOCATION_FIELD = "location";
+ private static final String CONFIG_ID_FIELD = "configId";
+ private static final String EXPIRE_AT_FIELD = "expireAt";
+ private static final String ERROR_FIELD = "error";
+ private static final String ARTIFACTS_FIELD = "artifacts";
+ private static final String DUMP_OPTIONS_FIELD = "dumpOptions";
+
+ private final Long startedAt;
+ private final Long completedAt;
+ private final Long failedAt;
+ private final String location;
+ private final String configId;
+ private final Long expireAt;
+ private final String error;
+ private final List<String> artifacts;
+ private final DumpOptions dumpOptions;
+
+ @JsonCreator
+ public ServiceDumpReport(@JsonProperty(CREATED_FIELD) Long createdAt,
+ @JsonProperty(STARTED_AT_FIELD) Long startedAt,
+ @JsonProperty(COMPLETED_AT_FIELD) Long completedAt,
+ @JsonProperty(FAILED_AT_FIELD) Long failedAt,
+ @JsonProperty(LOCATION_FIELD) String location,
+ @JsonProperty(CONFIG_ID_FIELD) String configId,
+ @JsonProperty(EXPIRE_AT_FIELD) Long expireAt,
+ @JsonProperty(ERROR_FIELD) String error,
+ @JsonProperty(ARTIFACTS_FIELD) List<String> artifacts,
+ @JsonProperty(DUMP_OPTIONS_FIELD) DumpOptions dumpOptions) {
+ super(createdAt, null);
+ this.startedAt = startedAt;
+ this.completedAt = completedAt;
+ this.failedAt = failedAt;
+ this.location = location;
+ this.configId = configId;
+ this.expireAt = expireAt;
+ this.error = error;
+ this.artifacts = artifacts;
+ this.dumpOptions = dumpOptions;
+ }
+
+ public static ServiceDumpReport createRequestReport(Instant createdAt, Instant expireAt, String configId,
+ List<String> artifacts, DumpOptions options) {
+ return new ServiceDumpReport(
+ createdAt.toEpochMilli(), null, null, null, null, configId,
+ expireAt != null ? expireAt.toEpochMilli() : null, null, artifacts, options);
+ }
+
+ public static ServiceDumpReport createStartedReport(ServiceDumpReport request, Instant startedAt) {
+ return new ServiceDumpReport(
+ request.getCreatedMillisOrNull(), startedAt.toEpochMilli(), null, null, null, request.configId(),
+ request.expireAt(), null, request.artifacts(), request.dumpOptions());
+ }
+
+ public static ServiceDumpReport createSuccessReport(
+ ServiceDumpReport request, Instant startedAt, Instant completedAt, URI location) {
+ return new ServiceDumpReport(
+ request.getCreatedMillisOrNull(), startedAt.toEpochMilli(), completedAt.toEpochMilli(), null,
+ location.toString(), request.configId(), request.expireAt(), null, request.artifacts(),
+ request.dumpOptions());
+ }
+
+ public static ServiceDumpReport createErrorReport(
+ ServiceDumpReport reqOrNull, Instant startedAt, Instant failedAt, String message) {
+ Long createdAt = reqOrNull != null ? reqOrNull.getCreatedMillisOrNull() : Long.valueOf(startedAt.toEpochMilli());
+ String configId = reqOrNull != null ? reqOrNull.configId() : "unknown";
+ Long expireAt = reqOrNull != null ? reqOrNull.expireAt() : null;
+ List<String> artifacts = reqOrNull != null ? reqOrNull.artifacts() : List.of();
+ DumpOptions dumpOptions = reqOrNull != null ? reqOrNull.dumpOptions() : null;
+ return new ServiceDumpReport(
+ createdAt, startedAt.toEpochMilli(), null, failedAt.toEpochMilli(), null,
+ configId, expireAt, message, artifacts, dumpOptions);
+ }
+
+ @JsonGetter(STARTED_AT_FIELD) public Long startedAt() { return startedAt; }
+ @JsonGetter(COMPLETED_AT_FIELD) public Long completedAt() { return completedAt; }
+ @JsonGetter(FAILED_AT_FIELD) public Long failedAt() { return failedAt; }
+ @JsonGetter(LOCATION_FIELD) public String location() { return location; }
+ @JsonGetter(CONFIG_ID_FIELD) public String configId() { return configId; }
+ @JsonGetter(EXPIRE_AT_FIELD) public Long expireAt() { return expireAt; }
+ @JsonGetter(ERROR_FIELD) public String error() { return error; }
+ @JsonGetter(ARTIFACTS_FIELD) public List<String> artifacts() { return artifacts; }
+ @JsonGetter(DUMP_OPTIONS_FIELD) public DumpOptions dumpOptions() { return dumpOptions; }
+
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class DumpOptions {
+
+ private static final String CALL_GRAPH_RECORDING_FIELD = "callGraphRecording";
+ private static final String DURATION_FIELD = "duration";
+ private static final String SEND_PROFILING_SIGNAL_FIELD = "sendProfilingSignal";
+
+ private final Boolean callGraphRecording;
+ private final Double duration;
+ private final Boolean sendProfilingSignal;
+
+ @JsonCreator
+ public DumpOptions(@JsonProperty(CALL_GRAPH_RECORDING_FIELD) Boolean callGraphRecording,
+ @JsonProperty(DURATION_FIELD) Double duration,
+ @JsonProperty(SEND_PROFILING_SIGNAL_FIELD) Boolean sendProfilingSignal) {
+ this.callGraphRecording = callGraphRecording;
+ this.duration = duration;
+ this.sendProfilingSignal = sendProfilingSignal;
+ }
+
+ @JsonGetter(CALL_GRAPH_RECORDING_FIELD) public Boolean callGraphRecording() { return callGraphRecording; }
+ @JsonGetter(DURATION_FIELD) public Double duration() { return duration; }
+ @JsonGetter(SEND_PROFILING_SIGNAL_FIELD) public Boolean sendProfilingSignal() { return sendProfilingSignal; }
+ }
+
+ @JsonIgnore public boolean isCompletedOrFailed() { return !isNullTimestamp(failedAt) || !isNullTimestamp(completedAt); }
+
+ public static boolean isNullTimestamp(Long timestamp) { return timestamp == null || timestamp == 0; }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java
new file mode 100644
index 00000000000..32814e38d39
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java
@@ -0,0 +1,47 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.yolean.concurrent.Sleeper;
+
+import java.nio.file.Files;
+import java.time.Duration;
+import java.util.List;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author bjorncs
+ */
+class VespaLogDumper implements ArtifactProducer {
+
+ private static final Logger log = Logger.getLogger(VespaLogDumper.class.getName());
+
+ private final Sleeper sleeper;
+
+ VespaLogDumper(Sleeper sleeper) { this.sleeper = sleeper; }
+
+ @Override public String artifactName() { return "vespa-log"; }
+ @Override public String description() { return "Current Vespa logs"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ if (ctx.options().sendProfilingSignal()) {
+ log.info("Sending SIGPROF to process to include vespa-malloc dump in Vespa log");
+ ctx.executeCommandInNode(List.of("kill", "-SIGPROF", Integer.toString(ctx.servicePid())), true);
+ sleeper.sleep(Duration.ofSeconds(3));
+ }
+ ContainerPath vespaLogFile = ctx.containerPathUnderVespaHome("logs/vespa/vespa.log");
+ ContainerPath destination = ctx.outputContainerPath().resolve("vespa.log");
+ if (Files.exists(vespaLogFile)) {
+ uncheck(() -> Files.copy(vespaLogFile, destination));
+ return List.of(
+ Artifact.newBuilder().classification(CONFIDENTIAL).file(destination).compressOnUpload().build());
+ } else {
+ log.info("Log file '" + vespaLogFile + "' does not exist");
+ return List.of();
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java
new file mode 100644
index 00000000000..1f474295660
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+
+/**
+ * @author bjorncs
+ */
+public interface VespaServiceDumper {
+ void processServiceDumpRequest(NodeAgentContext context);
+
+ VespaServiceDumper DUMMY_INSTANCE = context -> {};
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java
new file mode 100644
index 00000000000..1279d9a4b28
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java
@@ -0,0 +1,269 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.text.Lowercase;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.yolean.concurrent.Sleeper;
+
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.isNullTimestamp;
+
+/**
+ * Generates dumps for Vespa services and uploads resulting files to S3.
+ *
+ * @author bjorncs
+ */
+public class VespaServiceDumperImpl implements VespaServiceDumper {
+
+ private static final Logger log = Logger.getLogger(VespaServiceDumperImpl.class.getName());
+
+ private final ContainerOperations container;
+ private final SyncClient syncClient;
+ private final NodeRepository nodeRepository;
+ private final Timer timer;
+ private final ArtifactProducers artifactProducers;
+
+ public VespaServiceDumperImpl(ContainerOperations container, SyncClient syncClient, NodeRepository nodeRepository, Timer timer) {
+ this(ArtifactProducers.createDefault(Sleeper.DEFAULT), container, syncClient, nodeRepository, timer);
+ }
+
+ // For unit testing
+ VespaServiceDumperImpl(ArtifactProducers producers, ContainerOperations container, SyncClient syncClient,
+ NodeRepository nodeRepository, Timer timer) {
+ this.container = container;
+ this.syncClient = syncClient;
+ this.nodeRepository = nodeRepository;
+ this.timer = timer;
+ this.artifactProducers = producers;
+ }
+
+ @Override
+ public void processServiceDumpRequest(NodeAgentContext context) {
+ if (context.zone().getCloudName().equals(CloudName.GCP)) return;
+
+ Instant startedAt = timer.currentTime();
+ NodeSpec nodeSpec = context.node();
+ ServiceDumpReport request;
+ try {
+ request = nodeSpec.reports().getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class)
+ .orElse(null);
+ } catch (IllegalArgumentException | UncheckedIOException e) {
+ handleFailure(context, null, startedAt, e, "Invalid JSON in service dump request");
+ return;
+ }
+ if (request == null || request.isCompletedOrFailed()) {
+ context.log(log, Level.FINE, "No service dump requested or dump already completed/failed");
+ return;
+ }
+ if (isNullTimestamp(request.getCreatedMillisOrNull())) {
+ handleFailure(context, request, startedAt, "'createdMillis' is missing or null");
+ return;
+ }
+ String configId = request.configId();
+ if (configId == null) {
+ handleFailure(context, request, startedAt, "Service config id is missing from request");
+ return;
+ }
+ Instant expiry = expireAt(startedAt, request);
+ if (expiry.isBefore(startedAt)) {
+ handleFailure(context, request, startedAt, "Request already expired");
+ return;
+ }
+ List<String> requestedArtifacts = request.artifacts();
+ if (requestedArtifacts == null || requestedArtifacts.isEmpty()) {
+ handleFailure(context, request, startedAt, "No artifacts requested");
+ return;
+ }
+ ContainerPath directory = context.paths().underVespaHome("var/tmp/vespa-service-dump-" + request.getCreatedMillisOrNull());
+ UnixPath unixPathDirectory = new UnixPath(directory);
+ try {
+ context.log(log, Level.INFO,
+ "Creating service dump for " + configId + " requested at "
+ + Instant.ofEpochMilli(request.getCreatedMillisOrNull()));
+ storeReport(context, ServiceDumpReport.createStartedReport(request, startedAt));
+ if (unixPathDirectory.exists()) {
+ context.log(log, Level.INFO, "Removing existing directory '" + unixPathDirectory +"'.");
+ unixPathDirectory.deleteRecursively();
+ }
+ context.log(log, Level.INFO, "Creating '" + unixPathDirectory +"'.");
+ unixPathDirectory.createDirectory("rwxr-x---");
+ URI destination = serviceDumpDestination(nodeSpec, createDumpId(request));
+ ProducerContext producerCtx = new ProducerContext(context, directory, request);
+ List<Artifact> producedArtifacts = new ArrayList<>();
+ for (ArtifactProducer producer : artifactProducers.resolve(requestedArtifacts)) {
+ context.log(log, "Producing artifact of type '" + producer.artifactName() + "'");
+ producedArtifacts.addAll(producer.produceArtifacts(producerCtx));
+ }
+ uploadArtifacts(context, destination, producedArtifacts);
+ storeReport(context, ServiceDumpReport.createSuccessReport(request, startedAt, timer.currentTime(), destination));
+ } catch (Exception e) {
+ handleFailure(context, request, startedAt, e, e.getMessage());
+ } finally {
+ if (unixPathDirectory.exists()) {
+ context.log(log, Level.INFO, "Deleting directory '" + unixPathDirectory +"'.");
+ unixPathDirectory.deleteRecursively();
+ }
+ }
+ }
+
+ private void uploadArtifacts(NodeAgentContext ctx, URI destination,
+ List<Artifact> producedArtifacts) {
+ ApplicationId owner = ctx.node().owner().orElseThrow();
+ List<SyncFileInfo> filesToUpload = producedArtifacts.stream()
+ .map(a -> {
+ Compression compression = a.compressOnUpload() ? Compression.ZSTD : Compression.NONE;
+ String classification = a.classification().map(Artifact.Classification::value).orElse(null);
+ return SyncFileInfo.forServiceDump(destination, a.file(), compression, owner, classification);
+ })
+ .toList();
+ ctx.log(log, Level.INFO,
+ "Uploading " + filesToUpload.size() + " file(s) with destination " + destination);
+ if (!syncClient.sync(ctx, filesToUpload, Integer.MAX_VALUE)) {
+ throw new RuntimeException("Unable to upload all files");
+ }
+ ctx.log(log, Level.INFO, "Upload complete");
+ }
+
+ private static Instant expireAt(Instant startedAt, ServiceDumpReport request) {
+ return isNullTimestamp(request.expireAt())
+ ? startedAt.plus(7, ChronoUnit.DAYS)
+ : Instant.ofEpochMilli(request.expireAt());
+ }
+
+ private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt,
+ Exception failure, String message) {
+ context.log(log, Level.WARNING, failure.toString(), failure);
+ ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message);
+ storeReport(context, report);
+ }
+
+ private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, String message) {
+ context.log(log, Level.WARNING, message);
+ ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message);
+ storeReport(context, report);
+ }
+
+ private void storeReport(NodeAgentContext context, ServiceDumpReport report) {
+ NodeAttributes nodeAttributes = new NodeAttributes();
+ nodeAttributes.withReport(ServiceDumpReport.REPORT_ID, report.toJsonNode());
+ nodeRepository.updateNodeAttributes(context.hostname().value(), nodeAttributes);
+ }
+
+ static String createDumpId(ServiceDumpReport request) {
+ String sanitizedConfigId = Lowercase.toLowerCase(request.configId()).replaceAll("[^a-z_0-9]", "-");
+ return sanitizedConfigId + "-" + request.getCreatedMillisOrNull().toString();
+ }
+
+ private static URI serviceDumpDestination(NodeSpec spec, String dumpId) {
+ URI archiveUri = spec.archiveUri()
+ .orElseThrow(() -> new IllegalStateException("Archive URI is missing for " + spec.hostname()));
+ String targetDirectory = "service-dump/" + dumpId + "/";
+ return archiveUri.resolve(targetDirectory);
+ }
+
+ private class ProducerContext implements ArtifactProducer.Context, ArtifactProducer.Context.Options {
+
+ final NodeAgentContext nodeAgentCtx;
+ final ContainerPath path;
+ final ServiceDumpReport request;
+ volatile int pid = -1;
+
+ ProducerContext(NodeAgentContext nodeAgentCtx, ContainerPath path, ServiceDumpReport request) {
+ this.nodeAgentCtx = nodeAgentCtx;
+ this.path = path;
+ this.request = request;
+ }
+
+ @Override public String serviceId() { return request.configId(); }
+
+ @Override
+ public int servicePid() {
+ if (pid == -1) {
+ pid = findServicePid(serviceId());
+ }
+ return pid;
+ }
+
+ private int findServicePid(String serviceId) {
+ ContainerPath findPidBinary = nodeAgentCtx.paths().underVespaHome("libexec/vespa/find-pid");
+ CommandResult findPidResult = executeCommandInNode(List.of(findPidBinary.pathInContainer(), serviceId), true);
+ return Integer.parseInt(findPidResult.getOutput());
+ }
+
+ @Override
+ public CommandResult executeCommandInNode(List<String> command, boolean logOutput) {
+ CommandResult result = container.executeCommandInContainer(nodeAgentCtx, nodeAgentCtx.users().vespa(), command.toArray(new String[0]));
+ String cmdString = command.stream().map(s -> "'" + s + "'").collect(Collectors.joining(" ", "\"", "\""));
+ int exitCode = result.getExitCode();
+ String output = result.getOutput().trim();
+ String prefixedOutput = output.contains("\n")
+ ? "\n" + output
+ : (output.isEmpty() ? "<no output>" : output);
+ if (exitCode > 0) {
+ String errorMsg = logOutput
+ ? String.format("Failed to execute %s (exited with code %d): %s", cmdString, exitCode, prefixedOutput)
+ : String.format("Failed to execute %s (exited with code %d)", cmdString, exitCode);
+ throw new RuntimeException(errorMsg);
+ } else {
+ String logMsg = logOutput
+ ? String.format("Executed command %s. Exited with code %d and output: %s", cmdString, exitCode, prefixedOutput)
+ : String.format("Executed command %s. Exited with code %d.", cmdString, exitCode);
+ nodeAgentCtx.log(log, logMsg);
+ }
+ return result;
+ }
+
+ @Override public ContainerPath outputContainerPath() { return path; }
+
+ @Override
+ public ContainerPath containerPathUnderVespaHome(String relativePath) {
+ return nodeAgentCtx.paths().underVespaHome(relativePath);
+ }
+
+ @Override public Options options() { return this; }
+
+ @Override
+ public OptionalDouble duration() {
+ Double duration = dumpOptions()
+ .map(ServiceDumpReport.DumpOptions::duration)
+ .orElse(null);
+ return duration != null ? OptionalDouble.of(duration) : OptionalDouble.empty();
+ }
+
+ @Override
+ public boolean callGraphRecording() {
+ return dumpOptions().map(ServiceDumpReport.DumpOptions::callGraphRecording).orElse(false);
+ }
+
+ @Override
+ public boolean sendProfilingSignal() {
+ return dumpOptions().map(ServiceDumpReport.DumpOptions::sendProfilingSignal).orElse(false);
+ }
+
+ Optional<ServiceDumpReport.DumpOptions> dumpOptions() { return Optional.ofNullable(request.dumpOptions()); }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java
new file mode 100644
index 00000000000..c8f930464e0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java
@@ -0,0 +1,27 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+
+/**
+ * Performs dump of ZooKeeper snapshots. Can be used for controllers, config servers, cluster controllers and tenant containers
+ * where zookeeper is configured.
+ *
+ * @author hmusum
+ */
+class ZooKeeperSnapshotDumper implements ArtifactProducer {
+ @Override public String artifactName() { return "zookeeper-snapshot"; }
+ @Override public String description() { return "Dumps ZooKeeper snapshots"; }
+
+ @Override
+ public List<Artifact> produceArtifacts(Context ctx) {
+ ContainerPath zookeeperSnapshot = ctx.outputContainerPath().resolve("zookeeper-snapshot.tgz");
+ List<String> cmd = List.of("bash", "-c", String.format("/opt/vespa/bin/vespa-backup-zk-data.sh -o %s -k -f", zookeeperSnapshot.pathInContainer()));
+ ctx.executeCommandInNode(cmd, true);
+ return List.of(Artifact.newBuilder().classification(CONFIDENTIAL).file(zookeeperSnapshot).build());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java
new file mode 100644
index 00000000000..3ea43b6129a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java
new file mode 100644
index 00000000000..b1e467ad446
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java
@@ -0,0 +1,22 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.util.List;
+
+/**
+ * @author freva
+ */
+public interface SyncClient {
+
+ /**
+ * Syncs the given files, will only upload each file once.
+ *
+ * @param context context used to log which files were synced
+ * @param syncFileInfos list of files and their metadata to sync
+ * @param limit max number of files to upload for this invocation, to avoid blocking for too long
+ * @return true iff any files were uploaded
+ */
+ boolean sync(TaskContext context, List<SyncFileInfo> syncFileInfos, int limit);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
new file mode 100644
index 00000000000..c65f2abb6fd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
@@ -0,0 +1,143 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+
+/**
+ * @author freva
+ */
+public class SyncFileInfo {
+
+ private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter
+ .ofPattern("yyyy-MM-dd.HH-mm-ss").withZone(ZoneOffset.UTC);
+
+ private final Path source;
+ private final Function<String, URI> destination;
+ private final Compression uploadCompression;
+ private final Map<String, String> tags;
+ private final Optional<Duration> minDurationBetweenSync;
+
+ private SyncFileInfo(Path source, Function<String, URI> destination, Compression uploadCompression,
+ Map<String, String> tags, Duration minDurationBetweenSyncOrNull) {
+ this.source = source;
+ this.destination = destination;
+ this.uploadCompression = uploadCompression;
+ this.tags = Map.copyOf(tags);
+ this.minDurationBetweenSync = Optional.ofNullable(minDurationBetweenSyncOrNull);
+ }
+
+ /** Source path of the file to sync */
+ public Path source() {
+ return source;
+ }
+
+ /** Remote URI to store the file at */
+ public URI destination() {
+ return destination.apply("");
+ }
+
+ /** Returns a destination URI after adding a suffix to the base name of the filename. */
+ public URI destinationWithBasenameSuffix(String suffix) {
+ return destination.apply(suffix);
+ }
+
+ /** Compression algorithm to use when uploading the file */
+ public Compression uploadCompression() {
+ return uploadCompression;
+ }
+
+ public Map<String, String> tags() { return tags; }
+
+ public Optional<Duration> minDurationBetweenSync() { return minDurationBetweenSync; }
+
+ public static Optional<SyncFileInfo> forLogFile(URI uri, Path logFile, boolean rotatedOnly, ApplicationId owner) {
+ String filename = logFile.getFileName().toString();
+ Compression compression;
+ final String dir;
+ String remoteFilename = logFile.getFileName().toString();
+ Duration minDurationBetweenSync = null;
+
+ if (filename.startsWith("vespa.log")) {
+ dir = "logs/vespa/";
+ compression = Compression.ZSTD;
+ if (filename.length() == 9) {
+ if (!rotatedOnly) remoteFilename = "vespa.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime());
+ minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO;
+ }
+ } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) {
+ compression = Compression.ZSTD;
+ dir = "logs/zookeeper/";
+ remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" :
+ "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime());
+ minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
+ } else if (filename.startsWith("start-services.out-")) {
+ compression = Compression.ZSTD;
+ dir = "logs/start-services/";
+ } else if (filename.startsWith("nginx-error")) {
+ compression = Compression.ZSTD;
+ if ("nginx-error.log".equals(filename)) {
+ if (!rotatedOnly) remoteFilename = "nginx-error.log";
+ minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO;
+ }
+ dir = "logs/nginx/";
+ } else {
+ compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD;
+ if (rotatedOnly && compression != Compression.NONE)
+ dir = null;
+ else if (filename.contains(".metrics-proxy.")) // See AccessLogComponent.java for filename.
+ dir = null;
+ else if (filename.startsWith("JsonAccessLog.") || filename.startsWith("access"))
+ dir = "logs/access/";
+ else if (filename.startsWith("ConnectionLog."))
+ dir = "logs/connection/";
+ else
+ dir = null;
+ }
+
+ if (dir == null) return Optional.empty();
+ String finalRemoteFilename = remoteFilename;
+ Function<String, URI> destination = suffix -> uri.resolve(dir + finalRemoteFilename + suffix + compression.extension);
+ return Optional.of(new SyncFileInfo(logFile, destination, compression, defaultTags(owner), minDurationBetweenSync));
+ }
+
+ public static SyncFileInfo forServiceDump(URI destinationDir, Path file, Compression compression,
+ ApplicationId owner, String assetClassification) {
+ String filename = file.getFileName().toString();
+ Function<String, URI> location = suffix -> destinationDir.resolve(filename + suffix + compression.extension);
+ Map<String, String> tags = defaultTags(owner);
+ if (assetClassification != null) {
+ tags.put("vespa:AssetClassification", assetClassification);
+ }
+ return new SyncFileInfo(file, location, compression, tags, null);
+ }
+
+ private static Map<String, String> defaultTags(ApplicationId owner) {
+ var tags = new HashMap<String, String>();
+ tags.put("corp:Application", owner.toFullString());
+ return tags;
+ }
+
+ public boolean overwriteIfExists() {
+ return minDurationBetweenSync.isPresent();
+ }
+
+ public enum Compression {
+ NONE(""), ZSTD(".zst");
+
+ private final String extension;
+ Compression(String extension) {
+ this.extension = extension;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java
new file mode 100644
index 00000000000..eeec7e1b59c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java
@@ -0,0 +1,83 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.compress.ZstdCompressor;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * InputStream that outputs given InputStream compressed with the ZStandard.
+ *
+ * @author freva
+ */
+public class ZstdCompressingInputStream extends InputStream {
+
+ public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024;
+ private final ZstdCompressor compressor = new ZstdCompressor();
+
+ private final InputStream is;
+ private final byte[] inputBuffer;
+ private final byte[] outputBuffer;
+
+ private boolean firstRead = true;
+ private boolean eof = false;
+ private int outputPosition = 0;
+ private int outputLength = 0;
+ private boolean isClosed = false;
+
+ public ZstdCompressingInputStream(InputStream is, int inputBufferSize) {
+ this.is = is;
+ this.inputBuffer = new byte[inputBufferSize];
+ this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)];
+ }
+
+ public ZstdCompressingInputStream(InputStream is) {
+ this(is, DEFAULT_INPUT_BUFFER_SIZE);
+ }
+
+ @Override
+ public int read() throws IOException {
+ throwIfClosed();
+
+ if (outputPosition >= outputLength) {
+ int readLength = eof ? -1 : is.read(inputBuffer);
+ if (readLength == -1) {
+ if (!firstRead)
+ return -1;
+ // zstd compressing an empty file results in a 13 bytes file.
+ eof = true;
+ readLength = 0;
+ }
+ firstRead = false;
+
+ outputLength = compressor.compress(inputBuffer, 0, readLength, outputBuffer, 0, outputBuffer.length);
+ outputPosition = 0;
+ }
+
+ return Byte.toUnsignedInt(outputBuffer[outputPosition++]);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int first = read();
+ if (first == -1) return -1;
+
+ b[off++] = (byte) first;
+ len = Math.min(Math.min(len, outputLength - outputPosition), b.length - off);
+ System.arraycopy(outputBuffer, outputPosition, b, off, len);
+ outputPosition += len;
+ return len + 1;
+ }
+
+ @Override
+ public void close() throws IOException {
+ throwIfClosed();
+ is.close();
+ isClosed = true;
+ }
+
+ private void throwIfClosed() {
+ if (isClosed) throw new IllegalArgumentException("Input stream is already closed");
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java
new file mode 100644
index 00000000000..becf11945e3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author freva
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java
new file mode 100644
index 00000000000..686c32fd5ee
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java
@@ -0,0 +1,41 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+/**
+ * Exception specially handled to avoid dumping full stack trace on convergence failure.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class ConvergenceException extends RuntimeException {
+ /** Create an exception that will NOT increment the monitored unhandled_exceptions metric. */
+ public static ConvergenceException ofTransient(String message) { return ofTransient(message, null); }
+
+ /** Create an exception that will NOT increment the monitored unhandled_exceptions metric. */
+ public static ConvergenceException ofTransient(String message, Throwable t) { return new ConvergenceException(message, t, false); }
+
+ /** Create an exception that increments the monitored unhandled_exceptions metric. */
+ public static ConvergenceException ofError(String message) { return ofError(message, null); }
+
+ /** Create an exception that increments the monitored unhandled_exceptions metric. */
+ public static ConvergenceException ofError(String message, Throwable t) { return new ConvergenceException(message, t, true); }
+
+ /** Create an exception with the same transient/error as the cause. */
+ public static ConvergenceException ofNested(String message, ConvergenceException cause) { return new ConvergenceException(message, cause, cause.isError); }
+
+ private final boolean isError;
+
+ /** @param isError whether the exception should increment the monitored unhandled_exception metric. */
+ protected ConvergenceException(String message, boolean isError) {
+ this(message, null, isError);
+ }
+
+ /** @param isError whether the exception should increment the monitored unhandled_exception metric. */
+ protected ConvergenceException(String message, Throwable t, boolean isError) {
+ super(message, t);
+ this.isError = isError;
+ }
+
+ /** Whether the exception signals an error someone may want to look at, or whether it is expected to be transient (false). */
+ public boolean isError() { return isError; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java
new file mode 100644
index 00000000000..986f6b4eebc
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java
@@ -0,0 +1,56 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+
+import java.time.Duration;
+import java.util.Set;
+
+/**
+ * NodeAdmin manages the life cycle of NodeAgents.
+ * @author Haakon Dybdahl
+ */
+public interface NodeAdmin {
+
+ /** Start/stop NodeAgents and schedule next NodeAgent ticks with the given NodeAgentContexts */
+ void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts);
+
+ /** Update node admin metrics */
+ void updateMetrics(boolean isSuspended);
+
+ /**
+ * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that
+ * they will not pick up any changes from NodeRepository.
+ *
+ * @param frozen whether NodeAgents and NodeAdmin should be frozen
+ * @return True if all the NodeAgents and NodeAdmin has converged to the desired state
+ */
+ boolean setFrozen(boolean frozen);
+
+ /**
+ * Returns whether NodeAdmin itself is currently frozen, meaning it will not pick up any changes
+ * from NodeRepository.
+ */
+ boolean isFrozen();
+
+ /**
+ * Returns an upper bound on the time some or all parts of the node admin (including agents)
+ * have been frozen. Returns 0 if not frozen, nor trying to freeze.
+ */
+ Duration subsystemFreezeDuration();
+
+ /**
+ * Stop all services on these nodes
+ */
+ void stopNodeAgentServices();
+
+ /**
+ * Start node-admin schedulers.
+ */
+ void start();
+
+ /**
+ * Stop the NodeAgents. Will not delete the storage or stop the container.
+ */
+ void stop();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
new file mode 100644
index 00000000000..446f21d53e7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -0,0 +1,261 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import ai.vespa.metrics.ContainerMetrics;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerStats;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Counter;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Gauge;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextManager;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentScheduler;
+
+import java.nio.file.FileSystem;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Administers a host (for now only docker hosts) and its nodes (docker containers nodes).
+ *
+ * @author stiankri
+ */
+public class NodeAdminImpl implements NodeAdmin {
+ private static final Duration NODE_AGENT_FREEZE_TIMEOUT = Duration.ofSeconds(5);
+ private static final Duration NODE_AGENT_SPREAD = Duration.ofSeconds(3);
+
+ private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory;
+
+ private final Timer timer;
+ private final Duration freezeTimeout;
+ private final Duration spread;
+ private boolean previousWantFrozen;
+ private boolean isFrozen;
+ private Instant startOfFreezeConvergence;
+ private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>();
+
+ private final ProcMeminfoReader procMeminfoReader;
+ private final Gauge jvmHeapUsed;
+ private final Gauge jvmHeapFree;
+ private final Gauge jvmHeapTotal;
+ private final Gauge containerCount;
+ private final Counter numberOfUnhandledExceptions;
+ private final Metrics metrics;
+ private Dimensions previousMemoryOverheadDimensions = null;
+
+ public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Timer timer, FileSystem fileSystem) {
+ this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext),
+ metrics, timer, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD, new ProcMeminfoReader(fileSystem));
+ }
+
+ public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics,
+ Timer timer, Duration freezeTimeout, Duration spread, ProcMeminfoReader procMeminfoReader) {
+ this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext),
+ metrics, timer, freezeTimeout, spread, procMeminfoReader);
+ }
+
+ NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory,
+ Metrics metrics, Timer timer, Duration freezeTimeout, Duration spread,
+ ProcMeminfoReader procMeminfoReader) {
+ this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory;
+ this.timer = timer;
+ this.freezeTimeout = freezeTimeout;
+ this.spread = spread;
+ this.previousWantFrozen = true;
+ this.isFrozen = true;
+ this.startOfFreezeConvergence = timer.currentTime();
+
+ this.numberOfUnhandledExceptions = metrics.declareCounter("unhandled_exceptions",
+ new Dimensions(Map.of("src", "node-agents")));
+
+ this.procMeminfoReader = procMeminfoReader;
+ this.jvmHeapUsed = metrics.declareGauge(ContainerMetrics.MEM_HEAP_USED.baseName());
+ this.jvmHeapFree = metrics.declareGauge(ContainerMetrics.MEM_HEAP_FREE.baseName());
+ this.jvmHeapTotal = metrics.declareGauge(ContainerMetrics.MEM_HEAP_TOTAL.baseName());
+ this.containerCount = metrics.declareGauge("container.count");
+ this.metrics = metrics;
+ }
+
+ @Override
+ public void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts) {
+ Map<String, NodeAgentContext> nodeAgentContextsByHostname = nodeAgentContexts.stream()
+ .collect(Collectors.toMap(ctx -> ctx.node().id(), Function.identity()));
+
+ // Stop and remove NodeAgents that should no longer be running
+ diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet())
+ .forEach(hostname -> nodeAgentWithSchedulerByHostname.remove(hostname).stopForRemoval());
+
+ // Start NodeAgent for hostnames that should be running, but aren't yet
+ diff(nodeAgentContextsByHostname.keySet(), nodeAgentWithSchedulerByHostname.keySet()).forEach(hostname -> {
+ NodeAgentWithScheduler naws = nodeAgentWithSchedulerFactory.create(nodeAgentContextsByHostname.get(hostname));
+ naws.start();
+ nodeAgentWithSchedulerByHostname.put(hostname, naws);
+ });
+
+ Duration timeBetweenNodeAgents = spread.dividedBy(Math.max(nodeAgentContextsByHostname.size() - 1, 1));
+ Instant nextAgentStart = timer.currentTime();
+ // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys
+ for (Map.Entry<String, NodeAgentContext> entry : nodeAgentContextsByHostname.entrySet()) {
+ nodeAgentWithSchedulerByHostname.get(entry.getKey()).scheduleTickWith(entry.getValue(), nextAgentStart);
+ nextAgentStart = nextAgentStart.plus(timeBetweenNodeAgents);
+ }
+ }
+
+ @Override
+ public void updateMetrics(boolean isSuspended) {
+ int numContainers = 0;
+ long totalContainerMemoryBytes = 0;
+
+ for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) {
+ int count = nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions();
+ if (!isSuspended) numberOfUnhandledExceptions.add(count);
+ Optional<ContainerStats> containerStats = nodeAgentWithScheduler.updateContainerNodeMetrics(isSuspended);
+ if (containerStats.isPresent()) {
+ ++numContainers;
+ totalContainerMemoryBytes += containerStats.get().memoryStats().usage();
+ }
+ }
+
+ Runtime runtime = Runtime.getRuntime();
+ runtime.gc();
+ long freeMemory = runtime.freeMemory();
+ long totalMemory = runtime.totalMemory();
+ long usedMemory = totalMemory - freeMemory;
+ jvmHeapFree.sample(freeMemory);
+ jvmHeapUsed.sample(usedMemory);
+ jvmHeapTotal.sample(totalMemory);
+
+ // No container stats are found while suspended, so skip setting these if so.
+ if (!isSuspended) {
+ containerCount.sample(numContainers);
+ ProcMeminfo meminfo = procMeminfoReader.read();
+ updateMemoryOverheadMetric(numContainers, meminfo.memTotalBytes() - meminfo.memAvailableBytes() - totalContainerMemoryBytes);
+ }
+ }
+
+ private void updateMemoryOverheadMetric(int numContainers, double memoryOverhead) {
+ final String name = "mem.system.overhead";
+ Dimensions dimensions = new Dimensions(Map.of("containers", Integer.toString(numContainers)));
+ metrics.declareGauge(Metrics.APPLICATION_HOST, name, dimensions, Metrics.DimensionType.DEFAULT)
+ .sample(memoryOverhead);
+ if (previousMemoryOverheadDimensions != null && !previousMemoryOverheadDimensions.equals(dimensions))
+ metrics.deleteMetricByDimension(name, previousMemoryOverheadDimensions, Metrics.DimensionType.DEFAULT);
+ previousMemoryOverheadDimensions = dimensions;
+ }
+
+ @Override
+ public boolean setFrozen(boolean wantFrozen) {
+ if (wantFrozen != previousWantFrozen) {
+ if (wantFrozen) {
+ this.startOfFreezeConvergence = timer.currentTime();
+ } else {
+ this.startOfFreezeConvergence = null;
+ }
+
+ previousWantFrozen = wantFrozen;
+ }
+
+ // Use filter with count instead of allMatch() because allMatch() will short circuit on first non-match
+ boolean allNodeAgentsConverged = parallelStreamOfNodeAgentWithScheduler()
+ .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, freezeTimeout))
+ .count() == 0;
+
+ if (wantFrozen) {
+ if (allNodeAgentsConverged) isFrozen = true;
+ } else isFrozen = false;
+
+ return allNodeAgentsConverged;
+ }
+
+ @Override
+ public boolean isFrozen() {
+ return isFrozen;
+ }
+
+ @Override
+ public Duration subsystemFreezeDuration() {
+ if (startOfFreezeConvergence == null) {
+ return Duration.ZERO;
+ } else {
+ return Duration.between(startOfFreezeConvergence, timer.currentTime());
+ }
+ }
+
+ @Override
+ public void stopNodeAgentServices() {
+ // Each container may spend 1-1:30 minutes stopping
+ parallelStreamOfNodeAgentWithScheduler().forEach(NodeAgentWithScheduler::stopForHostSuspension);
+ }
+
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void stop() {
+ // Stop all node-agents in parallel, will block until the last NodeAgent is stopped
+ parallelStreamOfNodeAgentWithScheduler().forEach(NodeAgentWithScheduler::stopForRemoval);
+ }
+
+ /**
+ * Returns a parallel stream of NodeAgentWithScheduler.
+ *
+ * <p>Why not just call nodeAgentWithSchedulerByHostname.values().parallelStream()? Experiments
+ * with Java 11 have shown that with 10 nodes and forEach(), there are a maximum of 3 concurrent
+ * threads. With HashMap it produces 5. With List it produces 10 concurrent threads.</p>
+ */
+ private Stream<NodeAgentWithScheduler> parallelStreamOfNodeAgentWithScheduler() {
+ return List.copyOf(nodeAgentWithSchedulerByHostname.values()).parallelStream();
+ }
+
+ // Set-difference. Returns minuend minus subtrahend.
+ private static <T> Set<T> diff(Set<T> minuend, Set<T> subtrahend) {
+ var result = new HashSet<>(minuend);
+ result.removeAll(subtrahend);
+ return result;
+ }
+
+ static class NodeAgentWithScheduler implements NodeAgentScheduler {
+ private final NodeAgent nodeAgent;
+ private final NodeAgentScheduler nodeAgentScheduler;
+
+ private NodeAgentWithScheduler(NodeAgent nodeAgent, NodeAgentScheduler nodeAgentScheduler) {
+ this.nodeAgent = nodeAgent;
+ this.nodeAgentScheduler = nodeAgentScheduler;
+ }
+
+ void start() { nodeAgent.start(currentContext()); }
+ void stopForHostSuspension() { nodeAgent.stopForHostSuspension(currentContext()); }
+ void stopForRemoval() { nodeAgent.stopForRemoval(currentContext()); }
+ Optional<ContainerStats> updateContainerNodeMetrics(boolean isSuspended) { return nodeAgent.updateContainerNodeMetrics(currentContext(), isSuspended); }
+ int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); }
+
+ @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); }
+ @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); }
+ @Override public NodeAgentContext currentContext() { return nodeAgentScheduler.currentContext(); }
+ }
+
+ @FunctionalInterface
+ interface NodeAgentWithSchedulerFactory {
+ NodeAgentWithScheduler create(NodeAgentContext context);
+ }
+
+ private static NodeAgentWithScheduler create(Timer timer, NodeAgentFactory nodeAgentFactory, NodeAgentContext context) {
+ NodeAgentContextManager contextManager = new NodeAgentContextManager(timer, context);
+ NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context);
+ return new NodeAgentWithScheduler(nodeAgent, contextManager);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
new file mode 100644
index 00000000000..dc10eaee46c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
@@ -0,0 +1,180 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.TRANSITIONING;
+
+/**
+ * Pulls information from node repository and forwards containers to run to node admin.
+ *
+ * @author dybis, stiankri
+ */
+public class NodeAdminStateUpdater {
+ private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName());
+ private static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5);
+
+ private final ScheduledExecutorService metricsScheduler =
+ Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler"));
+
+ private final NodeAgentContextFactory nodeAgentContextFactory;
+ private final NodeRepository nodeRepository;
+ private final Orchestrator orchestrator;
+ private final NodeAdmin nodeAdmin;
+ private final String hostHostname;
+
+ public enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED }
+
+ private volatile State currentState = SUSPENDED_NODE_ADMIN;
+
+ public NodeAdminStateUpdater(
+ NodeAgentContextFactory nodeAgentContextFactory,
+ NodeRepository nodeRepository,
+ Orchestrator orchestrator,
+ NodeAdmin nodeAdmin,
+ HostName hostHostname) {
+ this.nodeAgentContextFactory = nodeAgentContextFactory;
+ this.nodeRepository = nodeRepository;
+ this.orchestrator = orchestrator;
+ this.nodeAdmin = nodeAdmin;
+ this.hostHostname = hostHostname.value();
+ }
+
+ public void start() {
+ nodeAdmin.start();
+
+ EnumSet<State> suspendedStates = EnumSet.of(SUSPENDED_NODE_ADMIN, SUSPENDED);
+ metricsScheduler.scheduleAtFixedRate(() -> {
+ try {
+ nodeAdmin.updateMetrics(suspendedStates.contains(currentState));
+ } catch (Throwable e) {
+ log.log(Level.WARNING, "Metric fetcher scheduler failed", e);
+ }
+ }, 10, 55, TimeUnit.SECONDS);
+ }
+
+ public void stop() {
+ metricsScheduler.shutdown();
+
+ // Stop all node-agents in parallel, will block until the last NodeAgent is stopped
+ nodeAdmin.stop();
+
+ do {
+ try {
+ metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ log.info("Was interrupted while waiting for metricsScheduler and shutdown");
+ }
+ } while (!metricsScheduler.isTerminated());
+ }
+
+ /**
+ * This method attempts to converge node-admin w/agents to a {@link State}
+ * with respect to: freeze, Orchestrator, and services running.
+ */
+ public void converge(State wantedState) {
+ NodeSpec node = nodeRepository.getNode(hostHostname);
+ boolean hostIsActiveInNR = node.state() == NodeState.active;
+ if (wantedState == RESUMED) {
+ adjustNodeAgentsToRunFromNodeRepository();
+ } else if (currentState == TRANSITIONING && nodeAdmin.subsystemFreezeDuration().compareTo(FREEZE_CONVERGENCE_TIMEOUT) > 0) {
+ // We have spent too much time trying to freeze and node admin is still not frozen.
+ // To avoid node agents stalling for too long, we'll force unfrozen ticks now.
+ adjustNodeAgentsToRunFromNodeRepository();
+ nodeAdmin.setFrozen(false);
+
+ if (hostIsActiveInNR) orchestrator.resume(hostHostname);
+
+ throw ConvergenceException.ofTransient("Timed out trying to freeze all nodes: will force an unfrozen tick");
+ }
+
+ boolean wantFrozen = wantedState != RESUMED;
+ if (currentState == wantedState && wantFrozen == node.orchestratorStatus().isSuspended()) return;
+ currentState = TRANSITIONING;
+
+ if (!nodeAdmin.setFrozen(wantFrozen))
+ throw ConvergenceException.ofTransient("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen"));
+
+ switch (wantedState) {
+ case RESUMED:
+ if (hostIsActiveInNR) orchestrator.resume(hostHostname);
+ break;
+ case SUSPENDED_NODE_ADMIN:
+ if (hostIsActiveInNR) orchestrator.suspend(hostHostname);
+ break;
+ case SUSPENDED:
+ // Fetch active nodes from node repo before suspending nodes.
+ // It is only possible to suspend active nodes,
+ // the orchestrator will fail if trying to suspend nodes in other states.
+ // Even though state is frozen we need to interact with node repo, but
+ // the data from node repo should not be used for anything else.
+ // We should also suspend host's hostname to suspend node-admin
+ List<String> nodesInActiveState = getNodesInActiveState();
+
+ List<String> nodesToSuspend = new ArrayList<>(nodesInActiveState);
+ if (hostIsActiveInNR) nodesToSuspend.add(hostHostname);
+ if (!nodesToSuspend.isEmpty()) {
+ orchestrator.suspend(hostHostname, nodesToSuspend);
+ log.info("Orchestrator allows suspension of " + nodesToSuspend);
+ }
+
+ // The node agent services are stopped by this thread, which is OK only
+ // because the node agents are frozen (see above).
+ nodeAdmin.stopNodeAgentServices();
+ break;
+ default:
+ throw new IllegalStateException("Unknown wanted state " + wantedState);
+ }
+
+ log.info("State changed from " + currentState + " to " + wantedState);
+ currentState = wantedState;
+ }
+
+ void adjustNodeAgentsToRunFromNodeRepository() {
+ try {
+ Map<String, Acl> aclByHostname = nodeRepository.getAcls(hostHostname);
+
+ Set<NodeAgentContext> nodeAgentContexts = nodeRepository.getNodes(hostHostname).stream()
+ .map(node -> nodeAgentContextFactory.create(node, aclByHostname.getOrDefault(node.hostname(), Acl.EMPTY)))
+ .collect(Collectors.toSet());
+ nodeAdmin.refreshContainersToRun(nodeAgentContexts);
+ } catch (ConvergenceException e) {
+ log.log(Level.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e));
+ } catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed to update which containers should be running", e);
+ }
+ }
+
+ private List<String> getNodesInActiveState() {
+ return nodeRepository.getNodes(hostHostname)
+ .stream()
+ .filter(node -> node.state() == NodeState.active)
+ .map(NodeSpec::hostname)
+ .toList();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java
new file mode 100644
index 00000000000..0c0d8dc348c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java
@@ -0,0 +1,12 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+/**
+ * Represents /proc/meminfo, see proc(5).
+ *
+ * @param memTotalBytes Total usable RAM (i.e., physical RAM minus a few reserved bits and the kernel binary code).
+ * @param memAvailableBytes An estimate of how much memory is available for starting new applications, without swapping.
+ *
+ * @author hakon
+ */
+public record ProcMeminfo(long memTotalBytes, long memAvailableBytes) { }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java
new file mode 100644
index 00000000000..d13aa1ea03c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java
@@ -0,0 +1,42 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.yolean.Exceptions;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Reads /proc/meminfo, see proc(5).
+ *
+ * @author hakon
+ */
+public class ProcMeminfoReader {
+ private static final String PROC_MEMINFO = "/proc/meminfo";
+ private static final Pattern MEM_TOTAL_PATTERN = Pattern.compile("MemTotal: *([0-9]+) kB");
+ private static final Pattern MEM_AVAILABLE_PATTERN = Pattern.compile("MemAvailable: *([0-9]+) kB");
+
+ private final FileSystem fileSystem;
+
+ public ProcMeminfoReader(FileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ public ProcMeminfo read() {
+ return read(Exceptions.uncheck(() -> Files.readString(fileSystem.getPath(PROC_MEMINFO))));
+ }
+
+ static ProcMeminfo read(String meminfoContent) {
+ return new ProcMeminfo(readKbGroup(meminfoContent, MEM_TOTAL_PATTERN),
+ readKbGroup(meminfoContent, MEM_AVAILABLE_PATTERN));
+ }
+
+ private static long readKbGroup(String string, Pattern pattern) {
+ Matcher matcher = pattern.matcher(string);
+ if (!matcher.find())
+ throw new IllegalArgumentException(pattern + " did not match anything in " + PROC_MEMINFO);
+ return Long.parseLong(matcher.group(1)) * 1024;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java
new file mode 100644
index 00000000000..68af4e59d45
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java
new file mode 100644
index 00000000000..3f7ff63c90b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java
@@ -0,0 +1,42 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.nio.file.Path;
+
+/**
+ * Utility for manipulating the initial file system the Docker container will start with.
+ *
+ * @author hakon
+ */
+public interface ContainerData {
+
+ /** Add or overwrite file in container at path. */
+ void addFile(ContainerPath path, String data);
+
+ /**
+ * @param path Container path to write
+ * @param data UTF-8 file content
+ * @param permissions file permissions, see {@link UnixPath#setPermissions(String)} for format.
+ */
+ void addFile(ContainerPath path, String data, String permissions);
+
+ /**
+ * @param path Container path to create directory at
+ * @param permissions optional file permissions, see {@link UnixPath#setPermissions(String)} for format.
+ */
+ void addDirectory(ContainerPath path, String... permissions);
+
+ /**
+ * Symlink to a file in container at path.
+ * @param symlink The path to the symlink inside the container
+ * @param target The path to the target file for the symbolic link inside the container
+ */
+ void addSymlink(ContainerPath symlink, Path target);
+
+ /** Writes all the files, directories and symlinks that were previously added */
+ void converge(NodeAgentContext context);
+}
+
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java
new file mode 100644
index 00000000000..78c907ad277
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java
@@ -0,0 +1,15 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+/**
+ * Interface for verifying the health of the node.
+ *
+ * @author hakonhall
+ */
+public interface HealthChecker extends AutoCloseable {
+ /** Verify the health of an active node, just before updating the node repo and calling Orchestrator resume. */
+ void verifyHealth(NodeAgentContext context);
+
+ @Override
+ void close();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
new file mode 100644
index 00000000000..b37b4dd665a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
@@ -0,0 +1,43 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.hosted.node.admin.container.ContainerStats;
+
+import java.util.Optional;
+
+/**
+ * Responsible for management of a single node over its lifecycle.
+ * May own its own resources, threads etc. Runs independently, but receives signals
+ * on state changes in the environment that may trigger this agent to take actions.
+ *
+ * @author bakksjo
+ */
+public interface NodeAgent {
+ /**
+ * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously
+ * striving to make the current state equal to the wanted state.
+ */
+ void start(NodeAgentContext context);
+
+ /**
+ * Stop the node in anticipation of host suspension, e.g. reboot or docker upgrade.
+ */
+ void stopForHostSuspension(NodeAgentContext context);
+
+ /**
+ * Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent.
+ * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this
+ * method returns, no more actions will be taken by the agent.
+ */
+ void stopForRemoval(NodeAgentContext context);
+
+ /**
+ * Updates metric receiver with the latest node-agent stats, and returns the container stats if available.
+ */
+ default Optional<ContainerStats> updateContainerNodeMetrics(NodeAgentContext context, boolean isSuspended) { return Optional.empty(); }
+
+ /**
+ * Returns and resets number of unhandled exceptions
+ */
+ int getAndResetNumberOfUnhandledExceptions();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
new file mode 100644
index 00000000000..9409ae2bee1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
@@ -0,0 +1,66 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode;
+
+import java.util.Optional;
+
+public interface NodeAgentContext extends TaskContext {
+
+ /** @return node specification from node-repository */
+ NodeSpec node();
+
+ /** @return node ACL from node-repository */
+ Acl acl();
+
+ /** @return name of the linux container this context applies to */
+ ContainerName containerName();
+
+ /** @return hostname of the linux container this context applies to */
+ default HostName hostname() {
+ return HostName.of(node().hostname());
+ }
+
+ default NodeType nodeType() {
+ return node().type();
+ }
+
+ AthenzIdentity identity();
+
+ ContainerNetworkMode networkMode();
+
+ ZoneApi zone();
+
+ /** @return information about users/user namespace of the linux container this context applies to */
+ UserScope users();
+
+ /** @return methods to resolve paths within container's file system */
+ PathScope paths();
+
+ default boolean isDisabled(NodeAgentTask task) {
+ return false;
+ }
+
+ /**
+ * The vcpu value in NodeSpec is the number of vcpus required by the node on a fixed historical
+ * baseline machine. However the current host has a faster per-vcpu performance by a scale factor
+ * (see flavors.def cpuSpeedup), and therefore do not need to set aside the full number of vcpus
+ * to run the node. This method returns that reduced number of vcpus.
+ *
+ * @return the vcpus required by the node on this host.
+ */
+ double vcpuOnThisHost();
+
+ Optional<ApplicationId> hostExclusiveTo();
+
+ boolean exclave();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java
new file mode 100644
index 00000000000..4e8db239867
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+
+/**
+ * @author freva
+ */
+@FunctionalInterface
+public interface NodeAgentContextFactory {
+ NodeAgentContext create(NodeSpec nodeSpec, Acl acl);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
new file mode 100644
index 00000000000..21d1cfd632c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
@@ -0,0 +1,283 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystem;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author freva
+ */
+public class NodeAgentContextImpl implements NodeAgentContext {
+
+ private final String logPrefix;
+ private final NodeSpec node;
+ private final Acl acl;
+ private final ContainerName containerName;
+ private final AthenzIdentity identity;
+ private final ContainerNetworkMode containerNetworkMode;
+ private final ZoneApi zone;
+ private final UserScope userScope;
+ private final PathScope pathScope;
+ private final double cpuSpeedup;
+ private final Set<NodeAgentTask> disabledNodeAgentTasks;
+ private final Optional<ApplicationId> hostExclusiveTo;
+ private final boolean exclave;
+
+ public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity,
+ ContainerNetworkMode containerNetworkMode, ZoneApi zone,
+ FlagSource flagSource, UserScope userScope, PathScope pathScope,
+ double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo, boolean exclave) {
+ if (cpuSpeedup <= 0)
+ throw new IllegalArgumentException("cpuSpeedUp must be positive, was: " + cpuSpeedup);
+
+ this.node = Objects.requireNonNull(node);
+ this.acl = Objects.requireNonNull(acl);
+ this.containerName = ContainerName.fromHostname(node.hostname());
+ this.identity = Objects.requireNonNull(identity);
+ this.containerNetworkMode = Objects.requireNonNull(containerNetworkMode);
+ this.zone = Objects.requireNonNull(zone);
+ this.userScope = Objects.requireNonNull(userScope);
+ this.pathScope = Objects.requireNonNull(pathScope);
+ this.logPrefix = containerName.asString() + ": ";
+ this.cpuSpeedup = cpuSpeedup;
+ this.disabledNodeAgentTasks = NodeAgentTask.fromString(
+ PermanentFlags.DISABLED_HOST_ADMIN_TASKS.bindTo(flagSource)
+ .with(FetchVector.Dimension.HOSTNAME, node.hostname())
+ .with(FetchVector.Dimension.NODE_TYPE, node.type().name()).value());
+ this.hostExclusiveTo = hostExclusiveTo;
+ this.exclave = exclave;
+ }
+
+ @Override
+ public NodeSpec node() {
+ return node;
+ }
+
+ @Override
+ public Acl acl() {
+ return acl;
+ }
+
+ @Override
+ public ContainerName containerName() {
+ return containerName;
+ }
+
+ @Override
+ public AthenzIdentity identity() {
+ return identity;
+ }
+
+ @Override
+ public ContainerNetworkMode networkMode() {
+ return containerNetworkMode;
+ }
+
+ @Override
+ public ZoneApi zone() {
+ return zone;
+ }
+
+ @Override
+ public UserScope users() {
+ return userScope;
+ }
+
+ @Override
+ public PathScope paths() {
+ return pathScope;
+ }
+
+ @Override
+ public boolean isDisabled(NodeAgentTask task) {
+ return disabledNodeAgentTasks.contains(task);
+ }
+
+ @Override
+ public double vcpuOnThisHost() {
+ return node.vcpu() / cpuSpeedup;
+ }
+
+ @Override
+ public Optional<ApplicationId> hostExclusiveTo() {
+ return hostExclusiveTo;
+ }
+
+ @Override
+ public void recordSystemModification(Logger logger, String message) {
+ log(logger, message);
+ }
+
+ @Override
+ public void log(Logger logger, Level level, String message) {
+ logger.log(level, logPrefix + message);
+ }
+
+ @Override
+ public void log(Logger logger, Level level, String message, Throwable throwable) {
+ logger.log(level, logPrefix + message, throwable);
+ }
+
+ @Override
+ public boolean exclave() {
+ return exclave;
+ }
+
+ public static NodeAgentContextImpl.Builder builder(NodeSpec node) {
+ return new Builder(new NodeSpec.Builder(node));
+ }
+
+ /**
+ * Creates a NodeAgentContext.Builder with a NodeSpec that has the given hostname and some
+ * reasonable values for the remaining required NodeSpec fields. Use {@link #builder(NodeSpec)}
+ * if you want to control the entire NodeSpec.
+ */
+ public static NodeAgentContextImpl.Builder builder(String hostname) {
+ return new Builder(NodeSpec.Builder.testSpec(hostname));
+ }
+
+ /** For testing only! */
+ public static class Builder {
+ private static final Path DEFAULT_CONTAINER_STORAGE = Path.of("/data/vespa/storage");
+
+ private NodeSpec.Builder nodeSpecBuilder;
+ private Acl acl;
+ private AthenzIdentity identity;
+ private ContainerNetworkMode containerNetworkMode;
+ private ZoneApi zone;
+ private UserNamespace userNamespace;
+ private Path containerStorage;
+ private FlagSource flagSource;
+ private double cpuSpeedUp = 1;
+ private Optional<ApplicationId> hostExclusiveTo = Optional.empty();
+ private boolean exclave = false;
+
+ private Builder(NodeSpec.Builder nodeSpecBuilder) {
+ this.nodeSpecBuilder = nodeSpecBuilder;
+ }
+
+ public Builder nodeSpecBuilder(Function<NodeSpec.Builder, NodeSpec.Builder> nodeSpecBuilderModifier) {
+ this.nodeSpecBuilder = nodeSpecBuilderModifier.apply(nodeSpecBuilder);
+ return this;
+ }
+
+ public Builder acl(Acl acl) {
+ this.acl = acl;
+ return this;
+ }
+
+ public Builder identity(AthenzIdentity identity) {
+ this.identity = identity;
+ return this;
+ }
+
+ public Builder networkMode(ContainerNetworkMode containerNetworkMode) {
+ this.containerNetworkMode = containerNetworkMode;
+ return this;
+ }
+
+ public Builder zone(ZoneApi zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public Builder userNamespace(UserNamespace userNamespace) {
+ this.userNamespace = userNamespace;
+ return this;
+ }
+
+ /** Sets the file system to use for paths. */
+ public Builder fileSystem(FileSystem fileSystem) {
+ return containerStorage(fileSystem.getPath(DEFAULT_CONTAINER_STORAGE.toString()));
+ }
+
+ public Builder flagSource(FlagSource flagSource) {
+ this.flagSource = flagSource;
+ return this;
+ }
+
+ public Builder cpuSpeedUp(double cpuSpeedUp) {
+ this.cpuSpeedUp = cpuSpeedUp;
+ return this;
+ }
+
+ public Builder containerStorage(Path path) {
+ this.containerStorage = path;
+ return this;
+ }
+
+ public Builder hostExclusiveTo(ApplicationId applicationId) {
+ this.hostExclusiveTo = Optional.ofNullable(applicationId);
+ return this;
+ }
+
+ public Builder exclave(boolean exclave) {
+ this.exclave = exclave;
+ return this;
+ }
+
+ public NodeAgentContextImpl build() {
+ Objects.requireNonNull(containerStorage, "Must set one of containerStorage or fileSystem");
+
+ UserScope userScope = UserScope.create(
+ Optional.ofNullable(userNamespace).orElseGet(() -> new UserNamespace(100000, 100000, 100000)));
+ ContainerFileSystem containerFs = ContainerFileSystem.create(containerStorage
+ .resolve(nodeSpecBuilder.hostname().split("\\.")[0]), userScope);
+ containerFs.createRoot();
+
+ return new NodeAgentContextImpl(
+ nodeSpecBuilder.build(),
+ Optional.ofNullable(acl).orElse(Acl.EMPTY),
+ Optional.ofNullable(identity).orElseGet(() -> new AthenzService("domain", "service")),
+ Optional.ofNullable(containerNetworkMode).orElse(ContainerNetworkMode.HOST_NETWORK),
+ Optional.ofNullable(zone).orElseGet(() -> new ZoneApi() {
+ @Override
+ public SystemName getSystemName() {
+ return SystemName.defaultSystem();
+ }
+
+ @Override
+ public ZoneId getId() {
+ return ZoneId.defaultId();
+ }
+
+ @Override
+ public CloudName getCloudName() {
+ return CloudName.DEFAULT;
+ }
+
+ @Override
+ public String getCloudNativeRegionName() {
+ return getId().region().value();
+ }
+ }),
+ Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new),
+ userScope,
+ new PathScope(containerFs, "/opt/vespa"),
+ cpuSpeedUp, hostExclusiveTo, exclave);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java
new file mode 100644
index 00000000000..ee3c86b838f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java
@@ -0,0 +1,124 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.jdisc.Timer;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * This class should be used by exactly 2 threads, 1 for each interface it implements.
+ *
+ * @author freva
+ */
+public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAgentScheduler {
+
+ private final Object monitor = new Object();
+ private final Timer timer;
+
+ private NodeAgentContext currentContext;
+ private NodeAgentContext nextContext;
+ private Instant nextContextAt;
+ private boolean wantFrozen = false;
+ private boolean isFrozen = true;
+ private boolean interrupted = false;
+ private boolean isWaitingForNextContext = false;
+
+ public NodeAgentContextManager(Timer timer, NodeAgentContext context) {
+ this.timer = timer;
+ this.currentContext = context;
+ }
+
+ @Override
+ public void scheduleTickWith(NodeAgentContext context, Instant at) {
+ synchronized (monitor) {
+ nextContext = Objects.requireNonNull(context);
+ nextContextAt = Objects.requireNonNull(at);
+ monitor.notifyAll(); // Notify of new context
+ }
+ }
+
+ @Override
+ public boolean setFrozen(boolean frozen, Duration timeout) {
+ synchronized (monitor) {
+ if (wantFrozen != frozen) {
+ wantFrozen = frozen;
+ monitor.notifyAll(); // Notify the supplier of the wantFrozen change
+ }
+
+ boolean successful;
+ long remainder;
+ long end = timer.currentTime().plus(timeout).toEpochMilli();
+ while (!(successful = isFrozen == frozen) && (remainder = end - timer.currentTimeMillis()) > 0) {
+ try {
+ monitor.wait(remainder); // Wait with timeout until the supplier is has reached wanted frozen state
+ } catch (InterruptedException ignored) { }
+ }
+
+ return successful;
+ }
+ }
+
+ @Override
+ public NodeAgentContext nextContext() throws ContextSupplierInterruptedException {
+ synchronized (monitor) {
+ nextContext = null; // Reset any previous context and wait for the next one
+ isWaitingForNextContext = true;
+ monitor.notifyAll();
+ Duration untilNextContext = Duration.ZERO;
+ while (true) {
+ if (interrupted) throw new ContextSupplierInterruptedException();
+
+ if (!setAndGetIsFrozen(wantFrozen) &&
+ nextContext != null &&
+ (untilNextContext = Duration.between(Instant.now(), nextContextAt)).toMillis() <= 0)
+ break;
+
+ try {
+ monitor.wait(Math.max(untilNextContext.toMillis(), 0L)); // Wait until scheduler provides a new context
+ } catch (InterruptedException ignored) { }
+ }
+
+ isWaitingForNextContext = false;
+ currentContext = nextContext;
+ return currentContext;
+ }
+ }
+
+ @Override
+ public NodeAgentContext currentContext() {
+ synchronized (monitor) {
+ return currentContext;
+ }
+ }
+
+ @Override
+ public void interrupt() {
+ synchronized (monitor) {
+ interrupted = true;
+ monitor.notifyAll();
+ }
+ }
+
+ private boolean setAndGetIsFrozen(boolean isFrozen) {
+ synchronized (monitor) {
+ if (this.isFrozen != isFrozen) {
+ this.isFrozen = isFrozen;
+ monitor.notifyAll(); // Notify the scheduler of the isFrozen change
+ }
+ return this.isFrozen;
+ }
+ }
+
+ /** FOR TESTING ONLY */
+ void waitUntilWaitingForNextContext() {
+ synchronized (monitor) {
+ while (!isWaitingForNextContext) {
+ try {
+ monitor.wait();
+ } catch (InterruptedException ignored) { }
+ }
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java
new file mode 100644
index 00000000000..a4450626766
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java
@@ -0,0 +1,20 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+/**
+ * @author freva
+ */
+public interface NodeAgentContextSupplier {
+
+ /**
+ * Blocks until the next context is ready
+ * @return context
+ * @throws ContextSupplierInterruptedException if {@link #interrupt()} was called before this method returned
+ */
+ NodeAgentContext nextContext() throws ContextSupplierInterruptedException;
+
+ /** Interrupts the thread(s) currently waiting in {@link #nextContext()} */
+ void interrupt();
+
+ class ContextSupplierInterruptedException extends RuntimeException { }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java
new file mode 100644
index 00000000000..ef67ff88471
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java
@@ -0,0 +1,10 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+/**
+ * @author freva
+ */
+@FunctionalInterface
+public interface NodeAgentFactory {
+ NodeAgent create(NodeAgentContextSupplier contextSupplier, NodeAgentContext nodeAgentContext);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
new file mode 100644
index 00000000000..43dc3d72c46
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -0,0 +1,633 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.flags.DoubleFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.container.Container;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerResources;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider;
+import com.yahoo.vespa.hosted.node.admin.maintenance.ContainerWireguardTask;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID;
+import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID;
+import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE;
+import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME;
+import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException;
+import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT;
+import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.STARTING;
+import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.UNKNOWN;
+
+/**
+ * @author dybis
+ * @author bakksjo
+ */
+public class NodeAgentImpl implements NodeAgent {
+
+ // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration
+ // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left
+ private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1));
+
+ private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
+
+ private final NodeAgentContextSupplier contextSupplier;
+ private final NodeRepository nodeRepository;
+ private final Orchestrator orchestrator;
+ private final ContainerOperations containerOperations;
+ private final RegistryCredentialsProvider registryCredentialsProvider;
+ private final StorageMaintainer storageMaintainer;
+ private final List<CredentialsMaintainer> credentialsMaintainers;
+ private final Optional<AclMaintainer> aclMaintainer;
+ private final Optional<HealthChecker> healthChecker;
+ private final Timer timer;
+ private final Duration warmUpDuration;
+ private final DoubleFlag containerCpuCap;
+ private final VespaServiceDumper serviceDumper;
+ private final List<ContainerWireguardTask> wireguardTasks;
+
+ private Thread loopThread;
+ private ContainerState containerState = UNKNOWN;
+ private NodeSpec lastNode;
+
+ private final AtomicBoolean terminated = new AtomicBoolean(false);
+ private boolean hasResumedNode = false;
+ private boolean hasStartedServices = true;
+ private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty();
+ private boolean suspendedInOrchestrator = false;
+
+ private int numberOfUnhandledException = 0;
+ private long currentRebootGeneration = 0;
+ private Optional<Long> currentRestartGeneration = Optional.empty();
+
+ /**
+ * ABSENT means container is definitely absent - A container that was absent will not suddenly appear without
+ * NodeAgent explicitly starting it.
+ * STARTING state is set just before we attempt to start a container, if successful we move to the next state.
+ * Otherwise we can't be certain. A container that was running a minute ago may no longer be running without
+ * NodeAgent doing anything (container could have crashed). Therefore we always have to ask docker daemon
+ * to get updated state of the container.
+ */
+ enum ContainerState {
+ ABSENT,
+ STARTING,
+ UNKNOWN
+ }
+
+
+ public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+ Orchestrator orchestrator, ContainerOperations containerOperations,
+ RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer,
+ FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers,
+ Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer,
+ VespaServiceDumper serviceDumper, List<ContainerWireguardTask> wireguardTasks) {
+ this(contextSupplier, nodeRepository, orchestrator, containerOperations, registryCredentialsProvider,
+ storageMaintainer, flagSource, credentialsMaintainers, aclMaintainer, healthChecker, timer,
+ DEFAULT_WARM_UP_DURATION, serviceDumper, wireguardTasks);
+ }
+
+ public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+ Orchestrator orchestrator, ContainerOperations containerOperations,
+ RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer,
+ FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers,
+ Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer,
+ Duration warmUpDuration, VespaServiceDumper serviceDumper,
+ List<ContainerWireguardTask> wireguardTasks) {
+ this.contextSupplier = contextSupplier;
+ this.nodeRepository = nodeRepository;
+ this.orchestrator = orchestrator;
+ this.containerOperations = containerOperations;
+ this.registryCredentialsProvider = registryCredentialsProvider;
+ this.storageMaintainer = storageMaintainer;
+ this.credentialsMaintainers = credentialsMaintainers;
+ this.aclMaintainer = aclMaintainer;
+ this.healthChecker = healthChecker;
+ this.timer = timer;
+ this.warmUpDuration = warmUpDuration;
+ this.containerCpuCap = PermanentFlags.CONTAINER_CPU_CAP.bindTo(flagSource);
+ this.serviceDumper = serviceDumper;
+ this.wireguardTasks = new ArrayList<>(wireguardTasks);
+ }
+
+ @Override
+ public void start(NodeAgentContext initialContext) {
+ if (loopThread != null)
+ throw new IllegalStateException("Can not re-start a node agent.");
+
+ loopThread = new Thread(() -> {
+ while (!terminated.get()) {
+ try {
+ converge(contextSupplier.nextContext());
+ } catch (ContextSupplierInterruptedException ignored) { }
+ }
+ });
+ loopThread.setName("tick-" + initialContext.hostname());
+ loopThread.start();
+ }
+
+ @Override
+ public void stopForRemoval(NodeAgentContext context) {
+ if (!terminated.compareAndSet(false, true))
+ throw new IllegalStateException("Can not re-stop a node agent.");
+
+ contextSupplier.interrupt();
+
+ do {
+ try {
+ loopThread.join();
+ } catch (InterruptedException ignored) { }
+ } while (loopThread.isAlive());
+
+ context.log(logger, "Stopped");
+ }
+
+ void startServicesIfNeeded(NodeAgentContext context) {
+ if (!hasStartedServices) {
+ context.log(logger, "Invoking vespa-nodectl to start services");
+ String output = containerOperations.startServices(context);
+ if (!output.isBlank()) {
+ context.log(logger, "Start services output: " + output);
+ }
+ hasStartedServices = true;
+ }
+ }
+
+ void resumeNodeIfNeeded(NodeAgentContext context) {
+ if (!hasResumedNode) {
+ context.log(logger, "Invoking vespa-nodectl to resume services");
+ String output = containerOperations.resumeNode(context);
+ if (!output.isBlank()) {
+ context.log(logger, "Resume services output: " + output);
+ }
+ hasResumedNode = true;
+ }
+ }
+
+ private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context, Optional<Instant> containerCreatedAt) {
+ final NodeAttributes currentNodeAttributes = new NodeAttributes();
+ final NodeAttributes newNodeAttributes = new NodeAttributes();
+ boolean changed = false;
+
+ if (context.node().wantedRestartGeneration().isPresent() &&
+ !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) {
+ currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration());
+ newNodeAttributes.withRestartGeneration(currentRestartGeneration);
+ changed = true;
+ }
+
+ boolean createdAtAfterRebootedEvent = context.node().events().stream()
+ .filter(event -> event.type().equals("rebooted"))
+ .map(event -> containerCreatedAt
+ .map(createdAt -> createdAt.isAfter(event.at()))
+ .orElse(false)) // Container not created
+ .findFirst()
+ .orElse(containerCreatedAt.isPresent()); // No rebooted event
+ if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration) || createdAtAfterRebootedEvent) {
+ currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration());
+ newNodeAttributes.withRebootGeneration(currentRebootGeneration);
+ changed = true;
+ }
+
+ Optional<DockerImage> wantedDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN);
+ if (!Objects.equals(context.node().currentDockerImage(), wantedDockerImage)) {
+ DockerImage currentImage = context.node().currentDockerImage().orElse(DockerImage.EMPTY);
+ DockerImage newImage = wantedDockerImage.orElse(DockerImage.EMPTY);
+
+ currentNodeAttributes.withDockerImage(currentImage);
+ currentNodeAttributes.withVespaVersion(context.node().currentVespaVersion().orElse(Version.emptyVersion));
+ newNodeAttributes.withDockerImage(newImage);
+ newNodeAttributes.withVespaVersion(context.node().wantedVespaVersion().orElse(Version.emptyVersion));
+ changed = true;
+ }
+
+ Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class);
+ if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) {
+ newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(timer.currentTimeMillis()).toJsonNode());
+ changed = true;
+ }
+
+ if (changed) {
+ context.log(logger, "Publishing new set of attributes to node repo: %s -> %s",
+ currentNodeAttributes, newNodeAttributes);
+ nodeRepository.updateNodeAttributes(context.hostname().value(), newNodeAttributes);
+ }
+ }
+
+ private Container startContainer(NodeAgentContext context) {
+ ContainerResources wantedResources = warmUpDuration(context).isNegative() ?
+ getContainerResources(context) : getContainerResources(context).withUnlimitedCpus();
+ ContainerData containerData = containerOperations.createContainer(context, wantedResources);
+ writeContainerData(context, containerData);
+ containerOperations.startContainer(context);
+
+ currentRebootGeneration = context.node().wantedRebootGeneration();
+ currentRestartGeneration = context.node().wantedRestartGeneration();
+ hasStartedServices = true; // Automatically started with the container
+ hasResumedNode = false;
+ context.log(logger, "Container successfully started, new containerState is " + containerState);
+ return containerOperations.getContainer(context).orElseThrow(() ->
+ ConvergenceException.ofError("Did not find container that was just started"));
+ }
+
+ private Optional<Container> removeContainerIfNeededUpdateContainerState(
+ NodeAgentContext context, Optional<Container> existingContainer) {
+ if (existingContainer.isPresent()) {
+ List<String> reasons = shouldRemoveContainer(context, existingContainer.get());
+ if (!reasons.isEmpty()) {
+ removeContainer(context, existingContainer.get(), reasons, false);
+ return Optional.empty();
+ }
+
+ shouldRestartServices(context, existingContainer.get()).ifPresent(restartReason -> {
+ context.log(logger, "Invoking vespa-nodectl to restart services: " + restartReason);
+ orchestratorSuspendNode(context);
+
+ ContainerResources currentResources = existingContainer.get().resources();
+ ContainerResources wantedResources = currentResources.withUnlimitedCpus();
+ if ( ! warmUpDuration(context).isNegative() && ! wantedResources.equals(currentResources)) {
+ context.log(logger, "Updating container resources: %s -> %s",
+ existingContainer.get().resources().toStringCpu(), wantedResources.toStringCpu());
+ containerOperations.updateContainer(context, existingContainer.get().id(), wantedResources);
+ }
+
+ String output = containerOperations.restartVespa(context);
+ if ( ! output.isBlank()) {
+ context.log(logger, "Restart services output: " + output);
+ }
+ currentRestartGeneration = context.node().wantedRestartGeneration();
+ firstSuccessfulHealthCheckInstant = Optional.empty();
+ });
+ }
+
+ return existingContainer;
+ }
+
+ private Optional<String> shouldRestartServices(NodeAgentContext context, Container existingContainer) {
+ NodeSpec node = context.node();
+ if (!existingContainer.state().isRunning() || node.state() != NodeState.active) return Optional.empty();
+
+ // Restart generation is only optional because it does not exist for unallocated nodes
+ if (currentRestartGeneration.get() < node.wantedRestartGeneration().get()) {
+ return Optional.of("Restart requested - wanted restart generation has been bumped: "
+ + currentRestartGeneration.get() + " -> " + node.wantedRestartGeneration().get());
+ }
+
+ return Optional.empty();
+ }
+
+ private void stopServicesIfNeeded(NodeAgentContext context) {
+ if (hasStartedServices && context.node().owner().isEmpty())
+ stopServices(context);
+ }
+
+ private void stopServices(NodeAgentContext context) {
+ context.log(logger, "Stopping services");
+ if (containerState == ABSENT) return;
+ hasStartedServices = hasResumedNode = false;
+ firstSuccessfulHealthCheckInstant = Optional.empty();
+ containerOperations.stopServices(context);
+ }
+
+ @Override
+ public void stopForHostSuspension(NodeAgentContext context) {
+ getContainer(context).ifPresent(container -> removeContainer(context, container, List.of("Suspending host"), true));
+ }
+
+ public void suspend(NodeAgentContext context) {
+ if (containerState == ABSENT) return;
+ try {
+ hasResumedNode = false;
+ context.log(logger, "Invoking vespa-nodectl to suspend services");
+ String output = containerOperations.suspendNode(context);
+ if (!output.isBlank()) {
+ context.log(logger, "Suspend services output: " + output);
+ }
+ } catch (RuntimeException e) {
+ // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to
+ // remove container, we will not be able to upgrade to fix any problems in the suspend logic!
+ context.log(logger, Level.WARNING, "Failed trying to suspend container", e);
+ }
+ }
+
+ private List<String> shouldRemoveContainer(NodeAgentContext context, Container existingContainer) {
+ final NodeState nodeState = context.node().state();
+ List<String> reasons = new ArrayList<>();
+ if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned)
+ reasons.add("Node in state " + nodeState + ", container should no longer be running");
+
+ if (context.node().wantedDockerImage().isPresent() &&
+ !context.node().wantedDockerImage().get().equals(existingContainer.image())) {
+ reasons.add("The node is supposed to run a new Docker image: "
+ + existingContainer.image().asString() + " -> " + context.node().wantedDockerImage().get().asString());
+ }
+
+ if (!existingContainer.state().isRunning())
+ reasons.add("Container no longer running");
+
+ if (currentRebootGeneration < context.node().wantedRebootGeneration()) {
+ reasons.add(String.format("Container reboot wanted. Current: %d, Wanted: %d",
+ currentRebootGeneration, context.node().wantedRebootGeneration()));
+ }
+
+ ContainerResources wantedContainerResources = getContainerResources(context);
+ if (!wantedContainerResources.equalsMemory(existingContainer.resources())) {
+ reasons.add("Container should be running with different memory allocation, wanted: " +
+ wantedContainerResources.toStringMemory() + ", actual: " + existingContainer.resources().toStringMemory());
+ }
+
+ if (containerState == STARTING)
+ reasons.add("Container failed to start");
+
+ return reasons;
+ }
+
+ private void removeContainer(NodeAgentContext context, Container existingContainer, List<String> reasons, boolean alreadySuspended) {
+ context.log(logger, "Will remove container: " + String.join(", ", reasons));
+
+ if (existingContainer.state().isRunning()) {
+ if (!alreadySuspended) {
+ orchestratorSuspendNode(context);
+ }
+
+ try {
+ if (context.node().state() == NodeState.active) {
+ suspend(context);
+ }
+ stopServices(context);
+ } catch (Exception e) {
+ context.log(logger, Level.WARNING, "Failed stopping services, ignoring", e);
+ }
+ }
+
+ storageMaintainer.handleCoreDumpsForContainer(context, Optional.of(existingContainer), true);
+ containerOperations.removeContainer(context, existingContainer);
+ containerState = ABSENT;
+ context.log(logger, "Container successfully removed, new containerState is " + containerState);
+ }
+
+
+ private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) {
+ ContainerResources wantedContainerResources = getContainerResources(context);
+
+ if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant
+ .map(timer.currentTime().minus(warmUpDuration(context))::isBefore)
+ .orElse(true))
+ return existingContainer;
+
+ if (wantedContainerResources.equalsCpu(existingContainer.resources())) return existingContainer;
+ context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s",
+ wantedContainerResources.toStringCpu(), existingContainer.resources().toStringCpu());
+
+ // Only update CPU resources
+ containerOperations.updateContainer(context, existingContainer.id(), wantedContainerResources.withMemoryBytes(existingContainer.resources().memoryBytes()));
+ return containerOperations.getContainer(context).orElseThrow(() ->
+ ConvergenceException.ofError("Did not find container that was just updated"));
+ }
+
+ private ContainerResources getContainerResources(NodeAgentContext context) {
+ double cpuCap = context.vcpuOnThisHost() * containerCpuCap
+ .with(INSTANCE_ID, context.node().owner().map(ApplicationId::serializedForm))
+ .with(CLUSTER_ID, context.node().membership().map(NodeMembership::clusterId))
+ .with(CLUSTER_TYPE, context.node().membership().map(membership -> membership.type().value()))
+ .with(HOSTNAME, context.node().hostname())
+ .value();
+
+ return ContainerResources.from(cpuCap, context.vcpuOnThisHost(), context.node().memoryGb());
+ }
+
+ private boolean downloadImageIfNeeded(NodeAgentContext context, Optional<Container> container) {
+ NodeSpec node = context.node();
+ if (node.wantedDockerImage().equals(container.map(c -> c.image()))) return false;
+
+ return node.wantedDockerImage()
+ .map(image -> containerOperations.pullImageAsyncIfNeeded(context, image, registryCredentialsProvider))
+ .orElse(false);
+ }
+
+ private void dropDocsIfNeeded(NodeAgentContext context, Optional<Container> container) {
+ Optional<DropDocumentsReport> report = context.node().reports()
+ .getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class);
+ if (report.isEmpty() || report.get().readiedAt() != null) return;
+
+ if (report.get().droppedAt() == null) {
+ container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true));
+ FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context);
+ nodeRepository.updateNodeAttributes(context.node().hostname(),
+ new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(timer.currentTimeMillis()).toJsonNode()));
+ }
+
+ throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container");
+ }
+
+ public void converge(NodeAgentContext context) {
+ try {
+ doConverge(context);
+ context.log(logger, Level.INFO, "Converged");
+ } catch (ConvergenceException e) {
+ context.log(logger, e.getMessage());
+ if (e.isError())
+ numberOfUnhandledException++;
+ } catch (Throwable e) {
+ numberOfUnhandledException++;
+ context.log(logger, Level.SEVERE, "Unhandled exception, ignoring", e);
+ }
+ }
+
+ // Non-private for testing
+ void doConverge(NodeAgentContext context) {
+ NodeSpec node = context.node();
+ Optional<Container> container = getContainer(context);
+
+ // Current reboot generation uninitialized or incremented from outside to cancel reboot
+ if (currentRebootGeneration < node.currentRebootGeneration())
+ currentRebootGeneration = node.currentRebootGeneration();
+
+ // Either we have changed allocation status (restart gen. only available to allocated nodes), or
+ // restart generation has been incremented from outside to cancel restart
+ if (currentRestartGeneration.isPresent() != node.currentRestartGeneration().isPresent() ||
+ currentRestartGeneration.map(current -> current < node.currentRestartGeneration().get()).orElse(false))
+ currentRestartGeneration = node.currentRestartGeneration();
+
+ if (!node.equals(lastNode)) {
+ logChangesToNodeSpec(context, lastNode, node);
+ lastNode = node;
+ }
+
+ // Run this here and now, even though we may immediately remove the container below.
+ // This ensures these maintainers are run even if something fails or returns early.
+ // These maintainers should also run immediately after starting the container (see below).
+ container.filter(c -> c.state().isRunning())
+ .ifPresent(c -> runImportantContainerMaintainers(context, c));
+
+ switch (node.state()) {
+ case ready, reserved, failed, inactive, parked -> {
+ storageMaintainer.syncLogs(context, true);
+ if (node.state() == NodeState.reserved) downloadImageIfNeeded(context, container);
+ removeContainerIfNeededUpdateContainerState(context, container);
+ updateNodeRepoWithCurrentAttributes(context, Optional.empty());
+ stopServicesIfNeeded(context);
+ }
+ case active -> {
+ storageMaintainer.syncLogs(context, true);
+ storageMaintainer.cleanDiskIfFull(context);
+ storageMaintainer.handleCoreDumpsForContainer(context, container, false);
+
+ if (downloadImageIfNeeded(context, container)) {
+ context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString());
+ return;
+ }
+ dropDocsIfNeeded(context, container);
+ container = removeContainerIfNeededUpdateContainerState(context, container);
+ credentialsMaintainers.forEach(maintainer -> maintainer.converge(context));
+ if (container.isEmpty()) {
+ containerState = STARTING;
+ container = Optional.of(startContainer(context));
+ containerState = UNKNOWN;
+ runImportantContainerMaintainers(context, container.get());
+ } else {
+ container = Optional.of(updateContainerIfNeeded(context, container.get()));
+ }
+
+ serviceDumper.processServiceDumpRequest(context);
+
+ startServicesIfNeeded(context);
+ resumeNodeIfNeeded(context);
+ if (healthChecker.isPresent()) {
+ healthChecker.get().verifyHealth(context);
+ if (firstSuccessfulHealthCheckInstant.isEmpty())
+ firstSuccessfulHealthCheckInstant = Optional.of(timer.currentTime());
+
+ Duration timeLeft = Duration.between(timer.currentTime(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context)));
+ if ( ! container.get().resources().equalsCpu(getContainerResources(context))
+ && context.node().currentDockerImage().isPresent()) // Immediately resume first-time deployments, when healthy.
+ throw ConvergenceException.ofTransient("Refusing to resume until warm up period ends (" +
+ (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")");
+ }
+
+ // Because it's more important to stop a bad release from rolling out in prod,
+ // we put the resume call last. So if we fail after updating the node repo attributes
+ // but before resume, the app may go through the tenant pipeline but will halt in prod.
+ //
+ // Note that this problem exists only because there are 2 different mechanisms
+ // that should really be parts of a single mechanism:
+ // - The content of node repo is used to determine whether a new Vespa+application
+ // has been successfully rolled out.
+ // - Slobrok and internal orchestrator state is used to determine whether
+ // to allow upgrade (suspend).
+ updateNodeRepoWithCurrentAttributes(context, container.map(Container::createdAt));
+ if (suspendedInOrchestrator || node.orchestratorStatus().isSuspended()) {
+ context.log(logger, "Call resume against Orchestrator");
+ orchestrator.resume(context.hostname().value());
+ suspendedInOrchestrator = false;
+ }
+ }
+ case dirty -> {
+ removeContainerIfNeededUpdateContainerState(context, container);
+ context.log(logger, "State is " + node.state() + ", will delete application storage and mark node as ready");
+ credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context));
+ storageMaintainer.syncLogs(context, false);
+ storageMaintainer.archiveNodeStorage(context);
+ updateNodeRepoWithCurrentAttributes(context, Optional.empty());
+ nodeRepository.setNodeState(context.hostname().value(), NodeState.ready);
+ }
+ default -> throw ConvergenceException.ofError("Unexpected state " + node.state().name());
+ }
+ }
+
+ private void runImportantContainerMaintainers(NodeAgentContext context, Container runningContainer) {
+ aclMaintainer.ifPresent(maintainer -> maintainer.converge(context));
+ wireguardTasks.forEach(task -> task.converge(context, runningContainer.id()));
+ }
+
+ private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) {
+ StringBuilder builder = new StringBuilder();
+ appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state);
+ if (builder.length() > 0) {
+ context.log(logger, Level.INFO, "Changes to node: " + builder);
+ }
+ }
+
+ private static <T> String fieldDescription(T value) {
+ return value == null ? "[absent]" : value.toString();
+ }
+
+ private static <T> void appendIfDifferent(StringBuilder builder, String name, NodeSpec oldNode, NodeSpec newNode, Function<NodeSpec, T> getter) {
+ T oldValue = oldNode == null ? null : getter.apply(oldNode);
+ T newValue = getter.apply(newNode);
+ if (!Objects.equals(oldValue, newValue)) {
+ if (builder.length() > 0) {
+ builder.append(", ");
+ }
+ builder.append(name).append(" ").append(fieldDescription(oldValue)).append(" -> ").append(fieldDescription(newValue));
+ }
+ }
+
+ private Optional<Container> getContainer(NodeAgentContext context) {
+ if (containerState == ABSENT) return Optional.empty();
+ Optional<Container> container = containerOperations.getContainer(context);
+ if (container.isEmpty()) containerState = ABSENT;
+ return container;
+ }
+
+ @Override
+ public int getAndResetNumberOfUnhandledExceptions() {
+ int temp = numberOfUnhandledException;
+ numberOfUnhandledException = 0;
+ return temp;
+ }
+
+ private void orchestratorSuspendNode(NodeAgentContext context) {
+ if (context.node().state() != NodeState.active) return;
+
+ context.log(logger, "Ask Orchestrator for permission to suspend node");
+ orchestrator.suspend(context.hostname().value());
+ suspendedInOrchestrator = true;
+ }
+
+ protected void writeContainerData(NodeAgentContext context, ContainerData containerData) { }
+
+ protected List<CredentialsMaintainer> credentialsMaintainers() {
+ return credentialsMaintainers;
+ }
+
+ private Duration warmUpDuration(NodeAgentContext context) {
+ ZoneApi zone = context.zone();
+ Optional<NodeMembership> membership = context.node().membership();
+ return zone.getEnvironment().isTest()
+ || context.nodeType() != NodeType.tenant
+ || membership.map(mem -> ! (mem.type().hasContainer() || mem.type().isAdmin())).orElse(false)
+ ? Duration.ofSeconds(-1)
+ : warmUpDuration.dividedBy(zone.getSystemName().isCd() ? 3 : 1);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
new file mode 100644
index 00000000000..59b3086988e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
@@ -0,0 +1,25 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import java.time.Duration;
+import java.time.Instant;
+
+/**
+ * @author freva
+ */
+public interface NodeAgentScheduler {
+
+ /** Schedule a tick for NodeAgent to run with the given NodeAgentContext, at no earlier than given instant */
+ void scheduleTickWith(NodeAgentContext context, Instant at);
+
+ /**
+ * Will eventually freeze/unfreeze the node agent
+ * @param frozen whether node agent should be frozen
+ * @param timeout maximum duration this method should block while waiting for NodeAgent to reach target state
+ * @return True if node agent has converged to the desired state
+ */
+ boolean setFrozen(boolean frozen, Duration timeout);
+
+ /** @return the last scheduled context or a default value */
+ NodeAgentContext currentContext();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java
new file mode 100644
index 00000000000..3e7895c1ebd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java
@@ -0,0 +1,31 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public enum NodeAgentTask {
+
+ // The full task name is prefixed with 'node>', e.g. 'node>DiskCleanup'
+ DiskCleanup,
+ CoreDumps,
+ CredentialsMaintainer,
+ AclMaintainer;
+
+ private static final Map<String, NodeAgentTask> tasksByName = Arrays.stream(NodeAgentTask.values())
+ .collect(Collectors.toUnmodifiableMap(NodeAgentTask::taskName, n -> n));
+
+ private final String taskName;
+ NodeAgentTask() {
+ this.taskName = "node>" + name();
+ }
+
+ public String taskName() { return taskName; }
+
+ public static Set<NodeAgentTask> fromString(List<String> tasks) {
+ return tasks.stream().filter(tasksByName::containsKey).map(tasksByName::get).collect(Collectors.toUnmodifiableSet());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java
new file mode 100644
index 00000000000..a8effa19b27
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java
@@ -0,0 +1,57 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystem;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.nio.file.Path;
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class PathScope {
+
+ private final ContainerFileSystem containerFs;
+ private final String pathToVespaHome;
+ private final UserScope users;
+
+ public PathScope(ContainerFileSystem containerFs, String pathToVespaHome) {
+ this.containerFs = Objects.requireNonNull(containerFs);
+ this.pathToVespaHome = Objects.requireNonNull(pathToVespaHome);
+ this.users = containerFs.getUserPrincipalLookupService().userScope();
+ }
+
+ public ContainerPath of(String pathInNode) {
+ return of(pathInNode, users.root());
+ }
+
+ public ContainerPath of(String pathInNode, UnixUser user) {
+ return ContainerPath.fromPathInContainer(containerFs, Path.of(pathInNode), user);
+ }
+
+ public ContainerPath underVespaHome(String relativePath) {
+ if (relativePath.startsWith("/"))
+ throw new IllegalArgumentException("Expected a relative path to the Vespa home, got: " + relativePath);
+
+ return ContainerPath.fromPathInContainer(containerFs, Path.of(pathToVespaHome, relativePath), users.vespa());
+ }
+
+ public ContainerPath fromPathOnHost(Path pathOnHost) {
+ return ContainerPath.fromPathOnHost(containerFs, pathOnHost, users.root());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PathScope pathScope = (PathScope) o;
+ return containerFs.equals(pathScope.containerFs) && pathToVespaHome.equals(pathScope.pathToVespaHome) && users.equals(pathScope.users);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(containerFs, pathToVespaHome, users);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java
new file mode 100644
index 00000000000..f44a19de36e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java
@@ -0,0 +1,67 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class UserNamespace {
+
+ /**
+ * IDs outside the ID range are translated to the overflow ID before being written to disk:
+ * https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/Documentation/admin-guide/sysctl/fs.rst#overflowgid--overflowuid
+ * Real value in /proc/sys/fs/overflowuid or overflowgid, hardcode default value*/
+ private static final int OVERFLOW_ID = 65_534;
+
+ private final int uidOffset;
+ private final int gidOffset;
+ private final int idRangeSize;
+
+ public UserNamespace(int uidOffset, int gidOffset, int idRangeSize) {
+ this.uidOffset = uidOffset;
+ this.gidOffset = gidOffset;
+ this.idRangeSize = idRangeSize;
+ }
+
+ public int userIdOnHost(int containerUid) { return toHostId(containerUid, uidOffset, idRangeSize); }
+ public int groupIdOnHost(int containerGid) { return toHostId(containerGid, gidOffset, idRangeSize); }
+ public int userIdInContainer(int hostUid) { return toContainerId(hostUid, uidOffset, idRangeSize); }
+ public int groupIdInContainer(int hostGid) { return toContainerId(hostGid, gidOffset, idRangeSize); }
+
+ public int idRangeSize() { return idRangeSize; }
+ public int overflowId() { return OVERFLOW_ID; }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UserNamespace that = (UserNamespace) o;
+ return uidOffset == that.uidOffset && gidOffset == that.gidOffset && idRangeSize == that.idRangeSize;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(uidOffset, gidOffset, idRangeSize);
+ }
+
+ @Override
+ public String toString() {
+ return "UserNamespace{" +
+ "uidOffset=" + uidOffset +
+ ", gidOffset=" + gidOffset +
+ ", idRangeSize=" + idRangeSize +
+ '}';
+ }
+
+ private static int toHostId(int containerId, int idOffset, int idRangeSize) {
+ if (containerId < 0 || containerId > idRangeSize)
+ throw new IllegalArgumentException("Invalid container id: " + containerId);
+ return idOffset + containerId;
+ }
+
+ private static int toContainerId(int hostId, int idOffset, int idRangeSize) {
+ hostId = hostId - idOffset;
+ return hostId < 0 || hostId >= idRangeSize ? OVERFLOW_ID : hostId;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java
new file mode 100644
index 00000000000..508adde5902
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class UserScope {
+
+ private final UnixUser root;
+ private final UnixUser vespa;
+ private final UserNamespace namespace;
+
+ private UserScope(UnixUser root, UnixUser vespa, UserNamespace namespace) {
+ this.root = Objects.requireNonNull(root);
+ this.vespa = Objects.requireNonNull(vespa);
+ this.namespace = Objects.requireNonNull(namespace);
+ }
+
+ public UnixUser root() {
+ return root;
+ }
+
+ public UnixUser vespa() {
+ return vespa;
+ }
+
+ public UserNamespace namespace() {
+ return namespace;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UserScope userScope = (UserScope) o;
+ return root.equals(userScope.root) && vespa.equals(userScope.vespa) && namespace.equals(userScope.namespace);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(root, vespa, namespace);
+ }
+
+ /** Creates user scope with default root and vespa user */
+ public static UserScope create(UserNamespace namespace) {
+ return new UserScope(UnixUser.ROOT, UnixUser.VESPA, namespace);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java
new file mode 100644
index 00000000000..42310c7233f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java
new file mode 100644
index 00000000000..59040abc4bf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java
@@ -0,0 +1,50 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.provider;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * Class to make it easier to implement a NodeAdminDebugHandler:
+ * - Forward to sub-NodeAdminDebugHandlers with addHandler,
+ * - Specify constants with addConstant
+ * - Forwarding to methods that dynamically build debug objects with addThreadSafeSupplier.
+ *
+ * @author hakonhall
+ */
+public class DebugHandlerHelper implements NodeAdminDebugHandler {
+ private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>();
+
+ public void addThreadSafeSupplier(String name, Supplier<Object> threadSafeSupplier) {
+ Supplier<Object> previousSupplier = suppliers.putIfAbsent(name, threadSafeSupplier);
+ if (previousSupplier != null) {
+ throw new IllegalArgumentException(name + " is already registered");
+ }
+ }
+
+ public void addHandler(String name, NodeAdminDebugHandler handler) {
+ addThreadSafeSupplier(name, handler::getDebugPage);
+ }
+
+ public void addConstant(String name, String value) {
+ addThreadSafeSupplier(name, () -> value);
+ }
+
+ public void remove(String name) {
+ Supplier<Object> supplier = suppliers.remove(name);
+ if (supplier == null) {
+ throw new IllegalArgumentException(name + " is not registered");
+ }
+ }
+
+ @Override
+ public Map<String, Object> getDebugPage() {
+ return suppliers.entrySet().stream().collect(Collectors.toMap(
+ Map.Entry::getKey,
+ entry -> entry.getValue().get()));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java
new file mode 100644
index 00000000000..2c38422e127
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java
@@ -0,0 +1,18 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.provider;
+
+import java.util.Map;
+
+/**
+ * Interface for supporting debug info to introspect e.g. internal state.
+ *
+ * @author hakonhall
+ */
+public interface NodeAdminDebugHandler {
+ /**
+ * The Object in the map values must be serializable with Jackson's ObjectMapper.
+ * May be called concurrently by different threads.
+ */
+ Map<String, Object> getDebugPage();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java
new file mode 100644
index 00000000000..8c8dd618869
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.provider;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java
new file mode 100644
index 00000000000..baf0142df4d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java
@@ -0,0 +1,116 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists;
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.util.stream.Collectors.joining;
+
+/**
+ * Rewrites default-env.txt files.
+ *
+ * @author bjorncs
+ */
+public class DefaultEnvWriter {
+
+ private static final Logger logger = Logger.getLogger(DefaultEnvWriter.class.getName());
+
+ private final Map<String, Operation> operations = new LinkedHashMap<>();
+
+ public DefaultEnvWriter addOverride(String name, String value) {
+ return addOperation("override", name, value);
+ }
+
+ public DefaultEnvWriter addFallback(String name, String value) {
+ return addOperation("fallback", name, value);
+ }
+
+ public DefaultEnvWriter addUnset(String name) {
+ return addOperation("unset", name, null);
+ }
+
+ private DefaultEnvWriter addOperation(String action, String name, String value) {
+ if (operations.containsKey(name)) {
+ throw new IllegalArgumentException(String.format("Operation on variable '%s' already added", name));
+ }
+ operations.put(name, new Operation(action, name, value));
+ return this;
+ }
+
+ /**
+ * Updates or created a default-env.txt file
+ *
+ * @return true if the file was modified
+ */
+ public boolean updateFile(TaskContext context, Path defaultEnvFile) {
+ List<String> currentDefaultEnvLines = ifExists(() -> Files.readAllLines(defaultEnvFile)).orElse(List.of());
+ List<String> newDefaultEnvLines = generateContent(currentDefaultEnvLines);
+ if (currentDefaultEnvLines.equals(newDefaultEnvLines)) {
+ return false;
+ } else {
+ context.log(logger, "Updating " + defaultEnvFile.toString());
+ Path tempFile = defaultEnvFile.resolveSibling(defaultEnvFile.getFileName() + ".tmp");
+ uncheck(() -> Files.write(tempFile, newDefaultEnvLines));
+ uncheck(() -> Files.move(tempFile, defaultEnvFile, ATOMIC_MOVE));
+ return true;
+ }
+ }
+
+ /**
+ * @return generated default-env.txt content
+ */
+ public String generateContent() {
+ return generateContent(List.of()).stream()
+ .collect(joining(System.lineSeparator(), "", System.lineSeparator()));
+ }
+
+ private List<String> generateContent(List<String> currentDefaultEnvLines) {
+ List<String> newDefaultEnvLines = new ArrayList<>();
+ Set<String> seenNames = new TreeSet<>();
+ for (String line : currentDefaultEnvLines) {
+ String[] items = line.split(" ");
+ if (items.length < 2) {
+ throw new IllegalArgumentException(String.format("Invalid line in file '%s': %s", currentDefaultEnvLines, line));
+ }
+ String name = items[1];
+ if (!seenNames.contains(name)) { // implicitly removes duplicated variables
+ seenNames.add(name);
+ Operation operation = operations.get(name);
+ if (operation != null) {
+ newDefaultEnvLines.add(operation.toLine());
+ } else {
+ newDefaultEnvLines.add(line);
+ }
+ }
+ }
+ for (var operation : operations.values()) {
+ if (!seenNames.contains(operation.name)) {
+ newDefaultEnvLines.add(operation.toLine());
+ }
+ }
+ return newDefaultEnvLines;
+ }
+
+ private record Operation(String action, String name, String value) {
+ String toLine() {
+ if (action.equals("unset")) {
+ return "unset " + name;
+ }
+ return action + " " + name + " " + value;
+ }
+ }
+}
+
+
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java
new file mode 100644
index 00000000000..4e9998bd40f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java
@@ -0,0 +1,96 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+
+/**
+ * Simulates an editor cursor.
+ *
+ * @author hakon
+ */
+public interface Cursor {
+ // CURSOR AND BUFFER QUERIES
+
+ String getBufferText();
+ String getLine();
+ String getPrefix();
+ String getSuffix();
+ String getTextTo(Mark mark);
+
+ Position getPosition();
+ Mark createMark();
+
+ // CURSOR MOVEMENT
+
+ Cursor moveToStartOfBuffer();
+ Cursor moveToEndOfBuffer();
+
+ Cursor moveToStartOfLine();
+ Cursor moveToStartOfPreviousLine();
+ Cursor moveToStartOfNextLine();
+ Cursor moveToStartOf(int lineIndex);
+
+ Cursor moveToEndOfLine();
+ Cursor moveToEndOfPreviousLine();
+ Cursor moveToEndOfNextLine();
+ Cursor moveToEndOf(int lineIndex);
+
+ Cursor moveForward();
+ Cursor moveForward(int times);
+ Cursor moveBackward();
+ Cursor moveBackward(int times);
+
+ Cursor moveTo(Mark mark);
+ Cursor moveTo(Position position);
+ Cursor moveTo(int lineIndex, int columnIndex);
+
+ Optional<Match> moveForwardToStartOfMatch(Pattern pattern);
+ Optional<Match> moveForwardToEndOfMatch(Pattern pattern);
+
+ boolean skipBackward(String text);
+ boolean skipForward(String text);
+
+ // BUFFER MODIFICATIONS
+
+ Cursor write(String text);
+ Cursor writeLine(String line);
+ Cursor writeLines(String... lines);
+ Cursor writeLines(Iterable<String> lines);
+
+ Cursor writeNewline();
+ Cursor writeNewlineAfter();
+
+ Cursor deleteAll();
+ Cursor deleteLine();
+ Cursor deletePrefix();
+ Cursor deleteSuffix();
+
+ Cursor deleteForward();
+ Cursor deleteForward(int times);
+ Cursor deleteBackward();
+ Cursor deleteBackward(int times);
+
+ Cursor deleteTo(Mark mark);
+
+ boolean replaceMatch(Pattern pattern, Function<Match, String> replacer);
+
+ /**
+ * Replace matches of a pattern.
+ *
+ * <p>The search for {@code pattern} starts at cursor and matches against the remaining line,
+ * and the full line for the following lines. Each match is replaced by a String returned by
+ * {@code replacer::apply}.
+ *
+ * <p>The cursor is unchanged without any matches, or moved to the end of the last replacement.
+ *
+ * <p>To replace all matches in a buffer, first call {@link #moveToStartOfBuffer()} to
+ * postion the cursor at the beginning of the buffer.
+ *
+ * @see #moveForwardToStartOfMatch(Pattern)
+ * @see #moveForwardToEndOfMatch(Pattern)
+ */
+ int replaceMatches(Pattern pattern, Function<Match, String> replacer);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java
new file mode 100644
index 00000000000..501db764d05
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java
@@ -0,0 +1,356 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+
+import static com.yahoo.collections.Comparables.max;
+import static com.yahoo.collections.Comparables.min;
+
+/**
+ * @author hakon
+ */
+public class CursorImpl implements Cursor {
+ private final TextBuffer textBuffer;
+ private final Object unique = new Object();
+
+ private Position position;
+
+ /**
+ * Creates a cursor to a text buffer.
+ *
+ * WARNING: The text buffer MUST NOT be accessed outside this cursor. This cursor
+ * takes sole ownership of the text buffer.
+ *
+ * @param textBuffer the text buffer this cursor owns and operates on
+ */
+ CursorImpl(TextBuffer textBuffer) {
+ this.textBuffer = textBuffer;
+ position = textBuffer.getStartOfText();
+ }
+
+ @Override
+ public Position getPosition() {
+ return position;
+ }
+
+ @Override
+ public Mark createMark() {
+ return new Mark(position, textBuffer.getVersion(), unique);
+ }
+
+ @Override
+ public String getBufferText() {
+ return textBuffer.getString();
+ }
+
+ @Override
+ public String getLine() {
+ return textBuffer.getLine(position);
+ }
+
+ @Override
+ public String getPrefix() {
+ return textBuffer.getLinePrefix(position);
+ }
+
+ @Override
+ public String getSuffix() {
+ return textBuffer.getLineSuffix(position);
+ }
+
+ @Override
+ public String getTextTo(Mark mark) {
+ validateMark(mark);
+
+ Position start = min(mark.position(), position);
+ Position end = max(mark.position(), position);
+
+ return textBuffer.getSubstring(start, end);
+ }
+
+ @Override
+ public Cursor moveToStartOfBuffer() {
+ position = textBuffer.getStartOfText();
+ return this;
+ }
+
+ @Override
+ public Cursor moveToEndOfBuffer() {
+ position = textBuffer.getEndOfText();
+ return this;
+ }
+
+ @Override
+ public Cursor moveToStartOfLine() {
+ position = textBuffer.getStartOfLine(position);
+ return this;
+ }
+
+ @Override
+ public Cursor moveToStartOfPreviousLine() {
+ position = textBuffer.getStartOfPreviousLine(position);
+ return this;
+ }
+
+ @Override
+ public Cursor moveToStartOfNextLine() {
+ position = textBuffer.getStartOfNextLine(position);
+ return this;
+ }
+
+ @Override
+ public Cursor moveToStartOf(int lineIndex) {
+ validateLineIndex(lineIndex);
+ position = new Position(lineIndex, 0);
+ return this;
+ }
+
+ @Override
+ public Cursor moveToEndOfLine() {
+ position = textBuffer.getEndOfLine(position);
+ return this;
+ }
+
+ @Override
+ public Cursor moveToEndOfPreviousLine() {
+ return moveToStartOfPreviousLine().moveToEndOfLine();
+ }
+
+ @Override
+ public Cursor moveToEndOfNextLine() {
+ return moveToStartOfNextLine().moveToEndOfLine();
+ }
+
+ @Override
+ public Cursor moveToEndOf(int lineIndex) {
+ return moveToStartOf(lineIndex).moveToEndOfLine();
+ }
+
+ @Override
+ public Cursor moveForward() {
+ return moveForward(1);
+ }
+
+ @Override
+ public Cursor moveForward(int times) {
+ position = textBuffer.forward(position, times);
+ return this;
+ }
+
+ @Override
+ public Cursor moveBackward() {
+ return moveBackward(1);
+ }
+
+ @Override
+ public Cursor moveBackward(int times) {
+ position = textBuffer.backward(position, times);
+ return this;
+ }
+
+ @Override
+ public Cursor moveTo(Mark mark) {
+ validateMark(mark);
+ position = mark.position();
+ return this;
+ }
+
+ @Override
+ public boolean skipBackward(String text) {
+ String prefix = getPrefix();
+ if (prefix.endsWith(text)) {
+ position = new Position(position.lineIndex(), position.columnIndex() - text.length());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public boolean skipForward(String text) {
+ String suffix = getSuffix();
+ if (suffix.startsWith(text)) {
+ position = new Position(position.lineIndex(), position.columnIndex() + text.length());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public Optional<Match> moveForwardToStartOfMatch(Pattern pattern) {
+ return moveForwardToXOfMatch(pattern, match -> position = match.startOfMatch());
+ }
+
+ @Override
+ public Optional<Match> moveForwardToEndOfMatch(Pattern pattern) {
+ return moveForwardToXOfMatch(pattern, match -> position = match.endOfMatch());
+ }
+
+ private Optional<Match> moveForwardToXOfMatch(Pattern pattern, Consumer<Match> callback) {
+ Optional<Match> match = textBuffer.findForward(position, pattern);
+ match.ifPresent(callback);
+ return match;
+ }
+
+ @Override
+ public Cursor moveTo(Position position) {
+ validatePosition(position);
+ this.position = position;
+ return this;
+ }
+
+ @Override
+ public Cursor moveTo(int lineIndex, int columnIndex) {
+ return moveTo(new Position(lineIndex, columnIndex));
+ }
+
+ @Override
+ public Cursor write(String text) {
+ position = textBuffer.write(position, text);
+ return this;
+ }
+
+ @Override
+ public Cursor writeLine(String line) {
+ return write(line).write("\n");
+ }
+
+ @Override
+ public Cursor writeLines(String... lines) {
+ return writeLines(List.of(lines));
+ }
+
+ @Override
+ public Cursor writeLines(Iterable<String> lines) {
+ return writeLine(String.join("\n", lines));
+ }
+
+ @Override
+ public Cursor writeNewline() {
+ return write("\n");
+ }
+
+ @Override
+ public Cursor writeNewlineAfter() {
+ return writeNewline().moveBackward();
+ }
+
+ @Override
+ public Cursor deleteAll() {
+ moveToStartOfBuffer();
+ textBuffer.clear();
+ return this;
+ }
+
+ @Override
+ public Cursor deleteLine() {
+ moveToStartOfLine();
+ textBuffer.delete(position, textBuffer.getStartOfNextLine(position));
+ return this;
+ }
+
+ @Override
+ public Cursor deletePrefix() {
+ Position originalPosition = position;
+ moveToStartOfLine();
+ textBuffer.delete(position, originalPosition);
+ return this;
+ }
+
+ @Override
+ public Cursor deleteSuffix() {
+ textBuffer.delete(position, textBuffer.getEndOfLine(position));
+ return this;
+ }
+
+ @Override
+ public Cursor deleteForward() {
+ return deleteForward(1);
+ }
+
+ @Override
+ public Cursor deleteForward(int times) {
+ Position end = textBuffer.forward(position, times);
+ textBuffer.delete(position, end);
+ return this;
+ }
+
+ @Override
+ public Cursor deleteBackward() {
+ return deleteBackward(1);
+ }
+
+ @Override
+ public Cursor deleteBackward(int times) {
+ Position end = position;
+ moveBackward(times);
+ textBuffer.delete(position, end);
+ return this;
+ }
+
+ @Override
+ public Cursor deleteTo(Mark mark) {
+ validateMark(mark);
+ Position start = min(mark.position(), position);
+ Position end = max(mark.position(), position);
+
+ textBuffer.delete(start, end);
+ return this;
+ }
+
+ @Override
+ public boolean replaceMatch(Pattern pattern, Function<Match, String> replacer) {
+ Optional<Match> match = moveForwardToStartOfMatch(pattern);
+ if (match.isEmpty()) {
+ return false;
+ }
+
+ textBuffer.delete(match.get().startOfMatch(), match.get().endOfMatch());
+ write(replacer.apply(match.get()));
+ return true;
+ }
+
+ @Override
+ public int replaceMatches(Pattern pattern, Function<Match, String> replacer) {
+ int count = 0;
+
+ for (; replaceMatch(pattern, replacer); ++count) {
+ // empty
+ }
+
+ return count;
+ }
+
+ private void validatePosition(Position position) {
+ validateLineIndex(position.lineIndex());
+
+ int maxColumnIndex = textBuffer.getLine(position.lineIndex()).length();
+ if (position.columnIndex() < 0 || position.columnIndex() > maxColumnIndex) {
+ throw new IndexOutOfBoundsException("Column index of " + position.coordinateString() +
+ " is not in permitted range [0," + maxColumnIndex + "]");
+ }
+ }
+
+ private void validateLineIndex(int lineIndex) {
+ int maxLineIndex = textBuffer.getMaxLineIndex();
+ if (lineIndex < 0 || lineIndex > maxLineIndex) {
+ throw new IndexOutOfBoundsException("Line index " + lineIndex +
+ " not in permitted range [0," + maxLineIndex + "]");
+ }
+ }
+
+ private void validateMark(Mark mark) {
+ if (mark.secret() != unique) {
+ throw new IllegalArgumentException("Unknown mark " + mark);
+ }
+
+ if (!mark.version().equals(textBuffer.getVersion())) {
+ throw new IllegalArgumentException("Mark " + mark + " is outdated");
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java
new file mode 100644
index 00000000000..fb09482a43a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+
+import java.nio.file.Path;
+
+/**
+ * @author hakon
+ */
+public class FileEditor {
+ private final UnixPath path;
+ private final StringEditor stringEditor;
+
+ private String fileText;
+ private Version fileVersion;
+
+ public static FileEditor open(Path path) {
+ UnixPath unixPath = new UnixPath(path);
+ String text = unixPath.readUtf8File();
+ StringEditor stringEditor = new StringEditor(text);
+ return new FileEditor(unixPath, text, stringEditor);
+ }
+
+ private FileEditor(UnixPath path, String fileText, StringEditor stringEditor) {
+ this.path = path;
+ this.fileText = fileText;
+ this.stringEditor = stringEditor;
+ fileVersion = stringEditor.bufferVersion();
+ }
+
+ public Cursor cursor() {
+ return stringEditor.cursor();
+ }
+
+ public void reloadFile() {
+ fileText = path.readUtf8File();
+ stringEditor.cursor().deleteAll().write(fileText);
+ fileVersion = stringEditor.bufferVersion();
+ }
+
+ public boolean save() {
+ Version bufferVersion = stringEditor.bufferVersion();
+ if (bufferVersion.equals(fileVersion)) {
+ return false;
+ }
+
+ String newText = stringEditor.cursor().getBufferText();
+ if (newText.equals(fileText)) {
+ return false;
+ }
+
+ path.writeUtf8File(newText);
+ fileVersion = bufferVersion;
+ return true;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java
new file mode 100644
index 00000000000..616c98c5b76
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.Objects;
+
+/**
+ * @author hakon
+ */
+public class Mark {
+ private final Position position;
+ private final Version version;
+ private final Object token;
+
+ Mark(Position position, Version version, Object token) {
+ this.position = position;
+ this.version = version;
+ this.token = token;
+ }
+
+ public Position position() {
+ return position;
+ }
+
+ public Version version() {
+ return version;
+ }
+
+ public Object secret() {
+ return token;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Mark mark = (Mark) o;
+ return Objects.equals(position, mark.position) &&
+ Objects.equals(version, mark.version) &&
+ token == mark.token;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(position, version, token);
+ }
+
+ @Override
+ public String toString() {
+ return position.coordinateString() + "@" + version;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java
new file mode 100644
index 00000000000..32e058c1067
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java
@@ -0,0 +1,53 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.regex.Matcher;
+
+/**
+ * Represents a pattern match of a line
+ *
+ * @author hakon
+ */
+public class Match {
+ private final int lineIndex;
+ private final String line;
+ private final Matcher matcher;
+
+ Match(int lineIndex, String line, Matcher matcher) {
+ this.lineIndex = lineIndex;
+ this.line = line;
+ this.matcher = matcher;
+ }
+
+ /** The part of the line before the match */
+ public String prefix() {
+ return line.substring(0, matcher.start());
+ }
+
+ /** The part of the line that matched */
+ public String match() {
+ return matcher.group();
+ }
+
+ /** The part of the line that followed the match */
+ public String suffix() {
+ return line.substring(matcher.end());
+ }
+
+ public Position startOfMatch() {
+ return new Position(lineIndex, matcher.start());
+ }
+
+ public Position endOfMatch() {
+ return new Position(lineIndex, matcher.end());
+ }
+
+ public int groupCount() {
+ return matcher.groupCount();
+ }
+
+ public String group(int groupnr) {
+ return matcher.group(groupnr);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java
new file mode 100644
index 00000000000..95aa778d57e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java
@@ -0,0 +1,72 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.Comparator;
+import java.util.Objects;
+
+/**
+ * Represents a position in the buffer
+ *
+ * @author hakon
+ */
+public class Position implements Comparable<Position> {
+ private static final Position START_POSITION = new Position(0, 0);
+
+ private static final Comparator<Position> COMPARATOR = Comparator
+ .comparingInt(Position::lineIndex)
+ .thenComparingInt(Position::columnIndex);
+
+ private final int lineIndex;
+ private final int columnIndex;
+
+ /** Returns the first position at line index 0 and column index 0 */
+ public static Position start() {
+ return START_POSITION;
+ }
+
+ Position(int lineIndex, int columnIndex) {
+ this.lineIndex = lineIndex;
+ this.columnIndex = columnIndex;
+ }
+
+ public int lineIndex() {
+ return lineIndex;
+ }
+
+ public int columnIndex() {
+ return columnIndex;
+ }
+
+ @Override
+ public int compareTo(Position that) {
+ return COMPARATOR.compare(this, that);
+ }
+
+ public boolean isAfter(Position that) { return compareTo(that) > 0; }
+ public boolean isNotAfter(Position that) { return !isAfter(that); }
+ public boolean isBefore(Position that) { return compareTo(that) < 0; }
+ public boolean isNotBefore(Position that) { return !isBefore(that); }
+
+ public String coordinateString() {
+ return "(" + lineIndex + "," + columnIndex + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Position position = (Position) o;
+ return lineIndex == position.lineIndex &&
+ columnIndex == position.columnIndex;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(lineIndex, columnIndex);
+ }
+
+ @Override
+ public String toString() {
+ return coordinateString();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java
new file mode 100644
index 00000000000..ea55e3c11a4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java
@@ -0,0 +1,30 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+/**
+ * Edits multi-line text.
+ *
+ * @author hakon
+ */
+public class StringEditor {
+ private final TextBuffer textBuffer;
+ private final Cursor cursor;
+
+ public StringEditor() {
+ this("");
+ }
+
+ public StringEditor(String text) {
+ textBuffer = new TextBufferImpl(text);
+ cursor = new CursorImpl(textBuffer);
+ }
+
+ public Cursor cursor() {
+ return cursor;
+ }
+
+ public Version bufferVersion() {
+ return textBuffer.getVersion();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java
new file mode 100644
index 00000000000..e6cf211d481
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java
@@ -0,0 +1,175 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author hakon
+ */
+interface TextBuffer {
+ // INTERFACE TO IMPLEMENT BY CONCRETE CLASS
+
+ /** Get the version of the buffer - edits increment the version. */
+ Version getVersion();
+
+ /** Return the text as a single String (likely) with embedded newlines. */
+ String getString();
+
+ /** Return the maximum line index (the minimum line index is 0). */
+ int getMaxLineIndex();
+
+ /** @param lineIndex must be in in {@code [0, getMaxLineIndex()]} */
+ String getLine(int lineIndex);
+
+ /** Insert the possibly multi-line text at position and return the end position. */
+ Position write(Position position, String text);
+
+ /** Delete everything. */
+ void clear();
+
+ /** Delete range. */
+ void delete(Position start, Position end);
+
+ // DERIVED IMPLEMENTATION
+
+ /**
+ * Return the Position closest to {@code position} which is in the range
+ * {@code [getStartOfText(), getEndOfText()]}.
+ */
+ default Position getValidPositionClosestTo(Position position) {
+ if (position.isBefore(getStartOfText())) {
+ return getStartOfText();
+ } else if (position.isAfter(getEndOfText())) {
+ return getEndOfText();
+ } else {
+ return position;
+ }
+ }
+
+ default String getLine(Position position) { return getLine(position.lineIndex()); }
+
+ default String getLinePrefix(Position position) {
+ return getLine(position.lineIndex()).substring(0, position.columnIndex());
+ }
+
+ default String getLineSuffix(Position position) {
+ return getLine(position.lineIndex()).substring(position.columnIndex());
+ }
+
+ default String getSubstring(Position start, Position end) {
+ if (start.lineIndex() < end.lineIndex()) {
+ StringBuilder builder = new StringBuilder(getLineSuffix(start));
+ for (int i = start.lineIndex() + 1; i < end.lineIndex(); ++i) {
+ builder.append('\n');
+ builder.append(getLine(i));
+ }
+ return builder.append('\n').append(getLinePrefix(end)).toString();
+ } else if (start.lineIndex() == end.lineIndex() && start.columnIndex() <= end.columnIndex()) {
+ return getLine(start).substring(start.columnIndex(), end.columnIndex());
+ }
+
+ throw new IllegalArgumentException(
+ "Bad range [" + start.coordinateString() + "," + end.coordinateString() + ">");
+ }
+
+ default Position getStartOfText() { return Position.start(); } // aka (0,0)
+
+ default Position getEndOfText() {
+ int maxLineIndex = getMaxLineIndex();
+ return new Position(maxLineIndex, getLine(maxLineIndex).length());
+ }
+
+ default Position getStartOfLine(Position position) {
+ return new Position(position.lineIndex(), 0);
+ }
+
+ default Position getEndOfLine(Position position) {
+ return new Position(position.lineIndex(), getLine(position).length());
+ }
+
+ default Position getStartOfNextLine(Position position) {
+ if (position.lineIndex() < getMaxLineIndex()) {
+ return new Position(position.lineIndex() + 1, 0);
+ } else {
+ return getEndOfText();
+ }
+ }
+
+ default Position getStartOfPreviousLine(Position position) {
+ int lineIndex = position.lineIndex();
+ if (lineIndex > 0) {
+ return new Position(lineIndex - 1, 0);
+ } else {
+ return getStartOfText();
+ }
+ }
+
+ default Position forward(Position position, int length) {
+ int lineIndex = position.lineIndex();
+ int columnIndex = position.columnIndex();
+
+ int offsetLeft = length;
+ do {
+ String line = getLine(lineIndex);
+ int columnIndexWithInfiniteLine = columnIndex + offsetLeft;
+ if (columnIndexWithInfiniteLine <= line.length()) {
+ return new Position(lineIndex, columnIndexWithInfiniteLine);
+ } else if (lineIndex >= getMaxLineIndex()) {
+ // End of text
+ return new Position(lineIndex, line.length());
+ }
+
+ offsetLeft -= line.length() - columnIndex;
+
+ // advance past newline
+ --offsetLeft;
+ ++lineIndex;
+ columnIndex = 0;
+
+ // At this point: offsetLeft is guaranteed to be >= 0, and lineIndex <= max line index
+ } while (true);
+ }
+
+ default Position backward(Position position, int length) {
+ int lineIndex = position.lineIndex();
+ int columnIndex = position.columnIndex();
+
+ int offsetLeft = length;
+ do {
+ int columnIndexWithInfiniteLine = columnIndex - offsetLeft;
+ if (columnIndexWithInfiniteLine >= 0) {
+ return new Position(lineIndex, columnIndexWithInfiniteLine);
+ } else if (lineIndex <= 0) {
+ // Start of text
+ return new Position(0, 0);
+ }
+
+ offsetLeft -= columnIndex;
+
+ // advance past newline
+ --offsetLeft;
+ --lineIndex;
+ columnIndex = getLine(lineIndex).length();
+
+ // At this point: offsetLeft is guaranteed to be <= 0, and lineIndex >= 0
+ } while (true);
+ }
+
+ default Optional<Match> findForward(Position startPosition, Pattern pattern) {
+ for (Position position = startPosition; ; position = getStartOfNextLine(position)) {
+ String line = getLine(position);
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.find(position.columnIndex())) {
+ return Optional.of(new Match(position.lineIndex(), line, matcher));
+ }
+
+ if (position.lineIndex() == getMaxLineIndex()) {
+ // search failed - no lines matched
+ return Optional.empty();
+ }
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java
new file mode 100644
index 00000000000..0a7ff26c73c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java
@@ -0,0 +1,117 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.editor.TextUtil.splitString;
+
+/**
+ * @author hakon
+ */
+public class TextBufferImpl implements TextBuffer {
+ /** Invariant: {@code size() >= 1}. An empty text buffer {@code => [""]} */
+ private final ArrayList<String> lines = new ArrayList<>();
+
+ private Version version = new Version();
+
+ TextBufferImpl() {
+ lines.add("");
+ }
+
+ TextBufferImpl(String text) {
+ this();
+ write(getStartOfText(), text);
+ // reset version
+ version = new Version();
+ }
+
+ @Override
+ public Version getVersion() {
+ return version;
+ }
+
+ @Override
+ public String getString() {
+ return String.join("\n", lines);
+ }
+
+ @Override
+ public int getMaxLineIndex() {
+ return lines.size() - 1;
+ }
+
+ @Override
+ public String getLine(int lineIndex) {
+ return lines.get(lineIndex);
+ }
+
+ @Override
+ public Position write(Position position, String text) {
+ List<String> linesToInsert = new LinkedList<>(splitString(text, true, false));
+ if (linesToInsert.isEmpty()) {
+ return position;
+ }
+
+ // The position splits that line in two, and both prefix and suffix must be preserved
+ linesToInsert.set(0, getLinePrefix(position) + linesToInsert.get(0));
+ String lastLine = linesToInsert.get(linesToInsert.size() - 1);
+ int endColumnIndex = lastLine.length();
+ linesToInsert.set(linesToInsert.size() - 1, lastLine + getLineSuffix(position));
+
+ // Set the first line at lineIndex, insert the rest.
+ int lineIndex = position.lineIndex();
+ int endLineIndex = lineIndex + linesToInsert.size() - 1;
+ lines.set(lineIndex, linesToInsert.remove(0));
+ lines.addAll(lineIndex + 1, linesToInsert);
+
+ incrementVersion();
+
+ return new Position(endLineIndex, endColumnIndex);
+ }
+
+ @Override
+ public void clear() {
+ lines.clear();
+ lines.add("");
+ }
+
+ @Override
+ public void delete(Position start, Position end) {
+ if (start.isAfter(end)) {
+ throw new IllegalArgumentException("start position " + start +
+ " is after end position " + end);
+ }
+
+ String prefix = getLinePrefix(start);
+ String suffix = getLineSuffix(end);
+ String stichedLine = prefix + suffix;
+
+ lines.set(start.lineIndex(), stichedLine);
+
+ deleteLines(start.lineIndex() + 1, end.lineIndex() + 1);
+
+ incrementVersion();
+ }
+
+ private void deleteLines(int startIndex, int endIndex) {
+ for (int fromIndex = endIndex, toIndex = startIndex; fromIndex <= getMaxLineIndex();
+ ++toIndex, ++fromIndex) {
+ lines.set(toIndex, lines.get(fromIndex));
+ }
+
+ truncate(getMaxLineIndex() - (endIndex - startIndex));
+ }
+
+ private void truncate(int newMaxLineIndex) {
+ while (getMaxLineIndex() > newMaxLineIndex) {
+ lines.remove(getMaxLineIndex());
+ }
+ }
+
+ private void incrementVersion() {
+ version = version.next();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java
new file mode 100644
index 00000000000..625bb608fd7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java
@@ -0,0 +1,59 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * @author hakon
+ */
+public class TextUtil {
+ private TextUtil() {}
+
+ /**
+ * Splits {@code text} by newline (LF {@code '\n'}).
+ *
+ * @param text the text to split into lines
+ * @param empty whether an empty text implies an empty List (true), or a List with one
+ * empty String element (false)
+ * @param prune whether a text ending with a newline will result in a List ending with the
+ * preceding line (true), or to add an empty String element (false)
+ */
+ public static List<String> splitString(String text, boolean empty, boolean prune) {
+ List<String> lines = new ArrayList<>();
+ splitString(text, empty, prune, lines::add);
+ return lines;
+ }
+
+ /**
+ * Splits text by newline, passing each line to a consumer.
+ *
+ * @see #splitString(String, boolean, boolean)
+ */
+ public static void splitString(String text,
+ boolean empty,
+ boolean prune,
+ Consumer<String> consumer) {
+ if (text.isEmpty()) {
+ if (!empty) {
+ consumer.accept(text);
+ }
+ return;
+ }
+
+ final int endIndex = text.length();
+
+ int start = 0;
+ for (int end = text.indexOf('\n');
+ end != -1;
+ start = end + 1, end = text.indexOf('\n', start)) {
+ consumer.accept(text.substring(start, end));
+ }
+
+ if (start < endIndex || !prune) {
+ consumer.accept(text.substring(start));
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java
new file mode 100644
index 00000000000..97d8cbb6a50
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import java.util.Objects;
+
+/**
+ * Represents a snapshot of the TextBuffer, between two edits (or the initial or final state)
+ *
+ * @author hakon
+ */
+public class Version {
+ private final int version;
+
+ Version() {
+ this(0);
+ }
+
+ private Version(int version) {
+ this.version = version;
+ }
+
+ public boolean isBefore(Version that) {
+ return version < that.version;
+ }
+
+ public int asInt() {
+ return version;
+ }
+
+ public Version next() {
+ return new Version(version + 1);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Version that = (Version) o;
+ return version == that.version;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(version);
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(version);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java
new file mode 100644
index 00000000000..73eddd2bbe2
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java
@@ -0,0 +1,125 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+/**
+ * Class to converge file/directory attributes like owner and permissions to wanted values.
+ * Typically used by higher abstraction layers working on files (FileSync/FileWriter) or
+ * directories (MakeDirectory).
+ *
+ * @author hakonhall
+ */
+public class AttributeSync {
+ private static final Logger logger = Logger.getLogger(AttributeSync.class.getName());
+
+ private final UnixPath path;
+
+ private Optional<Integer> ownerId = Optional.empty();
+ private Optional<Integer> groupId = Optional.empty();
+ private Optional<String> permissions = Optional.empty();
+
+ public AttributeSync(Path path) {
+ this.path = new UnixPath(path);
+ }
+
+ public Optional<String> getPermissions() {
+ return permissions;
+ }
+
+ public AttributeSync withPermissions(String permissions) {
+ this.permissions = Optional.of(permissions);
+ return this;
+ }
+
+ public Optional<Integer> ownerId() {
+ return ownerId;
+ }
+
+ public AttributeSync withOwnerId(int ownerId) {
+ this.ownerId = Optional.of(ownerId);
+ return this;
+ }
+
+ public Optional<Integer> groupId() {
+ return groupId;
+ }
+
+ public AttributeSync withGroupId(int groupId) {
+ this.groupId = Optional.of(groupId);
+ return this;
+ }
+
+ public AttributeSync with(PartialFileData fileData) {
+ ownerId = fileData.getOwnerId();
+ groupId = fileData.getGroupId();
+ permissions = fileData.getPermissions();
+ return this;
+ }
+
+ public boolean converge(TaskContext context) {
+ return converge(context, new FileAttributesCache(path));
+ }
+
+ /**
+ * Path must exist before calling converge.
+ */
+ public boolean converge(TaskContext context, FileAttributesCache currentAttributes) {
+ boolean systemModified = updateAttribute(
+ context,
+ "user ID",
+ ownerId,
+ () -> currentAttributes.getOrThrow().ownerId(),
+ path::setOwnerId);
+
+ systemModified |= updateAttribute(
+ context,
+ "group ID",
+ groupId,
+ () -> currentAttributes.getOrThrow().groupId(),
+ path::setGroupId);
+
+ systemModified |= updateAttribute(
+ context,
+ "permissions",
+ permissions,
+ () -> currentAttributes.getOrThrow().permissions(),
+ path::setPermissions);
+
+ return systemModified;
+ }
+
+ private <T> boolean updateAttribute(TaskContext context,
+ String attributeName,
+ Optional<T> wantedValue,
+ Supplier<T> currentValueSupplier,
+ Consumer<T> valueSetter) {
+ if (wantedValue.isEmpty()) {
+ return false;
+ }
+
+ T currentValue = currentValueSupplier.get();
+ if (Objects.equals(currentValue, wantedValue.get())) {
+ return false;
+ }
+
+ context.recordSystemModification(
+ logger,
+ String.format("Changing %s of %s from %s to %s",
+ attributeName,
+ path,
+ currentValue,
+ wantedValue.get()));
+
+ valueSetter.accept(wantedValue.get());
+
+ return true;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java
new file mode 100644
index 00000000000..b1fedd47e60
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java
@@ -0,0 +1,71 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * @author freva
+ */
+public class DiskSize implements Comparable<DiskSize> {
+
+ public static final DiskSize ZERO = DiskSize.of(0);
+ private static final char[] UNITS = "kMGTPE".toCharArray();
+
+ public enum Unit {
+ kB(1_000), kiB(1 << 10),
+ MB(1_000_000), MiB(1 << 20),
+ GB(1_000_000_000), GiB(1 << 30),
+ TB(1_000_000_000_000L), TiB(1L << 40),
+ PB(1_000_000_000_000_000L), PiB(1L << 50);
+
+ private final long size;
+ Unit(long size) { this.size = size; }
+ }
+
+ private final long bytes;
+ private DiskSize(long bytes) { this.bytes = bytes; }
+
+ public long bytes() { return bytes; }
+ public long as(Unit unit) { return bytes / unit.size; }
+ public double asDouble(Unit unit) { return (double) bytes / unit.size; }
+
+ public DiskSize add(DiskSize other) { return new DiskSize(bytes + other.bytes); }
+
+ public static DiskSize of(long bytes) { return new DiskSize(bytes); }
+ public static DiskSize of(double bytes, Unit unit) { return new DiskSize((long) (bytes * unit.size)); }
+ public static DiskSize of(long bytes, Unit unit) { return new DiskSize(bytes * unit.size); }
+
+ public String asString() { return asString(0); }
+ public String asString(int decimals) {
+ if (bytes < 1000) return bytes + " bytes";
+
+ int unit = -1;
+ double remaining = bytes;
+ for (; remaining >= 1000; unit++) remaining /= 1000;
+ return String.format(Locale.ENGLISH, "%." + decimals + "f %sB", remaining, UNITS[unit]);
+ }
+
+ @Override
+ public int compareTo(DiskSize rhs) {
+ return Long.compare(this.bytes, rhs.bytes);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ DiskSize size = (DiskSize) o;
+ return bytes == size.bytes;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(bytes);
+ }
+
+ @Override
+ public String toString() {
+ return asString();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java
new file mode 100644
index 00000000000..66269602afd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java
@@ -0,0 +1,135 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * An editor meant to edit small line-based files like /etc/fstab.
+ *
+ * @author hakonhall
+ */
+public class Editor {
+ private static final Logger logger = Logger.getLogger(Editor.class.getName());
+ private static final Charset ENCODING = StandardCharsets.UTF_8;
+
+ private static final int MAX_LENGTH = 500;
+
+ private final Supplier<List<String>> supplier;
+ private final Consumer<List<String>> consumer;
+ private final String name;
+ private final LineEditor editor;
+ private int diffSize = 0;
+
+ /**
+ * Read the file which must be encoded in UTF-8, use the LineEditor to edit it,
+ * and any modifications were done write it back and return true.
+ */
+ public Editor(Path path, LineEditor editor) {
+ this(path.toString(),
+ () -> ifExists(() -> Files.readAllLines(path, ENCODING)).orElseGet(List::of),
+ (newLines) -> uncheck(() -> Files.write(path, newLines, ENCODING)),
+ editor);
+ }
+
+ /**
+ * @param name The name of what is being edited - used in logging
+ * @param supplier Supplies the editor with a list of lines to edit
+ * @param consumer Consumes the lines to presist if any changes is detected
+ * @param editor The line operations to execute on the lines supplied
+ */
+ public Editor(String name,
+ Supplier<List<String>> supplier,
+ Consumer<List<String>> consumer,
+ LineEditor editor) {
+ this.supplier = supplier;
+ this.consumer = consumer;
+ this.name = name;
+ this.editor = editor;
+ }
+
+ public boolean edit(Consumer<String> logConsumer) {
+ List<String> lines = supplier.get();
+ List<String> newLines = new LinkedList<>();
+ StringBuilder diff = new StringBuilder();
+ boolean modified = false;
+
+ for (String line : lines) {
+ LineEdit edit = editor.edit(line);
+ if (!edit.prependLines().isEmpty()) {
+ modified = true;
+ maybeAdd(diff, edit.prependLines());
+ newLines.addAll(edit.prependLines());
+ }
+
+ switch (edit.getType()) {
+ case REPLACE -> {
+ modified = true;
+ maybeRemove(diff, line);
+ }
+ case NONE -> newLines.add(line);
+ default -> throw new IllegalArgumentException("Unknown EditType " + edit.getType());
+ }
+
+ if (!edit.appendLines().isEmpty()) {
+ modified = true;
+ maybeAdd(diff, edit.appendLines());
+ newLines.addAll(edit.appendLines());
+ }
+ }
+
+ List<String> linesToAppend = editor.onComplete();
+ if (!linesToAppend.isEmpty()) {
+ modified = true;
+ newLines.addAll(linesToAppend);
+ maybeAdd(diff, linesToAppend);
+ }
+
+ if (!modified) {
+ return false;
+ }
+
+ String diffDescription = diffTooLarge() ? ": Diff too large (" + diffSize + ")" : ":\n" + diff;
+ logConsumer.accept("Patching file " + name + diffDescription);
+ consumer.accept(newLines);
+ return true;
+ }
+
+ public boolean converge(TaskContext context) {
+ return this.edit(line -> context.recordSystemModification(logger, line));
+ }
+
+ private void maybeAdd(StringBuilder diff, List<String> lines) {
+ for (String line : lines) {
+ // 2 for '+' and '\n'
+ diffSize += 2 + line.length();
+ if (!diffTooLarge()) {
+ diff.append('+').append(line).append('\n');
+ }
+ }
+ }
+
+ private void maybeRemove(StringBuilder diff, String line) {
+ // 2 for '-' and '\n'
+ diffSize += 2 + line.length();
+ if (!diffTooLarge()) {
+ diff.append('-').append(line).append('\n');
+ }
+ }
+
+ private boolean diffTooLarge() {
+ return diffSize > MAX_LENGTH;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java
new file mode 100644
index 00000000000..66f54fc1967
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.nio.file.Path;
+
+/**
+ * @author hakonhall
+ */
+public class EditorFactory {
+ public Editor create(Path path, LineEditor editor) {
+ return new Editor(path, editor);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java
new file mode 100644
index 00000000000..06490bac3a4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java
@@ -0,0 +1,37 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.nio.file.attribute.FileTime;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.time.Instant;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This wrapper around PosixFileAttributes.
+ *
+ * @author hakonhall
+ */
+public record FileAttributes(Instant lastModifiedTime, int ownerId, int groupId, String permissions,
+ boolean isRegularFile, boolean isDirectory, long size, int deviceMajor, int deviceMinor) {
+
+ @SuppressWarnings("unchecked")
+ static FileAttributes fromAttributes(Map<String, Object> attributes) {
+ long dev_t = (long) attributes.get("dev");
+
+ return new FileAttributes(
+ ((FileTime) attributes.get("lastModifiedTime")).toInstant(),
+ (int) attributes.get("uid"),
+ (int) attributes.get("gid"),
+ PosixFilePermissions.toString(((Set<PosixFilePermission>) attributes.get("permissions"))),
+ (boolean) attributes.get("isRegularFile"),
+ (boolean) attributes.get("isDirectory"),
+ (long) attributes.get("size"),
+ deviceMajor(dev_t), deviceMinor(dev_t));
+ }
+
+ // Encoded as MMMM Mmmm mmmM MMmm, where M is a hex digit of the major number and m is a hex digit of the minor number.
+ static int deviceMajor(long dev_t) { return (int) (((dev_t & 0xFFFFF00000000000L) >> 32) | ((dev_t & 0xFFF00) >> 8)); }
+ static int deviceMinor(long dev_t) { return (int) (((dev_t & 0x00000FFFFFF00000L) >> 12) | (dev_t & 0x000FF)); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java
new file mode 100644
index 00000000000..ca81669adcc
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java
@@ -0,0 +1,32 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.util.Optional;
+
+// @ThreadUnsafe
+public class FileAttributesCache {
+ private final UnixPath path;
+
+ private Optional<FileAttributes> attributes = Optional.empty();
+
+ public FileAttributesCache(UnixPath path) {
+ this.path = path;
+ }
+
+ public Optional<FileAttributes> get() {
+ if (attributes.isEmpty()) {
+ attributes = path.getAttributesIfExists();
+ }
+
+ return attributes;
+ }
+
+ public FileAttributes getOrThrow() {
+ return get().orElseThrow();
+ }
+
+ public Optional<FileAttributes> forceGet() {
+ attributes = Optional.empty();
+ return get();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java
new file mode 100644
index 00000000000..0a081ac53b4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java
@@ -0,0 +1,35 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.time.Instant;
+import java.util.Optional;
+
+/**
+ * Class to avoid repeated reads of file content when the file seldom changes.
+ *
+ * @author hakonhall
+ */
+class FileContentCache {
+ private final UnixPath path;
+
+ private Optional<byte[]> value = Optional.empty();
+ private Optional<Instant> modifiedTime = Optional.empty();
+
+ FileContentCache(UnixPath path) {
+ this.path = path;
+ }
+
+ byte[] get(Instant lastModifiedTime) {
+ if (modifiedTime.isEmpty() || lastModifiedTime.isAfter(modifiedTime.get())) {
+ value = Optional.of(path.readBytes());
+ modifiedTime = Optional.of(lastModifiedTime);
+ }
+
+ return value.get();
+ }
+
+ void updateWith(byte[] content, Instant modifiedTime) {
+ this.value = Optional.of(content);
+ this.modifiedTime = Optional.of(modifiedTime);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java
new file mode 100644
index 00000000000..a443e683df0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java
@@ -0,0 +1,34 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Deletes a file or empty directory.
+ *
+ * @author hakonhall
+ */
+public class FileDeleter {
+ private static final Logger logger = Logger.getLogger(FileDeleter.class.getName());
+
+ private final Path path;
+
+ public FileDeleter(Path path) {
+ this.path = path;
+ }
+
+ public boolean converge(TaskContext context) {
+ boolean deleted = uncheck(() -> Files.deleteIfExists(path));
+ if (deleted) {
+ context.recordSystemModification(logger, "Deleted " + path);
+ }
+
+ return deleted;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java
new file mode 100644
index 00000000000..1b3fa1854e7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java
@@ -0,0 +1,272 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.lang.MutableInteger;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+/**
+ * Helper class to find and list or deleteRecursively files and directories. Follows the general syntax of command line
+ * tool `find`.
+ *
+ * @author freva
+ */
+public class FileFinder {
+ private static final Logger logger = Logger.getLogger(FileFinder.class.getName());
+
+ private final Path basePath;
+ private final Set<Path> pruned = new HashSet<>();
+ private Predicate<FileAttributes> matcher;
+ private int maxDepth = Integer.MAX_VALUE;
+
+ private FileFinder(Path basePath, Predicate<FileAttributes> initialMatcher) {
+ this.basePath = basePath;
+ this.matcher = initialMatcher;
+ }
+
+ /** Creates a FileFinder at the given basePath */
+ public static FileFinder from(Path basePath) {
+ return new FileFinder(basePath, attrs -> true);
+ }
+
+ /** Creates a FileFinder at the given basePath that will match all files */
+ public static FileFinder files(Path basePath) {
+ return new FileFinder(basePath, FileAttributes::isRegularFile);
+ }
+
+
+ /** Creates a FileFinder at the given basePath that will match all directories */
+ public static FileFinder directories(Path basePath) {
+ return new FileFinder(basePath, FileAttributes::isDirectory);
+ }
+
+
+ /**
+ * Predicate that will be used to match files and directories under the base path.
+ *
+ * NOTE: Consecutive calls to this method are ANDed (this include the initial filter from
+ * {@link #files(Path)} or {@link #directories(Path)}.
+ */
+ public FileFinder match(Predicate<FileAttributes> matcher) {
+ this.matcher = this.matcher.and(matcher);
+ return this;
+ }
+
+ /**
+ * Path for which whole directory tree will be skipped, including the path itself.
+ * The path must be under {@code basePath} or be relative to {@code basePath}.
+ */
+ public FileFinder prune(Path path) {
+ if (!path.isAbsolute())
+ path = basePath.resolve(path);
+
+ if (!path.startsWith(basePath))
+ throw new IllegalArgumentException("Prune path " + path + " is not under base path " + basePath);
+
+ this.pruned.add(path);
+ return this;
+ }
+
+ /** Convenience method for pruning multiple paths, see {@link #prune(Path)}. */
+ public FileFinder prune(Collection<Path> paths) {
+ paths.forEach(this::prune);
+ return this;
+ }
+
+ /**
+ * Maximum depth (relative to basePath) where contents should be matched with the given filters.
+ * Default is unlimited.
+ */
+ public FileFinder maxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ return this;
+ }
+
+ /**
+ * Recursively deletes all matching elements
+ *
+ * @return true iff anything was matched and deleted
+ */
+ public boolean deleteRecursively(TaskContext context) {
+ final int maxNumberOfDeletedPathsToLog = 20;
+ MutableInteger numDeleted = new MutableInteger(0);
+ List<Path> deletedPaths = new ArrayList<>();
+
+ try {
+ forEach(attributes -> {
+ if (attributes.unixPath().deleteRecursively()) {
+ if (numDeleted.next() <= maxNumberOfDeletedPathsToLog) deletedPaths.add(attributes.path());
+ }
+ });
+ } finally {
+ if (numDeleted.get() > maxNumberOfDeletedPathsToLog) {
+ context.log(logger, "Deleted " + numDeleted.get() + " paths under " + basePath);
+ } else if (deletedPaths.size() > 0) {
+ List<Path> paths = deletedPaths.stream()
+ .map(basePath::relativize)
+ .sorted()
+ .toList();
+ context.log(logger, "Deleted these paths in " + basePath + ": " + paths);
+ }
+ }
+
+ return deletedPaths.size() > 0;
+ }
+
+ public List<FileAttributes> list() {
+ LinkedList<FileAttributes> list = new LinkedList<>();
+ forEach(list::add);
+ return list;
+ }
+
+ public Stream<FileAttributes> stream() {
+ return list().stream();
+ }
+
+ public void forEachPath(Consumer<Path> action) {
+ forEach(attributes -> action.accept(attributes.path()));
+ }
+
+ /** Applies a given consumer to all the matching {@link FileFinder.FileAttributes} */
+ public void forEach(Consumer<FileAttributes> action) {
+ applyForEachToMatching(basePath, matcher, maxDepth, action);
+ }
+
+
+ /**
+ * <p> This method walks a file tree rooted at a given starting file. The file tree traversal is
+ * <em>depth-first</em>: The filter function is applied in pre-order (NLR), but the given
+ * {@link Consumer} will be called in post-order (LRN).
+ */
+ private void applyForEachToMatching(Path basePath, Predicate<FileAttributes> matcher,
+ int maxDepth, Consumer<FileAttributes> action) {
+ try {
+ // Only need to traverse as deep as we want to match, unless we want to match everything in directories
+ // already matched
+ Files.walkFileTree(basePath, Set.of(), maxDepth, new SimpleFileVisitor<>() {
+ private final Deque<FileAttributes> matchingDirectoryStack = new ArrayDeque<>();
+ private int currentLevel = -1;
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+ if (pruned.contains(dir)) return FileVisitResult.SKIP_SUBTREE;
+
+ currentLevel++;
+
+ FileAttributes attributes = new FileAttributes(dir, attrs);
+ if (currentLevel > 0 && matcher.test(attributes))
+ matchingDirectoryStack.push(attributes);
+
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+ // When we find a directory at the max depth given to Files.walkFileTree, the directory
+ // will be passed to visitFile() rather than (pre|post)VisitDirectory
+ if (attrs.isDirectory()) {
+ preVisitDirectory(file, attrs);
+ return postVisitDirectory(file, null);
+ }
+
+ FileAttributes attributes = new FileAttributes(file, attrs);
+ if (matcher.test(attributes))
+ action.accept(attributes);
+
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+ if (!matchingDirectoryStack.isEmpty())
+ action.accept(matchingDirectoryStack.pop());
+
+ currentLevel--;
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ } catch (NoSuchFileException ignored) {
+
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+
+ // Ideally, we would reuse the FileAttributes in this package, but unfortunately we only get
+ // BasicFileAttributes and not PosixFileAttributes from FileVisitor
+ public static class FileAttributes {
+ private final Path path;
+ private final BasicFileAttributes attributes;
+
+ public FileAttributes(Path path, BasicFileAttributes attributes) {
+ this.path = path;
+ this.attributes = attributes;
+ }
+
+ public Path path() { return path; }
+ public UnixPath unixPath() { return new UnixPath(path); }
+ public String filename() { return path.getFileName().toString(); }
+ public Instant lastModifiedTime() { return attributes.lastModifiedTime().toInstant(); }
+ public boolean isRegularFile() { return attributes.isRegularFile(); }
+ public boolean isDirectory() { return attributes.isDirectory(); }
+ public long size() { return attributes.size(); }
+ }
+
+
+ // Filters
+ public static Predicate<FileAttributes> olderThan(Duration duration) {
+ return attrs -> Duration.between(attrs.lastModifiedTime(), Instant.now()).compareTo(duration) > 0;
+ }
+
+ public static Predicate<FileAttributes> youngerThan(Duration duration) {
+ return olderThan(duration).negate();
+ }
+
+ public static Predicate<FileAttributes> largerThan(long sizeInBytes) {
+ return attrs -> attrs.size() > sizeInBytes;
+ }
+
+ public static Predicate<FileAttributes> smallerThan(long sizeInBytes) {
+ return largerThan(sizeInBytes).negate();
+ }
+
+ public static Predicate<FileAttributes> nameMatches(Pattern pattern) {
+ return attrs -> pattern.matcher(attrs.filename()).matches();
+ }
+
+ public static Predicate<FileAttributes> nameStartsWith(String string) {
+ return attrs -> attrs.filename().startsWith(string);
+ }
+
+ public static Predicate<FileAttributes> nameEndsWith(String string) {
+ return attrs -> attrs.filename().endsWith(string);
+ }
+
+ public static Predicate<FileAttributes> all() {
+ return attrs -> true;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java
new file mode 100644
index 00000000000..3c53609b84e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java
@@ -0,0 +1,55 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.CopyOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Utility for idempotent move of (any type of) file.
+ *
+ * @author hakonhall
+ */
+public class FileMover {
+ private static final Logger logger = Logger.getLogger(FileMover.class.getName());
+
+ private final Path source;
+ private final Path destination;
+ private final Set<CopyOption> moveOptions = new HashSet<>();
+
+ public FileMover(Path source, Path destination) {
+ this.source = source;
+ this.destination = destination;
+ }
+
+ public FileMover replaceExisting() {
+ moveOptions.add(StandardCopyOption.REPLACE_EXISTING);
+ return this;
+ }
+
+ public FileMover atomic() {
+ moveOptions.add(StandardCopyOption.ATOMIC_MOVE);
+ return this;
+ }
+
+ /**
+ * Move file.
+ *
+ * @return false if the source doesn't exist while the destination do.
+ * @see Files#move(Path, Path, CopyOption...) Files.move()
+ */
+ public boolean converge(TaskContext context) {
+ if (!Files.exists(source) && Files.exists(destination)) return false;
+ uncheck(() -> Files.move(source, destination, moveOptions.toArray(CopyOption[]::new)));
+ context.recordSystemModification(logger, "Moved " + source + " to " + destination);
+ return true;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java
new file mode 100644
index 00000000000..b466b878ce5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java
@@ -0,0 +1,83 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Optional;
+
+/**
+ * A snapshot of the attributes of the file for a given path, and file content if it is a regular file.
+ *
+ * @author hakonhall
+ */
+public class FileSnapshot {
+ private final Path path;
+ private final Optional<FileAttributes> attributes;
+ private final Optional<byte[]> content;
+
+ public static FileSnapshot forPath(Path path) { return forNonExistingFile(path).snapshot(); }
+
+ /** Guaranteed to not throw any exceptions. */
+ public static FileSnapshot forNonExistingFile(Path path) {
+ return new FileSnapshot(path, Optional.empty(), Optional.empty());
+ }
+
+ private static FileSnapshot forRegularFile(Path path, FileAttributes attributes, byte[] content) {
+ if (!attributes.isRegularFile()) throw new IllegalArgumentException(path + " is not a regular file");
+ return new FileSnapshot(path, Optional.of(attributes), Optional.of(content));
+ }
+
+ private static FileSnapshot forOtherFile(Path path, FileAttributes attributes) {
+ if (attributes.isRegularFile()) throw new IllegalArgumentException(path + " is a regular file");
+ return new FileSnapshot(path, Optional.of(attributes), Optional.empty());
+ }
+
+ private FileSnapshot(Path path, Optional<FileAttributes> attributes, Optional<byte[]> content) {
+ this.path = path;
+ this.attributes = attributes;
+ this.content = content;
+ }
+
+ public Path path() { return path; }
+
+ /** Whether there was a file (or directory) at path. */
+ public boolean exists() { return attributes.isPresent(); }
+
+ /** Returns the file attributes if the file exists. */
+ public Optional<FileAttributes> attributes() { return attributes; }
+
+ /** Returns the file content if the file exists and is a regular file. */
+ public Optional<byte[]> content() { return content; }
+
+ /** Returns the file UTF-8 content if it exists and is a regular file. */
+ public Optional<String> utf8Content() { return content.map(c -> new String(c, StandardCharsets.UTF_8)); }
+
+ /** Returns an up-to-date snapshot of the path, possibly {@code this} if last modified time has not changed. */
+ public FileSnapshot snapshot() {
+ Optional<FileAttributes> currentAttributes = new UnixPath(path).getAttributesIfExists();
+ if (currentAttributes.isPresent()) {
+
+ // 'this' may still be valid, depending on last modified times.
+ if (attributes.isPresent()) {
+ Instant previousModifiedTime = attributes.get().lastModifiedTime();
+ Instant currentModifiedTime = currentAttributes.get().lastModifiedTime();
+ if (currentModifiedTime.compareTo(previousModifiedTime) <= 0) {
+ return this;
+ }
+ }
+
+ if (currentAttributes.get().isRegularFile()) {
+ Optional<byte[]> content = IOExceptionUtil.ifExists(() -> Files.readAllBytes(path));
+ return content.map(bytes -> FileSnapshot.forRegularFile(path, currentAttributes.get(), bytes))
+ // File was removed after getting attributes and before getting content.
+ .orElseGet(() -> FileSnapshot.forNonExistingFile(path));
+ } else {
+ return FileSnapshot.forOtherFile(path, currentAttributes.get());
+ }
+ } else {
+ return attributes.isPresent() ? FileSnapshot.forNonExistingFile(path) : this /* avoid allocation */;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java
new file mode 100644
index 00000000000..bc572ce82a9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java
@@ -0,0 +1,107 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Class to minimize resource usage with repetitive and mostly identical, idempotent, and
+ * mutating file operations, e.g. setting file content, setting owner, etc.
+ *
+ * Only changes to the file is logged.
+ *
+ * @author hakohall
+ */
+// @ThreadUnsafe
+public class FileSync {
+ private static final Logger logger = Logger.getLogger(FileSync.class.getName());
+
+ private final UnixPath path;
+ private final FileContentCache contentCache;
+ private final FileAttributesCache attributesCache;
+
+ public FileSync(Path path) {
+ this.path = new UnixPath(path);
+ this.contentCache = new FileContentCache(this.path);
+ this.attributesCache = new FileAttributesCache(this.path);
+ }
+
+ public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) {
+ return convergeTo(taskContext, partialFileData, false);
+ }
+
+ /**
+ * CPU, I/O, and memory usage is optimized for repeated calls with the same arguments.
+ *
+ * @param atomicWrite Whether to write updates to a temporary file in the same directory, and atomically move it
+ * to path. Ensures the file cannot be read while in the middle of writing it.
+ * @return true if the system was modified: content was written, or owner was set, etc.
+ * system is only modified if necessary (different).
+ */
+ public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData, boolean atomicWrite) {
+ boolean modifiedSystem = false;
+
+ if (partialFileData.getContent().isPresent()) {
+ modifiedSystem |= convergeTo(taskContext, partialFileData.getContent().get(), atomicWrite, partialFileData.getPermissions());
+ }
+
+ AttributeSync attributeSync = new AttributeSync(path.toPath()).with(partialFileData);
+ modifiedSystem |= attributeSync.converge(taskContext, this.attributesCache);
+
+ return modifiedSystem;
+ }
+
+ /**
+ * CPU, I/O, and memory usage is optimized for repeated calls with the same argument.
+ *
+ * @param atomicWrite Whether to write updates to a temporary file in the same directory, and atomically move it
+ * to path. Ensures the file cannot be read while in the middle of writing it.
+ * @param permissions Permissions if the file is created.
+ * @return true if the content was written. Only modified if necessary (different).
+ */
+ public boolean convergeTo(TaskContext taskContext, byte[] content, boolean atomicWrite, Optional<String> permissions) {
+ Optional<Instant> lastModifiedTime = attributesCache.forceGet().map(FileAttributes::lastModifiedTime);
+
+ if (lastModifiedTime.isEmpty()) {
+ taskContext.recordSystemModification(logger, "Creating file " + path +
+ permissions.map(p -> " with permissions " + p).orElse(""));
+ path.createParents();
+ writeBytes(content, atomicWrite, permissions);
+ contentCache.updateWith(content, attributesCache.forceGet().orElseThrow().lastModifiedTime());
+ return true;
+ }
+
+ if (Arrays.equals(content, contentCache.get(attributesCache.getOrThrow().lastModifiedTime()))) {
+ return false;
+ } else {
+ taskContext.recordSystemModification(logger, "Patching file " + path);
+ // empty permissions here, because the file already exists and won't be applied anyway
+ writeBytes(content, atomicWrite, Optional.empty());
+ contentCache.updateWith(content, attributesCache.forceGet().orElseThrow().lastModifiedTime());
+ return true;
+ }
+ }
+
+ private void writeBytes(byte[] content, boolean atomic, Optional<String> permissions) {
+ if (atomic) {
+ UnixPath tmpPath = new UnixPath(path.toPath().getFileSystem().getPath(path.toPath().toString() + ".FileSyncTmp"));
+ if (permissions.isPresent()) {
+ tmpPath.writeBytes(content, permissions.get());
+ } else {
+ tmpPath.writeBytes(content);
+ }
+ tmpPath.atomicMove(path.toPath());
+ } else {
+ if (permissions.isPresent()) {
+ path.writeBytes(content, permissions.get());
+ } else {
+ path.writeBytes(content);
+ }
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java
new file mode 100644
index 00000000000..aa6364f2a98
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java
@@ -0,0 +1,92 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+/**
+ * Write a file
+ *
+ * @author hakonhall
+ */
+public class FileWriter {
+ private final Path path;
+ private final FileSync fileSync;
+ private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder();
+ private final Optional<ByteArraySupplier> contentProducer;
+
+ private boolean atomicWrite = false;
+ private boolean overwriteExistingFile = true;
+
+ public FileWriter(Path path) {
+ this(path, Optional.empty());
+ }
+
+ public FileWriter(Path path, Supplier<String> contentProducer) {
+ this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8));
+ }
+
+ public FileWriter(Path path, ByteArraySupplier contentProducer) {
+ this(path, Optional.of(contentProducer));
+ }
+
+ private FileWriter(Path path, Optional<ByteArraySupplier> contentProducer) {
+ this.path = path;
+ this.fileSync = new FileSync(path);
+ this.contentProducer = contentProducer;
+ }
+
+ public Path path() { return path; }
+
+ public FileWriter withOwnerId(int ownerId) {
+ fileDataBuilder.withOwnerId(ownerId);
+ return this;
+ }
+
+ public FileWriter withGroupId(int groupId) {
+ fileDataBuilder.withGroupId(groupId);
+ return this;
+ }
+
+ /** @see UnixPath#setPermissions */
+ public FileWriter withPermissions(String permissions) {
+ fileDataBuilder.withPermissions(permissions);
+ return this;
+ }
+
+ public FileWriter atomicWrite(boolean atomicWrite) {
+ this.atomicWrite = atomicWrite;
+ return this;
+ }
+
+ public FileWriter onlyIfFileDoesNotAlreadyExist() {
+ overwriteExistingFile = false;
+ return this;
+ }
+
+ public boolean converge(TaskContext context) {
+ return converge(context, contentProducer.orElseThrow().get());
+ }
+
+ public boolean converge(TaskContext context, String utf8Content) {
+ return converge(context, utf8Content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ public boolean converge(TaskContext context, byte[] content) {
+ if (!overwriteExistingFile && Files.isRegularFile(path)) {
+ return false;
+ }
+
+ fileDataBuilder.withContent(content);
+ PartialFileData fileData = fileDataBuilder.create();
+ return fileSync.convergeTo(context, fileData, atomicWrite);
+ }
+
+ @FunctionalInterface
+ public interface ByteArraySupplier extends Supplier<byte[]> { }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java
new file mode 100644
index 00000000000..a0db5a3cb16
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java
@@ -0,0 +1,33 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.yolean.Exceptions;
+
+import java.io.UncheckedIOException;
+import java.nio.file.NoSuchFileException;
+import java.util.Optional;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Utils related to IOException.
+ *
+ * @author hakonhall
+ */
+public class IOExceptionUtil {
+ /**
+ * Useful if it's not known whether a file or directory exists, in case e.g.
+ * NoSuchFileException is thrown and the caller wants an Optional.empty() in that case.
+ */
+ public static <T> Optional<T> ifExists(Exceptions.SupplierThrowingIOException<T> supplier) {
+ try {
+ return Optional.ofNullable(uncheck(supplier));
+ } catch (UncheckedIOException e) {
+ if (e.getCause() instanceof NoSuchFileException) {
+ return Optional.empty();
+ }
+
+ throw e;
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java
new file mode 100644
index 00000000000..88b403ba443
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java
@@ -0,0 +1,40 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.REPLACE;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.NONE;
+
+/**
+ * @author hakonhall
+ */
+public class LineEdit {
+ enum Type { NONE, REPLACE }
+
+ public static LineEdit none() { return insert(List.of(), List.of()); }
+ public static LineEdit remove() { return replaceWith(List.of()); }
+
+ public static LineEdit insertBefore(String... prepend) { return insertBefore(List.of(prepend)); }
+ public static LineEdit insertBefore(List<String> prepend) { return insert(prepend, List.of()); }
+ public static LineEdit insertAfter(String... append) { return insertAfter(List.of(append)); }
+ public static LineEdit insertAfter(List<String> append) { return insert(List.of(), append); }
+ public static LineEdit insert(List<String> prepend, List<String> append) { return new LineEdit(NONE, prepend, append); }
+
+ public static LineEdit replaceWith(String... lines) { return replaceWith(List.of(lines)); }
+ public static LineEdit replaceWith(List<String> insertLines) { return new LineEdit(REPLACE, List.of(), insertLines); }
+
+ private final Type type;
+ private final List<String> prependLines;
+ private final List<String> appendLines;
+
+ private LineEdit(Type type, List<String> prependLines, List<String> appendLines) {
+ this.type = type;
+ this.prependLines = List.copyOf(prependLines);
+ this.appendLines = List.copyOf(appendLines);
+ }
+
+ public Type getType() { return type; }
+ public List<String> prependLines() { return prependLines; }
+ public List<String> appendLines() { return appendLines; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java
new file mode 100644
index 00000000000..a7dcb4dd32a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java
@@ -0,0 +1,21 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.util.List;
+
+/**
+ * @author hakonhall
+ */
+public interface LineEditor {
+ /**
+ * @param line The line of a file.
+ * @return The edited line, or empty if the line should be removed.
+ */
+ LineEdit edit(String line);
+
+ /**
+ * Called after edit() has been called on all lines in the file.
+ * @return Lines to append to the file.
+ */
+ List<String> onComplete();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java
new file mode 100644
index 00000000000..24c2ae8543d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java
@@ -0,0 +1,70 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.io.UncheckedIOException;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Class to ensure a directory exists with the correct owner, group, and permissions.
+ *
+ * @author hakonhall
+ */
+public class MakeDirectory {
+ private static final Logger logger = Logger.getLogger(MakeDirectory.class.getName());
+
+ private final UnixPath path;
+ private final AttributeSync attributeSync;
+ private final FileAttributesCache attributesCache;
+
+ private boolean createParents = false;
+
+ public MakeDirectory(Path path) {
+ this.path = new UnixPath(path);
+ this.attributeSync = new AttributeSync(path);
+ this.attributesCache = new FileAttributesCache(this.path);
+ }
+
+ /**
+ * Warning: The owner, group, and permissions of any created parent directories are NOT modified
+ */
+ public MakeDirectory createParents() { this.createParents = true; return this; }
+
+ public MakeDirectory withOwnerId(int ownerId) { attributeSync.withOwnerId(ownerId); return this; }
+ public MakeDirectory withGroupId(int groupId) { attributeSync.withGroupId(groupId); return this; }
+ public MakeDirectory withPermissions(String permissions) {
+ attributeSync.withPermissions(permissions);
+ return this;
+ }
+
+ public boolean converge(TaskContext context) {
+ boolean systemModified = false;
+
+ Optional<FileAttributes> attributes = attributesCache.forceGet();
+ if (attributes.isPresent()) {
+ if (!attributes.get().isDirectory()) {
+ throw new UncheckedIOException(new NotDirectoryException(path.toString()));
+ }
+ } else {
+ Optional<String> permissions = attributeSync.getPermissions();
+ if (createParents) {
+ // We'll skip logging system modification here, as we'll log about the creation
+ // of the directory next.
+ permissions.ifPresentOrElse(path::createParents, path::createParents);
+ }
+
+ context.recordSystemModification(logger, "Creating directory " + path);
+ systemModified = true;
+
+ permissions.ifPresentOrElse(path::createDirectory, path::createDirectory);
+ }
+
+ systemModified |= attributeSync.converge(context, attributesCache);
+
+ return systemModified;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java
new file mode 100644
index 00000000000..b1d56b131bb
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java
@@ -0,0 +1,67 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+/**
+ * Represents a subset of a file's content, owner, group, and permissions.
+ *
+ * @author hakonhall
+ */
+// @Immutable
+public class PartialFileData {
+ private final Optional<byte[]> content;
+ private final Optional<Integer> ownerId;
+ private final Optional<Integer> groupId;
+ private final Optional<String> permissions;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private PartialFileData(Optional<byte[]> content,
+ Optional<Integer> ownerId,
+ Optional<Integer> groupId,
+ Optional<String> permissions) {
+ this.content = content;
+ this.ownerId = ownerId;
+ this.groupId = groupId;
+ this.permissions = permissions;
+ }
+
+ public Optional<byte[]> getContent() {
+ return content;
+ }
+
+ public Optional<Integer> getOwnerId() {
+ return ownerId;
+ }
+
+ public Optional<Integer> getGroupId() {
+ return groupId;
+ }
+
+ public Optional<String> getPermissions() {
+ return permissions;
+ }
+
+ public static class Builder {
+ private Optional<byte[]> content = Optional.empty();
+ private Optional<Integer> ownerId = Optional.empty();
+ private Optional<Integer> groupId = Optional.empty();
+ private Optional<String> permissions = Optional.empty();
+
+ public Builder withContent(byte[] content) { this.content = Optional.of(content); return this; }
+ public Builder withContent(String content, Charset charset) { return withContent(content.getBytes(charset)); }
+ public Builder withContent(String content) { return withContent(content, StandardCharsets.UTF_8); }
+ public Builder withOwnerId(int ownerId) { this.ownerId = Optional.of(ownerId); return this; }
+ public Builder withGroupId(int groupId) { this.groupId = Optional.of(groupId); return this; }
+ public Builder withPermissions(String permissions) { this.permissions = Optional.of(permissions); return this; }
+
+ public PartialFileData create() {
+ return new PartialFileData(content, ownerId, groupId, permissions);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java
new file mode 100644
index 00000000000..50ca5db9d3d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java
@@ -0,0 +1,53 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+/**
+ * Class wrapping a boolean stored on disk.
+ *
+ * <p>The implementation is compatible with {@link StoredInteger} when absence or 0 means false.
+ *
+ * @author hakonhall
+ */
+public class StoredBoolean {
+ private static final Logger logger = Logger.getLogger(StoredBoolean.class.getName());
+
+ private final UnixPath path;
+
+ /** The parent directory must exist. Value is false by default. */
+ public StoredBoolean(Path path) {
+ this.path = new UnixPath(path);
+ }
+
+ public boolean value() {
+ return path.readUtf8FileIfExists().map(String::trim).map(s -> !"0".equals(s)).orElse(false);
+ }
+
+ /** Sets value to true. */
+ public void set(TaskContext context) {
+ if (!value()) {
+ context.log(logger, "Writes " + path);
+ path.writeUtf8File("1");
+ }
+ }
+
+ public void set(TaskContext context, boolean value) {
+ if (value) {
+ set(context);
+ } else {
+ clear(context);
+ }
+ }
+
+ /** Sets value to false. */
+ public void clear(TaskContext context) {
+ if (value()) {
+ context.log(logger, "Deletes " + path);
+ path.deleteIfExists();
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java
new file mode 100644
index 00000000000..19e8bcfcf93
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java
@@ -0,0 +1,46 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+/**
+ * Class wrapping a float stored on disk
+ *
+ * @author freva
+ */
+public class StoredDouble implements Supplier<OptionalDouble> {
+
+ private static final Logger logger = Logger.getLogger(StoredDouble.class.getName());
+
+ private final UnixPath path;
+
+ public StoredDouble(Path path) {
+ this.path = new UnixPath(path);
+ }
+
+ @Override
+ public OptionalDouble get() {
+ return path.readUtf8FileIfExists().stream().mapToDouble(Double::parseDouble).findAny();
+ }
+
+ public void write(TaskContext taskContext, double value) {
+ path.writeUtf8File(Double.toString(value));
+ taskContext.log(logger, "Stored new double in %s: %f", path, value);
+ }
+
+ public void clear() {
+ path.deleteIfExists();
+ }
+
+ public Optional<Instant> getLastModifiedTime() {
+ return path.getAttributesIfExists().map(FileAttributes::lastModifiedTime);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
new file mode 100644
index 00000000000..ec4d64db0e3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
@@ -0,0 +1,42 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+/**
+ * Class wrapping an integer stored on disk
+ *
+ * @author freva
+ */
+public class StoredInteger implements Supplier<OptionalInt> {
+
+ private static final Logger logger = Logger.getLogger(StoredInteger.class.getName());
+
+ private final UnixPath path;
+
+ public StoredInteger(Path path) {
+ this.path = new UnixPath(path);
+ }
+
+ @Override
+ public OptionalInt get() {
+ return path.readUtf8FileIfExists().stream().mapToInt(Integer::parseInt).findAny();
+ }
+
+ public void write(TaskContext taskContext, int value) {
+ path.writeUtf8File(Integer.toString(value));
+ taskContext.log(logger, "Stored new integer in %s: %d", path, value);
+ }
+
+ public Optional<Instant> getLastModifiedTime() {
+ return path.getAttributesIfExists().map(FileAttributes::lastModifiedTime);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java
new file mode 100644
index 00000000000..2436ba306ac
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.slf4j.helpers.NOPLogger;
+
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Uses the Velocity engine to render a template, to and from both String and Path objects.
+ *
+ * @author hakonhall
+ * @author jonmv
+ */
+public class Template {
+
+ static {
+ Velocity.addProperty(RuntimeConstants.RUNTIME_LOG_INSTANCE, NOPLogger.NOP_LOGGER);
+ Velocity.init();
+ }
+
+ private final VelocityContext velocityContext = new VelocityContext();
+ private final String template;
+
+ private Template(String template) {
+ this.template = template;
+ }
+
+ public static Template at(Path templatePath) {
+ return of(uncheck(() -> new String(Files.readAllBytes(templatePath))));
+ }
+
+ public static Template of(String template) {
+ return new Template(template);
+ }
+
+ public Template set(String name, Object value) {
+ velocityContext.put(name, value);
+ return this;
+ }
+
+ public FileWriter getFileWriterTo(Path destinationPath) {
+ return new FileWriter(destinationPath, this::render);
+ }
+
+ public String render() {
+ StringWriter writer = new StringWriter();
+ Velocity.evaluate(velocityContext, writer, "Template", template);
+ return writer.toString();
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
new file mode 100644
index 00000000000..1983e94e6f5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
@@ -0,0 +1,350 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.FileTime;
+import java.nio.file.attribute.GroupPrincipal;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.nio.file.attribute.UserPrincipal;
+import java.nio.file.attribute.UserPrincipalLookupService;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Thin wrapper around java.nio.file.Path, especially nice for UNIX-specific features.
+ *
+ * @author hakonhall
+ */
+// @Immutable
+public class UnixPath {
+
+ private static final Set<OpenOption> DEFAULT_OPEN_OPTIONS =
+ Set.of(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
+
+ private final Path path;
+
+ public UnixPath(Path path) { this.path = path; }
+ public UnixPath(String path) { this(Path.of(path)); }
+
+ public Path toPath() { return path; }
+ public UnixPath resolve(String relativeOrAbsolutePath) { return new UnixPath(path.resolve(relativeOrAbsolutePath)); }
+
+ public UnixPath getParent() {
+ Path parentPath = path.getParent();
+ if (parentPath == null) {
+ throw new IllegalStateException("Path has no parent directory: '" + path + "'");
+ }
+
+ return new UnixPath(parentPath);
+ }
+
+ public String getFilename() {
+ Path filename = path.getFileName();
+ if (filename == null) {
+ // E.g. "/".
+ throw new IllegalStateException("Path has no filename: '" + path + "'");
+ }
+
+ return filename.toString();
+ }
+
+ public boolean exists() { return Files.exists(path); }
+
+ public String readUtf8File() {
+ return new String(readBytes(), StandardCharsets.UTF_8);
+ }
+
+ public Optional<String> readUtf8FileIfExists() {
+ try {
+ return Optional.of(Files.readString(path));
+ } catch (NoSuchFileException ignored) {
+ return Optional.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public byte[] readBytes() {
+ return uncheck(() -> Files.readAllBytes(path));
+ }
+
+ /** Reads and returns all bytes contained in this path, if any such path exists. */
+ public Optional<byte[]> readBytesIfExists() {
+ try {
+ return Optional.of(Files.readAllBytes(path));
+ } catch (NoSuchFileException ignored) {
+ return Optional.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public List<String> readLines() {
+ return uncheck(() -> Files.readAllLines(path));
+ }
+
+ /** Create an empty file and return true, or false if the file already exists (the file may not be regular). */
+ public boolean create() {
+ try {
+ Files.createFile(path);
+ return true;
+ } catch (FileAlreadyExistsException ignored) {
+ return false;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public UnixPath writeUtf8File(String content, OpenOption... options) {
+ return writeBytes(content.getBytes(StandardCharsets.UTF_8), options);
+ }
+
+ public UnixPath writeUtf8File(String content, String permissions, OpenOption... options) {
+ return writeBytes(content.getBytes(StandardCharsets.UTF_8), permissions, options);
+ }
+
+ public UnixPath writeBytes(byte[] content, OpenOption... options) {
+ return writeBytes(content, null, options);
+ }
+
+ public UnixPath writeBytes(byte[] content, String permissions, OpenOption... options) {
+ FileAttribute<?>[] attributes = Optional.ofNullable(permissions)
+ .map(this::permissionsAsFileAttributes)
+ .orElseGet(() -> new FileAttribute<?>[0]);
+
+ Set<OpenOption> optionsSet = options.length == 0 ? DEFAULT_OPEN_OPTIONS : Set.of(options);
+
+ try (SeekableByteChannel channel = Files.newByteChannel(path, optionsSet, attributes)) {
+ channel.write(ByteBuffer.wrap(content));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return this;
+ }
+
+ /** Write a file to the same dir as this, and then atomically move it to this' path. */
+ public UnixPath atomicWriteBytes(byte[] content) {
+ UnixPath temporaryPath = getParent().resolve(getFilename() + ".10Ia2f4N5");
+ temporaryPath.writeBytes(content);
+ temporaryPath.atomicMove(path);
+ return this;
+ }
+
+ public String getPermissions() {
+ return getAttributes().permissions();
+ }
+
+ /**
+ * @param permissions Example: "rwxr-x---" means rwx for owner, rx for group,
+ * and no permissions for others.
+ */
+ public UnixPath setPermissions(String permissions) {
+ Set<PosixFilePermission> permissionSet = getPosixFilePermissionsFromString(permissions);
+ uncheck(() -> Files.setPosixFilePermissions(path, permissionSet));
+ return this;
+ }
+
+ public int getOwnerId() {
+ return getAttributes().ownerId();
+ }
+
+ public UnixPath setOwner(String user) { return setOwner(user, "user"); }
+ public UnixPath setOwnerId(int uid) { return setOwner(String.valueOf(uid), "uid"); }
+ private UnixPath setOwner(String owner, String type) {
+ UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService();
+ UserPrincipal principal = uncheck(
+ () -> service.lookupPrincipalByName(owner),
+ "While looking up %s %s", type, owner);
+ uncheck(() -> Files.setOwner(path, principal));
+ return this;
+ }
+
+ public int getGroupId() {
+ return getAttributes().groupId();
+ }
+
+ public UnixPath setGroup(String group) { return setGroup(group, "group"); }
+ public UnixPath setGroupId(int gid) { return setGroup(String.valueOf(gid), "gid"); }
+ private UnixPath setGroup(String group, String type) {
+ UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService();
+ GroupPrincipal principal = uncheck(
+ () -> service.lookupPrincipalByGroupName(group),
+ "While looking up group %s %s", type, group);
+ uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).setGroup(principal));
+ return this;
+ }
+
+ public Instant getLastModifiedTime() {
+ return getAttributes().lastModifiedTime();
+ }
+
+ public UnixPath updateLastModifiedTime() {
+ return setLastModifiedTime(Instant.now());
+ }
+
+ public UnixPath setLastModifiedTime(Instant instant) {
+ uncheck(() -> Files.setLastModifiedTime(path, FileTime.from(instant)));
+ return this;
+ }
+
+ public FileAttributes getAttributes() {
+ return uncheck(() -> FileAttributes.fromAttributes(Files.readAttributes(path, "unix:*")));
+ }
+
+ public Optional<FileAttributes> getAttributesIfExists() {
+ return ifExists(this::getAttributes);
+ }
+
+ public UnixPath createNewFile(String... permissions) {
+ uncheck(() -> Files.createFile(path, permissionsAsFileAttributes(permissions)));
+ return this;
+ }
+
+ public UnixPath createParents(String... permissions) {
+ getParent().createDirectories(permissions);
+ return this;
+ }
+
+ /** Create directory with given permissions and return true, or false if it already exists. */
+ public boolean createDirectory(String... permissions) {
+ try {
+ Files.createDirectory(path, permissionsAsFileAttributes(permissions));
+ } catch (FileAlreadyExistsException ignore) {
+ return false;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return true;
+ }
+
+ public UnixPath createDirectories(String... permissions) {
+ uncheck(() -> Files.createDirectories(path, permissionsAsFileAttributes(permissions)));
+ return this;
+ }
+
+ /**
+ * Returns whether this path is a directory. Symlinks are followed, so this returns true for symlinks pointing to a
+ * directory.
+ */
+ public boolean isDirectory() {
+ return uncheck(() -> Files.isDirectory(path));
+ }
+
+ /** Returns whether this is a symlink */
+ public boolean isSymbolicLink() {
+ return Files.isSymbolicLink(path);
+ }
+
+ /**
+ * Similar to rm -rf file:
+ * - It's not an error if file doesn't exist
+ * - If file is a directory, it and all content is removed
+ * - For symlinks: Only the symlink is removed, not what the symlink points to
+ */
+ public boolean deleteRecursively() {
+ if (!isSymbolicLink() && isDirectory()) {
+ try (Stream<UnixPath> paths = listContentsOfDirectory()) {
+ paths.forEach(UnixPath::deleteRecursively);
+ }
+ }
+ return uncheck(() -> Files.deleteIfExists(path));
+ }
+
+ public boolean deleteIfExists() {
+ return uncheck(() -> Files.deleteIfExists(path));
+ }
+
+ /** @return false path does not exist, is not a directory, or has at least one entry. */
+ public boolean isEmptyDirectory() {
+ try (var entryStream = Files.list(path)) {
+ return entryStream.findAny().isEmpty();
+ } catch (NotDirectoryException | NoSuchFileException e) {
+ return false;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /** Lists the contents of this as a stream. Callers should use try-with to ensure that the stream is closed */
+ public Stream<UnixPath> listContentsOfDirectory() {
+ try {
+ // Avoid the temptation to collect the stream here as collecting a directory with a high number of entries
+ // can quickly lead to out of memory conditions
+ return Files.list(path).map(UnixPath::new);
+ } catch (NoSuchFileException ignored) {
+ return Stream.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to list contents of directory " + path.toAbsolutePath(), e);
+ }
+ }
+
+ /** This path must be on the same file system as the to-path. Returns UnixPath of 'to'. */
+ public UnixPath atomicMove(Path to) {
+ uncheck(() -> Files.move(path, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING));
+ return new UnixPath(to);
+ }
+
+ public boolean moveIfExists(Path to) {
+ try {
+ Files.move(path, to);
+ return true;
+ } catch (NoSuchFileException ignored) {
+ return false;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Creates a symbolic link from {@code link} to {@code this} (the target)
+ * @param link the path for the symbolic link
+ * @return the path to the symbolic link
+ */
+ public UnixPath createSymbolicLink(Path link) {
+ uncheck(() -> Files.createSymbolicLink(link, path));
+ return new UnixPath(link);
+ }
+
+ @Override
+ public String toString() {
+ return path.toString();
+ }
+
+ private FileAttribute<?>[] permissionsAsFileAttributes(String... permissions) {
+ if (permissions.length == 0) return new FileAttribute<?>[0];
+ if (permissions.length > 1)
+ throw new IllegalArgumentException("Expected permissions to not be set or be a single string");
+
+ return new FileAttribute<?>[]{PosixFilePermissions.asFileAttribute(getPosixFilePermissionsFromString(permissions[0]))};
+ }
+
+ private Set<PosixFilePermission> getPosixFilePermissionsFromString(String permissions) {
+ try {
+ return PosixFilePermissions.fromString(permissions);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Failed to set permissions '" +
+ permissions + "' on path " + path, e);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java
new file mode 100644
index 00000000000..93ad0f21fe0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import java.util.Objects;
+
+/**
+ * A regular UNIX-style user and its primary group.
+ *
+ * @author mpolden
+ */
+public class UnixUser {
+
+ public static final UnixUser ROOT = new UnixUser("root", 0, "root", 0);
+ public static final UnixUser VESPA = new UnixUser("vespa", 1000, "vespa", 1000);
+
+ private final String name;
+ private final int uid;
+ private final String group;
+ private final int gid;
+
+ public UnixUser(String name, int uid, String group, int gid) {
+ this.name = name;
+ this.uid = uid;
+ this.group = group;
+ this.gid = gid;
+ }
+
+ /** Username of this */
+ public String name() { return name; }
+
+ /** User ID of this */
+ public int uid() { return uid; }
+
+ /** Primary group of this */
+ public String group() { return group; }
+
+ /** Primary group ID of this */
+ public int gid() { return gid; }
+
+ @Override
+ public String toString() {
+ return "user " + name + ":" + group;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UnixUser unixUser = (UnixUser) o;
+ return uid == unixUser.uid && name.equals(unixUser.name) &&
+ gid == unixUser.gid && group.equals(unixUser.group);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(uid, name, gid, group);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java
new file mode 100644
index 00000000000..a15b918913a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java
new file mode 100644
index 00000000000..77978e65f42
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java
@@ -0,0 +1,81 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import java.io.IOException;
+import java.nio.file.ProviderMismatchException;
+import java.nio.file.attribute.FileTime;
+import java.nio.file.attribute.GroupPrincipal;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.UserPrincipal;
+import java.util.Map;
+import java.util.Set;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
+
+/**
+ * @author freva
+ */
+class ContainerAttributeViews {
+
+ static class ContainerPosixFileAttributeView implements PosixFileAttributeView {
+ private final PosixFileAttributeView posixFileAttributeView;
+ private final ContainerPosixFileAttributes fileAttributes;
+
+ ContainerPosixFileAttributeView(PosixFileAttributeView posixFileAttributeView,
+ ContainerPosixFileAttributes fileAttributes) {
+ this.posixFileAttributeView = posixFileAttributeView;
+ this.fileAttributes = fileAttributes;
+ }
+
+ @Override public String name() { return "posix"; }
+ @Override public UserPrincipal getOwner() { return fileAttributes.owner(); }
+ @Override public PosixFileAttributes readAttributes() { return fileAttributes; }
+
+ @Override
+ public void setOwner(UserPrincipal owner) throws IOException {
+ if (!(owner instanceof ContainerUserPrincipal)) throw new ProviderMismatchException();
+ posixFileAttributeView.setOwner(((ContainerUserPrincipal) owner).baseFsPrincipal());
+ }
+
+ @Override
+ public void setGroup(GroupPrincipal group) throws IOException {
+ if (!(group instanceof ContainerGroupPrincipal)) throw new ProviderMismatchException();
+ posixFileAttributeView.setGroup(((ContainerGroupPrincipal) group).baseFsPrincipal());
+ }
+
+ @Override
+ public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException {
+ posixFileAttributeView.setTimes(lastModifiedTime, lastAccessTime, createTime);
+ }
+
+ @Override
+ public void setPermissions(Set<PosixFilePermission> perms) throws IOException {
+ posixFileAttributeView.setPermissions(perms);
+ }
+ }
+
+ static class ContainerPosixFileAttributes implements PosixFileAttributes {
+ private final Map<String, Object> attributes;
+
+ ContainerPosixFileAttributes(Map<String, Object> attributes) {
+ this.attributes = attributes;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override public Set<PosixFilePermission> permissions() { return (Set<PosixFilePermission>) attributes.get("permissions"); }
+ @Override public ContainerUserPrincipal owner() { return (ContainerUserPrincipal) attributes.get("owner"); }
+ @Override public ContainerGroupPrincipal group() { return (ContainerGroupPrincipal) attributes.get("group"); }
+ @Override public FileTime lastModifiedTime() { return (FileTime) attributes.get("lastModifiedTime"); }
+ @Override public FileTime lastAccessTime() { return (FileTime) attributes.get("lastAccessTime"); }
+ @Override public FileTime creationTime() { return (FileTime) attributes.get("creationTime"); }
+ @Override public boolean isRegularFile() { return (boolean) attributes.get("isRegularFile"); }
+ @Override public boolean isDirectory() { return (boolean) attributes.get("isDirectory"); }
+ @Override public boolean isSymbolicLink() { return (boolean) attributes.get("isSymbolicLink"); }
+ @Override public boolean isOther() { return (boolean) attributes.get("isOther"); }
+ @Override public long size() { return (long) attributes.get("size"); }
+ @Override public Object fileKey() { return attributes.get("fileKey"); }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java
new file mode 100644
index 00000000000..3329a646671
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java
@@ -0,0 +1,98 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.WatchService;
+import java.util.Set;
+
+/**
+ * @author freva
+ */
+public class ContainerFileSystem extends FileSystem {
+
+ private final ContainerFileSystemProvider containerFsProvider;
+ private final Path containerRootOnHost;
+
+ ContainerFileSystem(ContainerFileSystemProvider containerFsProvider, Path containerRootOnHost) {
+ this.containerFsProvider = containerFsProvider;
+ this.containerRootOnHost = containerRootOnHost;
+ }
+
+ public Path containerRootOnHost() {
+ return containerRootOnHost;
+ }
+
+ public void createRoot() {
+ provider().createFileSystemRoot();
+ }
+
+ @Override
+ public ContainerFileSystemProvider provider() {
+ return containerFsProvider;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public boolean isReadOnly() {
+ return false;
+ }
+
+ @Override
+ public String getSeparator() {
+ return "/";
+ }
+
+ @Override
+ public Set<String> supportedFileAttributeViews() {
+ return Set.of("basic", "posix", "unix", "owner");
+ }
+
+ @Override
+ public ContainerUserPrincipalLookupService getUserPrincipalLookupService() {
+ return containerFsProvider.userPrincipalLookupService();
+ }
+
+ @Override
+ public ContainerPath getPath(String first, String... more) {
+ return ContainerPath.fromPathInContainer(this, Path.of(first, more), getUserPrincipalLookupService().userScope().root());
+ }
+
+ @Override
+ public void close() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterable<Path> getRootDirectories() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterable<FileStore> getFileStores() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public PathMatcher getPathMatcher(String syntaxAndPattern) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WatchService newWatchService() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static ContainerFileSystem create(Path containerStorageRoot, UserScope userScope) {
+ return new ContainerFileSystemProvider(containerStorageRoot, userScope).getFileSystem(null);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java
new file mode 100644
index 00000000000..469ddd89ea3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java
@@ -0,0 +1,348 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AccessMode;
+import java.nio.file.CopyOption;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystemAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.ProviderMismatchException;
+import java.nio.file.SecureDirectoryStream;
+import java.nio.file.attribute.BasicFileAttributeView;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.attribute.GroupPrincipal;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.nio.file.attribute.UserPrincipal;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributeView;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author freva
+ */
+class ContainerFileSystemProvider extends FileSystemProvider {
+
+ private static final FileAttribute<?> DEFAULT_FILE_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0640
+ PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.GROUP_READ));
+ private static final FileAttribute<?> DEFAULT_DIRECTORY_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0750
+ PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE));
+
+ private final ContainerFileSystem containerFs;
+ private final ContainerUserPrincipalLookupService userPrincipalLookupService;
+
+ ContainerFileSystemProvider(Path containerRootOnHost, UserScope userScope) {
+ this.containerFs = new ContainerFileSystem(this, containerRootOnHost);
+ this.userPrincipalLookupService = new ContainerUserPrincipalLookupService(
+ containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), userScope);
+ }
+
+ public ContainerUserPrincipalLookupService userPrincipalLookupService() {
+ return userPrincipalLookupService;
+ }
+
+ @Override
+ public String getScheme() {
+ return "file";
+ }
+
+ @Override
+ public FileSystem newFileSystem(URI uri, Map<String, ?> env) {
+ throw new FileSystemAlreadyExistsException();
+ }
+
+ @Override
+ public ContainerFileSystem getFileSystem(URI uri) {
+ return containerFs;
+ }
+
+ @Override
+ public Path getPath(URI uri) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ try (SecureDirectoryStream<Path> sds = leafDirectoryStream(pathOnHost)) {
+ boolean existedBefore = Files.exists(pathOnHost);
+ SeekableByteChannel seekableByteChannel = sds.newByteChannel(
+ pathOnHost.getFileName(), addNoFollow(options), addPermissions(DEFAULT_FILE_PERMISSIONS, attrs));
+ if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(path));
+ return seekableByteChannel;
+ }
+ }
+
+ @Override
+ public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException {
+ Path pathOnHost = pathOnHost(dir);
+ return new ContainerDirectoryStream(provider(pathOnHost).newDirectoryStream(pathOnHost, filter),
+ toContainerPath(dir).user());
+ }
+
+ @Override
+ public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
+ Path pathOnHost = pathOnHost(dir);
+ boolean existedBefore = Files.exists(pathOnHost);
+ provider(pathOnHost).createDirectory(pathOnHost, addPermissions(DEFAULT_DIRECTORY_PERMISSIONS, attrs));
+ if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(dir));
+ }
+
+ @Override
+ public void delete(Path path) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ provider(pathOnHost).delete(pathOnHost);
+ }
+
+ @Override
+ public void copy(Path source, Path target, CopyOption... options) throws IOException {
+ // Only called when both 'source' and 'target' have 'this' as the FS provider
+ Path targetPathOnHost = pathOnHost(target);
+ provider(targetPathOnHost).copy(pathOnHost(source), targetPathOnHost, addNoFollow(options));
+ }
+
+ @Override
+ public void move(Path source, Path target, CopyOption... options) throws IOException {
+ // Only called when both 'source' and 'target' have 'this' as the FS provider
+ Path targetPathOnHost = pathOnHost(target);
+ provider(targetPathOnHost).move(pathOnHost(source), targetPathOnHost, addNoFollow(options));
+ }
+
+ @Override
+ public void createSymbolicLink(Path link, Path target, FileAttribute<?>... attrs) throws IOException {
+ Path pathOnHost = pathOnHost(link);
+ boolean existedBefore = Files.exists(pathOnHost, LinkOption.NOFOLLOW_LINKS);
+ if (target instanceof ContainerPath)
+ target = pathOnHost.getFileSystem().getPath(toContainerPath(target).pathInContainer());
+ provider(pathOnHost).createSymbolicLink(pathOnHost, target, attrs);
+ if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(link));
+ }
+
+ @Override
+ public Path readSymbolicLink(Path link) throws IOException {
+ Path pathOnHost = pathOnHost(link);
+ return provider(pathOnHost).readSymbolicLink(pathOnHost);
+ }
+
+ @Override
+ public boolean isSameFile(Path path, Path path2) throws IOException {
+ // 'path' FS provider should be 'this'
+ if (path2 instanceof ContainerPath)
+ path2 = pathOnHost(path2);
+ Path pathOnHost = pathOnHost(path);
+ return provider(pathOnHost).isSameFile(pathOnHost, path2);
+ }
+
+ @Override
+ public boolean isHidden(Path path) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ return provider(pathOnHost).isHidden(pathOnHost);
+ }
+
+ @Override
+ public FileStore getFileStore(Path path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void checkAccess(Path path, AccessMode... modes) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ provider(pathOnHost).checkAccess(pathOnHost, modes);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
+ if (!type.isAssignableFrom(PosixFileAttributeView.class)) return null;
+ Path pathOnHost = pathOnHost(path);
+ FileSystemProvider provider = pathOnHost.getFileSystem().provider();
+ if (type == BasicFileAttributeView.class) // Basic view doesn't have owner/group fields, forward to base FS provider
+ return provider.getFileAttributeView(pathOnHost, type, addNoFollow(options));
+
+ PosixFileAttributeView view = provider.getFileAttributeView(pathOnHost, PosixFileAttributeView.class, addNoFollow(options));
+ return (V) new ContainerPosixFileAttributeView(view,
+ uncheck(() -> new ContainerPosixFileAttributes(readAttributes(path, "unix:*", addNoFollow(options)))));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
+ if (!type.isAssignableFrom(PosixFileAttributes.class)) throw new UnsupportedOperationException();
+ Path pathOnHost = pathOnHost(path);
+ if (type == BasicFileAttributes.class)
+ return pathOnHost.getFileSystem().provider().readAttributes(pathOnHost, type, addNoFollow(options));
+
+ // Non-basic requests need to be upgraded to unix:* to get owner,group,uid,gid fields, which are then re-mapped
+ return (A) new ContainerPosixFileAttributes(readAttributes(path, "unix:*", addNoFollow(options)));
+ }
+
+ @Override
+ public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ int index = attributes.indexOf(':');
+ if (index < 0 || attributes.startsWith("basic:"))
+ return provider(pathOnHost).readAttributes(pathOnHost, attributes, addNoFollow(options));
+
+ Map<String, Object> attrs = new HashMap<>(provider(pathOnHost).readAttributes(pathOnHost, "unix:*", addNoFollow(options)));
+ int uid = userPrincipalLookupService.userIdInContainer((int) attrs.get("uid"));
+ int gid = userPrincipalLookupService.groupIdInContainer((int) attrs.get("gid"));
+ attrs.put("uid", uid);
+ attrs.put("gid", gid);
+ attrs.put("owner", userPrincipalLookupService.userPrincipal(uid, (UserPrincipal) attrs.get("owner")));
+ attrs.put("group", userPrincipalLookupService.groupPrincipal(gid, (GroupPrincipal) attrs.get("group")));
+ return attrs;
+ }
+
+ @Override
+ public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
+ Path pathOnHost = pathOnHost(path);
+ provider(pathOnHost).setAttribute(pathOnHost, attribute, fixAttributeValue(attribute, value), addNoFollow(options));
+ }
+
+ private Object fixAttributeValue(String attribute, Object value) {
+ int index = attribute.indexOf(':');
+ if (index > 0) {
+ switch (attribute.substring(index + 1)) {
+ case "owner": return cast(value, ContainerUserPrincipal.class).baseFsPrincipal();
+ case "group": return cast(value, ContainerGroupPrincipal.class).baseFsPrincipal();
+ case "uid": return userPrincipalLookupService.userIdOnHost(cast(value, Integer.class));
+ case "gid": return userPrincipalLookupService.groupIdOnHost(cast(value, Integer.class));
+ }
+ } // else basic file attribute
+ return value;
+ }
+
+ void createFileSystemRoot() {
+ ContainerPath root = containerFs.getPath("/");
+ if (!Files.exists(root)) {
+ uncheck(() -> {
+ Files.createDirectories(root.pathOnHost());
+ fixOwnerToContainerRoot(root);
+ });
+ }
+ }
+
+ private void fixOwnerToContainerRoot(ContainerPath path) throws IOException {
+ setAttribute(path, "unix:uid", path.user().uid(), LinkOption.NOFOLLOW_LINKS);
+ setAttribute(path, "unix:gid", path.user().gid(), LinkOption.NOFOLLOW_LINKS);
+ }
+
+ private SecureDirectoryStream<Path> leafDirectoryStream(Path pathOnHost) throws IOException {
+ Path containerRoot = containerFs.containerRootOnHost();
+ SecureDirectoryStream<Path> sds = ((SecureDirectoryStream<Path>) Files.newDirectoryStream(containerRoot));
+ for (int i = containerRoot.getNameCount(); i < pathOnHost.getNameCount() - 1; i++) {
+ SecureDirectoryStream<Path> next = sds.newDirectoryStream(pathOnHost.getName(i), LinkOption.NOFOLLOW_LINKS);
+ sds.close();
+ sds = next;
+ }
+ return sds;
+ }
+
+ private class ContainerDirectoryStream implements DirectoryStream<Path> {
+ private final DirectoryStream<Path> hostDirectoryStream;
+ private final UnixUser user;
+
+ private ContainerDirectoryStream(DirectoryStream<Path> hostDirectoryStream, UnixUser user) {
+ this.hostDirectoryStream = hostDirectoryStream;
+ this.user = user;
+ }
+
+ @Override
+ public Iterator<Path> iterator() {
+ Iterator<Path> hostPathIterator = hostDirectoryStream.iterator();
+ return new Iterator<>() {
+ @Override
+ public boolean hasNext() {
+ return hostPathIterator.hasNext();
+ }
+
+ @Override
+ public Path next() {
+ Path pathOnHost = hostPathIterator.next();
+ return ContainerPath.fromPathOnHost(containerFs, pathOnHost, user);
+ }
+ };
+ }
+
+ @Override
+ public void close() throws IOException {
+ hostDirectoryStream.close();
+ }
+ }
+
+ static ContainerPath toContainerPath(Path path) {
+ return cast(path, ContainerPath.class);
+ }
+
+ private static <T> T cast(Object value, Class<T> type) {
+ if (type.isInstance(value)) return type.cast(value);
+ throw new ProviderMismatchException("Expected " + type.getSimpleName() + ", was " + value.getClass().getName());
+ }
+
+ private static Path pathOnHost(Path path) {
+ return toContainerPath(path).pathOnHost();
+ }
+
+ private static FileSystemProvider provider(Path path) {
+ return path.getFileSystem().provider();
+ }
+
+ private static Set<? extends OpenOption> addNoFollow(Set<? extends OpenOption> options) {
+ if (options.contains(LinkOption.NOFOLLOW_LINKS)) return options;
+ Set<OpenOption> copy = new HashSet<>(options);
+ copy.add(LinkOption.NOFOLLOW_LINKS);
+ return copy;
+ }
+
+ private static LinkOption[] addNoFollow(LinkOption... options) {
+ if (Set.of(options).contains(LinkOption.NOFOLLOW_LINKS)) return options;
+ LinkOption[] copy = new LinkOption[options.length + 1];
+ System.arraycopy(options, 0, copy, 0, options.length);
+ copy[options.length] = LinkOption.NOFOLLOW_LINKS;
+ return copy;
+ }
+
+ private static CopyOption[] addNoFollow(CopyOption... options) {
+ if (Set.of(options).contains(LinkOption.NOFOLLOW_LINKS)) return options;
+ CopyOption[] copy = new CopyOption[options.length + 1];
+ System.arraycopy(options, 0, copy, 0, options.length);
+ copy[options.length] = LinkOption.NOFOLLOW_LINKS;
+ return copy;
+ }
+
+ private static FileAttribute<?>[] addPermissions(FileAttribute<?> defaultPermissions, FileAttribute<?>... attrs) {
+ for (FileAttribute<?> attr : attrs) {
+ if (attr.name().equals("posix:permissions") || attr.name().equals("unix:permissions"))
+ return attrs;
+ }
+
+ FileAttribute<?>[] copy = new FileAttribute<?>[attrs.length + 1];
+ System.arraycopy(attrs, 0, copy, 0, attrs.length);
+ copy[attrs.length] = defaultPermissions;
+ return copy;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java
new file mode 100644
index 00000000000..314e7cde5e2
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java
@@ -0,0 +1,224 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystemProvider.toContainerPath;
+
+/**
+ * Represents a path in container that is mapped in from the host. ContainerPaths are always normalized and absolute.
+ *
+ * @author freva
+ */
+public class ContainerPath implements Path {
+ private final ContainerFileSystem containerFs;
+ private final Path pathOnHost;
+ private final String[] parts;
+ private final UnixUser user;
+
+ private ContainerPath(ContainerFileSystem containerFs, Path pathOnHost, String[] parts, UnixUser user) {
+ this.containerFs = Objects.requireNonNull(containerFs);
+ this.pathOnHost = Objects.requireNonNull(pathOnHost);
+ this.parts = Objects.requireNonNull(parts);
+ this.user = Objects.requireNonNull(user);
+
+ if (!pathOnHost.isAbsolute())
+ throw new IllegalArgumentException("Path host must be absolute: " + pathOnHost);
+ Path containerRootOnHost = containerFs.containerRootOnHost();
+ if (!pathOnHost.startsWith(containerRootOnHost))
+ throw new IllegalArgumentException("Path on host (" + pathOnHost + ") must start with container root on host (" + containerRootOnHost + ")");
+ }
+
+ public Path pathOnHost() { return pathOnHost; }
+ public String pathInContainer() { return '/' + String.join("/", parts); }
+ public ContainerPath withUser(UnixUser user) { return new ContainerPath(containerFs, pathOnHost, parts, user); }
+ public UnixUser user() { return user; }
+
+ @Override
+ public ContainerFileSystem getFileSystem() {
+ return containerFs;
+ }
+
+ @Override
+ public ContainerPath getRoot() {
+ return resolve(containerFs, new String[0], Path.of("/"), user);
+ }
+
+ @Override
+ public Path getFileName() {
+ if (parts.length == 0) return null;
+ return Path.of(parts[parts.length - 1]);
+ }
+
+ @Override
+ public ContainerPath getParent() {
+ if (parts.length == 0) return null;
+ return new ContainerPath(containerFs, pathOnHost.getParent(), Arrays.copyOf(parts, parts.length-1), user);
+ }
+
+ @Override
+ public int getNameCount() {
+ return parts.length;
+ }
+
+ @Override
+ public Path getName(int index) {
+ return Path.of(parts[index]);
+ }
+
+ @Override
+ public Path subpath(int beginIndex, int endIndex) {
+ if (beginIndex < 0 || beginIndex >= endIndex || endIndex > parts.length)
+ throw new IllegalArgumentException();
+ if (endIndex - beginIndex == 1) return getName(beginIndex);
+
+ String[] rest = new String[endIndex - beginIndex - 1];
+ System.arraycopy(parts, beginIndex + 1, rest, 0, rest.length);
+ return Path.of(parts[beginIndex], rest);
+ }
+
+ @Override public ContainerPath resolve(Path other) { return resolve(containerFs, parts, other, user); }
+ @Override public ContainerPath resolve(String other) { return resolve(Path.of(other)); }
+ @Override public ContainerPath resolveSibling(String other) { return resolve(Path.of("..", other)); }
+
+ @Override
+ public boolean startsWith(Path other) {
+ if (other.getFileSystem() != containerFs) return false;
+ String[] otherParts = toContainerPath(other).parts;
+ if (parts.length < otherParts.length) return false;
+
+ for (int i = 0; i < otherParts.length; i++) {
+ if ( ! parts[i].equals(otherParts[i])) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean endsWith(Path other) {
+ int offset = parts.length - other.getNameCount();
+ // If the other path is longer than this, or the other path is absolute and shorter than this
+ if (offset < 0 || (other.isAbsolute() && offset > 0)) return false;
+
+ for (int i = 0; i < other.getNameCount(); i++) {
+ if ( ! parts[offset + i].equals(other.getName(i).toString())) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isAbsolute() {
+ // All container paths are normalized and absolute
+ return true;
+ }
+
+ @Override
+ public ContainerPath normalize() {
+ // All container paths are normalized and absolute
+ return this;
+ }
+
+ @Override
+ public ContainerPath toAbsolutePath() {
+ // All container paths are normalized and absolute
+ return this;
+ }
+
+ @Override
+ public ContainerPath toRealPath(LinkOption... options) throws IOException {
+ Path realPathOnHost = pathOnHost.toRealPath(options);
+ if (realPathOnHost.equals(pathOnHost)) return this;
+ return fromPathOnHost(containerFs, realPathOnHost, user);
+ }
+
+ @Override
+ public Path relativize(Path other) {
+ return pathOnHost.relativize(toContainerPath(other).pathOnHost);
+ }
+
+ @Override
+ public URI toUri() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException {
+ return pathOnHost.register(watcher, events, modifiers);
+ }
+
+ @Override
+ public int compareTo(Path other) {
+ return pathOnHost.compareTo(toContainerPath(other));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ContainerPath paths = (ContainerPath) o;
+ return containerFs.equals(paths.containerFs) && pathOnHost.equals(paths.pathOnHost) && Arrays.equals(parts, paths.parts);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(containerFs, pathOnHost);
+ result = 31 * result + Arrays.hashCode(parts);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return containerFs.containerRootOnHost().getFileName() + ":" + pathInContainer();
+ }
+
+ private static ContainerPath resolve(ContainerFileSystem containerFs, String[] currentParts, Path other, UnixUser user) {
+ List<String> parts = other.isAbsolute() ? new ArrayList<>() : new ArrayList<>(Arrays.asList(currentParts));
+ for (int i = 0; i < other.getNameCount(); i++) {
+ String part = other.getName(i).toString();
+ if (part.isEmpty() || part.equals(".")) continue;
+ if (part.equals("..")) {
+ if (!parts.isEmpty()) parts.remove(parts.size() - 1);
+ continue;
+ }
+ parts.add(part);
+ }
+
+ return new ContainerPath(containerFs,
+ containerFs.containerRootOnHost().resolve(String.join("/", parts)),
+ parts.toArray(String[]::new),
+ user);
+ }
+
+ public static ContainerPath fromPathInContainer(ContainerFileSystem containerFs, Path pathInContainer, UnixUser user) {
+ if (!pathInContainer.isAbsolute())
+ throw new IllegalArgumentException("Path in container must be absolute: " + pathInContainer);
+ return resolve(containerFs, new String[0], pathInContainer, user);
+ }
+
+ public static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost, UnixUser user) {
+ pathOnHost = pathOnHost.normalize();
+ Path containerRootOnHost = containerFs.containerRootOnHost();
+ Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost);
+
+ if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty())
+ return new ContainerPath(containerFs, pathOnHost, new String[0], user);
+ if (pathUnderContainerStorage.getName(0).toString().equals(".."))
+ throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost);
+
+ List<String> parts = new ArrayList<>();
+ for (int i = 0; i < pathUnderContainerStorage.getNameCount(); i++)
+ parts.add(pathUnderContainerStorage.getName(i).toString());
+ return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new), user);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java
new file mode 100644
index 00000000000..1a9b9b60cd4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java
@@ -0,0 +1,130 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+
+import java.io.IOException;
+import java.nio.file.attribute.GroupPrincipal;
+import java.nio.file.attribute.UserPrincipal;
+import java.nio.file.attribute.UserPrincipalLookupService;
+import java.nio.file.attribute.UserPrincipalNotFoundException;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * @author freva
+ */
+public class ContainerUserPrincipalLookupService extends UserPrincipalLookupService {
+
+ private final UserPrincipalLookupService baseFsUserPrincipalLookupService;
+ private final UserScope userScope;
+
+ ContainerUserPrincipalLookupService(UserPrincipalLookupService baseFsUserPrincipalLookupService, UserScope userScope) {
+ this.baseFsUserPrincipalLookupService = Objects.requireNonNull(baseFsUserPrincipalLookupService);
+ this.userScope = Objects.requireNonNull(userScope);
+ }
+
+ public UserScope userScope() { return userScope; }
+
+ public int userIdOnHost(int containerUid) { return userScope.namespace().userIdOnHost(containerUid); }
+ public int groupIdOnHost(int containerGid) { return userScope.namespace().groupIdOnHost(containerGid); }
+ public int userIdInContainer(int hostUid) { return userScope.namespace().userIdInContainer(hostUid); }
+ public int groupIdInContainer(int hostGid) { return userScope.namespace().groupIdInContainer(hostGid); }
+
+ @Override
+ public ContainerUserPrincipal lookupPrincipalByName(String name) throws IOException {
+ int containerUid = resolveName(name, UnixUser::uid, UnixUser::name);
+ String user = resolveId(containerUid, UnixUser::uid, UnixUser::name);
+ String hostUid = String.valueOf(userIdOnHost(containerUid));
+ return new ContainerUserPrincipal(containerUid, user, baseFsUserPrincipalLookupService.lookupPrincipalByName(hostUid));
+ }
+
+ @Override
+ public ContainerGroupPrincipal lookupPrincipalByGroupName(String group) throws IOException {
+ int containerGid = resolveName(group, UnixUser::gid, UnixUser::group);
+ String name = resolveId(containerGid, UnixUser::gid, UnixUser::group);
+ String hostGid = String.valueOf(groupIdOnHost(containerGid));
+ return new ContainerGroupPrincipal(containerGid, name, baseFsUserPrincipalLookupService.lookupPrincipalByGroupName(hostGid));
+ }
+
+ public ContainerUserPrincipal userPrincipal(int uid, UserPrincipal baseFsPrincipal) {
+ String name = resolveId(uid, UnixUser::uid, UnixUser::name);
+ return new ContainerUserPrincipal(uid, name, baseFsPrincipal);
+ }
+
+ public ContainerGroupPrincipal groupPrincipal(int gid, GroupPrincipal baseFsPrincipal) {
+ String name = resolveId(gid, UnixUser::gid, UnixUser::group);
+ return new ContainerGroupPrincipal(gid, name, baseFsPrincipal);
+ }
+
+ private String resolveId(int id, Function<UnixUser, Integer> idExtractor, Function<UnixUser, String> nameExtractor) {
+ if (idExtractor.apply(userScope.root()) == id) return nameExtractor.apply(userScope.root());
+ if (idExtractor.apply(userScope.vespa()) == id) return nameExtractor.apply(userScope.vespa());
+ return String.valueOf(id);
+ }
+
+ private int resolveName(String name, Function<UnixUser, Integer> idExtractor, Function<UnixUser, String> nameExtractor) throws UserPrincipalNotFoundException {
+ if (name.equals(nameExtractor.apply(userScope.root()))) return idExtractor.apply(userScope.root());
+ if (name.equals(nameExtractor.apply(userScope.vespa()))) return idExtractor.apply(userScope.vespa());
+
+ try {
+ return Integer.parseInt(name);
+ } catch (NumberFormatException ignored) {
+ throw new UserPrincipalNotFoundException(name);
+ }
+ }
+
+ private abstract static class NamedPrincipal implements UserPrincipal {
+ private final int id;
+ private final String name;
+ private final UserPrincipal baseFsPrincipal;
+
+ private NamedPrincipal(int id, String name, UserPrincipal baseFsPrincipal) {
+ this.id = id;
+ this.name = Objects.requireNonNull(name);
+ this.baseFsPrincipal = Objects.requireNonNull(baseFsPrincipal);
+ }
+
+ @Override
+ public final String getName() {
+ return name;
+ }
+
+ public int id() {
+ return id;
+ }
+
+ public UserPrincipal baseFsPrincipal() {
+ return baseFsPrincipal;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ NamedPrincipal that = (NamedPrincipal) o;
+ return id == that.id && baseFsPrincipal.equals(that.baseFsPrincipal);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, baseFsPrincipal);
+ }
+
+ @Override
+ public String toString() {
+ return "{id=" + id + ", baseFsPrincipal=" + baseFsPrincipal + '}';
+ }
+ }
+
+ static final class ContainerUserPrincipal extends NamedPrincipal {
+ private ContainerUserPrincipal(int id, String name, UserPrincipal baseFsPrincipal) { super(id, name, baseFsPrincipal); }
+ }
+
+ static final class ContainerGroupPrincipal extends NamedPrincipal implements GroupPrincipal {
+ private ContainerGroupPrincipal(int id, String name, GroupPrincipal baseFsPrincipal) { super(id, name, baseFsPrincipal); }
+
+ @Override public GroupPrincipal baseFsPrincipal() { return (GroupPrincipal) super.baseFsPrincipal(); }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java
new file mode 100644
index 00000000000..6891089ff71
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java
new file mode 100644
index 00000000000..965cd9942d6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java
@@ -0,0 +1,142 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import ai.vespa.net.CidrBlock;
+import com.google.common.net.InetAddresses;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+
+import java.io.UncheckedIOException;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * IP addresses - IP utilities to retrieve and manipulate addresses for docker host and docker containers in a
+ * multi-home environment.
+ * <p>
+ * The assumption is that DNS is the source of truth for which address are assigned to the host and which
+ * that belongs to the containers. Only one address should be assigned to each.
+ * <p>
+ * The behavior with respect to site-local addresses are distinct for IPv4 and IPv6. For IPv4 we choose
+ * the site-local address (assume the public is a NAT address not assigned to the host interface (the typical aws setup)).
+ * <p>
+ * For IPv6 we disregard any site-local addresses (these are normally not in DNS anyway).
+ * <p>
+ * This class also provides some utilities for prefix translation.
+ *
+ * @author smorgrav
+ */
+public interface IPAddresses {
+ CidrBlock gcpInternalBlock = CidrBlock.fromString("2600:2d00::/32");
+
+ InetAddress[] getAddresses(String hostname);
+
+ default Optional<InetAddress> getAddress(String hostname, IPVersion ipVersion) {
+ return ipVersion == IPVersion.IPv6
+ ? getIPv6Address(hostname).map(InetAddress.class::cast)
+ : getIPv4Address(hostname).map(InetAddress.class::cast);
+ }
+
+ /**
+ * Returns a list of string representation of the IP addresses (RFC 5952 compact format)
+ */
+ default List<String> getAddresses(String hostname, IPVersion ipVersion) {
+ return Stream.of(getAddresses(hostname))
+ .filter(inetAddress -> isOfType(inetAddress, ipVersion))
+ .map(InetAddresses::toAddrString)
+ .toList();
+ }
+
+ /**
+ * Get the IPv6 address for the host if any.
+ *
+ * @throws ConvergenceException if multiple addresses are found
+ */
+ default Optional<Inet6Address> getIPv6Address(String hostname) {
+ List<Inet6Address> ipv6addresses = Stream.of(getAddresses(hostname))
+ .filter(Inet6Address.class::isInstance)
+ .filter(inetAddress -> !inetAddress.isLoopbackAddress())
+ .map(Inet6Address.class::cast)
+ .filter(inetAddress -> !inetAddress.isLinkLocalAddress())
+ .filter(inetAddress -> !inetAddress.isSiteLocalAddress())
+ .filter(inet6Address -> !gcpInternalBlock.contains(inet6Address))
+ .toList();
+
+ if (ipv6addresses.size() <= 1) return ipv6addresses.stream().findFirst();
+
+ String addresses = ipv6addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(","));
+ throw ConvergenceException.ofError(
+ String.format(
+ "Multiple IPv6 addresses found: %s. Perhaps a missing DNS entry or multiple AAAA records in DNS?",
+ addresses));
+ }
+
+ /** Returns the hostname of given inetAddress */
+ default String getHostname(InetAddress inetAddress) {
+ String hostname = inetAddress.getHostName();
+ if (hostname.equals(inetAddress.getHostAddress())) {
+ throw new IllegalArgumentException("Could not find hostname for address " + inetAddress.getHostAddress());
+ }
+ return hostname;
+ }
+
+ /**
+ * Get the IPv4 address for the host if any.
+ *
+ * @throws ConvergenceException if multiple site-local addresses are found
+ */
+ default Optional<Inet4Address> getIPv4Address(String hostname) {
+ List<Inet4Address> ipv4Addresses = Stream.of(getAddresses(hostname))
+ .filter(Inet4Address.class::isInstance)
+ .filter(inetAddress -> !inetAddress.isLoopbackAddress())
+ .map(Inet4Address.class::cast)
+ .toList();
+
+ if (ipv4Addresses.size() <= 1) return ipv4Addresses.stream().findFirst();
+
+ List<Inet4Address> siteLocalIPv4Addresses = ipv4Addresses.stream()
+ .filter(InetAddress::isSiteLocalAddress)
+ .toList();
+
+ if (siteLocalIPv4Addresses.size() == 1) return Optional.of(siteLocalIPv4Addresses.get(0));
+
+ String addresses = ipv4Addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(","));
+ throw ConvergenceException.ofError(
+ String.format(
+ "Multiple IPv4 addresses found: %s. Perhaps a missing DNS entry or multiple A records in DNS?",
+ addresses));
+ }
+
+ static boolean isOfType(InetAddress address, IPVersion ipVersion) {
+ if (ipVersion.equals(IPVersion.IPv4) && address instanceof Inet4Address) return true;
+ if (ipVersion.equals(IPVersion.IPv6) && address instanceof Inet6Address) return true;
+ return false;
+ }
+
+ /**
+ * For NPTed networks we want to find the private address from a public.
+ *
+ * @param address The original address to translate
+ * @param prefix The prefix address
+ * @param subnetSizeInBytes in bits - e.g a /64 subnet equals 8 bytes
+ * @return The translated address
+ * @throws ConvergenceException if
+ */
+ static InetAddress prefixTranslate(InetAddress address, InetAddress prefix, int subnetSizeInBytes) {
+ return prefixTranslate(address.getAddress(), prefix.getAddress(), subnetSizeInBytes);
+ }
+
+ static InetAddress prefixTranslate(byte[] address, byte[] prefix, int nofBytes) {
+ System.arraycopy(prefix, 0, address, 0, nofBytes);
+ try {
+ return InetAddress.getByAddress(address);
+ } catch (UnknownHostException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java
new file mode 100644
index 00000000000..4680502cee7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java
@@ -0,0 +1,21 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesImpl implements IPAddresses {
+
+ @Override
+ public InetAddress[] getAddresses(String hostname) {
+ try {
+ return InetAddress.getAllByName(hostname);
+ } catch (UnknownHostException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java
new file mode 100644
index 00000000000..eb92cbdd303
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java
@@ -0,0 +1,85 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.google.common.net.InetAddresses;
+
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Strong type IPv4 and IPv6 with common executables for ip related commands.
+ *
+ * @author smorgrav
+ */
+public enum IPVersion {
+
+ IPv6(6, "ip6tables", "ip -6", "ipv6-icmp", 128, "icmp6-port-unreachable", "ip6tables-restore", "fe80::/10"),
+ IPv4(4, "iptables", "ip", "icmp", 32, "icmp-port-unreachable", "iptables-restore", "169.254.0.0/16");
+
+ private static final Pattern cidrNotationPattern = Pattern.compile("/\\d+$");
+
+ IPVersion(int version, String iptablesCmd, String ipCmd,
+ String icmpProtocol, int size, String icmpPortUnreachable,
+ String iptablesRestore, String linkLocalCidr) {
+ this.version = version;
+ this.ipCmd = ipCmd;
+ this.iptablesCmd = iptablesCmd;
+ this.icmpProtocol = icmpProtocol;
+ this.size = size;
+ this.icmpPortUnreachable = icmpPortUnreachable;
+ this.iptablesRestore = iptablesRestore;
+ this.linkLocalCidr = linkLocalCidr;
+ }
+
+ private final int version;
+ private final String iptablesCmd;
+ private final String ipCmd;
+ private final String icmpProtocol;
+ private final int size;
+ private final String icmpPortUnreachable;
+ private final String iptablesRestore;
+ private final String linkLocalCidr;
+
+ /** The ID of the IP version, either IPv4 or IPv6. */
+ public String id() { return "IPv" + version; }
+
+ /** The IP version, either 4 or 6 */
+ public int version() { return version; }
+
+ public String versionString() { return String.valueOf(version); }
+ public String iptablesCmd() { return iptablesCmd; }
+ public String iptablesRestore() { return iptablesRestore;}
+ public String ipCmd() { return ipCmd; }
+ public String icmpProtocol() { return icmpProtocol; }
+ public String singleHostCidr() { return "/" + size; }
+ public String icmpPortUnreachable() { return icmpPortUnreachable; }
+
+ /** The address size (in bits) of the IP version: 32 or 128. */
+ public int addressSize() { return size; }
+
+ /** Both IPv4 and IPv6 have exactly one link-local address space: 169.254.0.0/16 or fe80::/10. */
+ public String linkLocalAddressCidr() { return linkLocalCidr; }
+
+ public boolean match(InetAddress address) {
+ return this == IPVersion.get(address);
+ }
+
+ public boolean match(String address) {
+ return this == IPVersion.get(address);
+ }
+
+ public static IPVersion get(String address) {
+ Matcher matcher = cidrNotationPattern.matcher(address);
+ if (matcher.find()) {
+ address = matcher.replaceFirst("");
+ }
+ return get(InetAddresses.forString(address));
+ }
+
+ public static IPVersion get(InetAddress address) {
+ return address instanceof Inet4Address ? IPv4 : IPv6;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java
new file mode 100644
index 00000000000..1186a58f53d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java
@@ -0,0 +1,85 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.google.common.net.InetAddresses;
+
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.util.Objects;
+
+/**
+ * Encapsulates an IP address and its version along with some convenience methods.
+ * Default sorting is by version (IPv6 first), then by address.
+ *
+ * @author gjoranv
+ */
+public class VersionedIpAddress implements Comparable<VersionedIpAddress> {
+
+ private final InetAddress address;
+ private final IPVersion version;
+
+ private VersionedIpAddress(InetAddress address) {
+ this.address = Objects.requireNonNull(address);
+ version = getVersionOrThrow(address);
+ }
+
+ public static VersionedIpAddress from(InetAddress address) {
+ return new VersionedIpAddress(address);
+ }
+
+ public static VersionedIpAddress from(String address) {
+ return from(InetAddresses.forString(address));
+ }
+
+ public IPVersion version() {
+ return version;
+ }
+
+ public String asString() {
+ return InetAddresses.toAddrString(address);
+ }
+
+ public String asEndpoint(int port) {
+ var format = (version == IPVersion.IPv6) ? "[%s]:%d" : "%s:%d";
+ return String.format(format, asString(), port);
+ }
+
+ @Override
+ public int compareTo(VersionedIpAddress o) {
+ int version = version().compareTo(o.version());
+ return (version != 0) ? version : asString().compareTo(o.asString());
+ }
+
+ @Override
+ public String toString() {
+ return "VersionedIpAddress{" +
+ "address=" + address +
+ ", version=" + version +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ VersionedIpAddress that = (VersionedIpAddress) o;
+ return address.equals(that.address) && version == that.version;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(address, version);
+ }
+
+ private static IPVersion getVersionOrThrow(InetAddress address) {
+ if (address instanceof Inet4Address) {
+ return IPVersion.IPv4;
+ } else if (address instanceof Inet6Address) {
+ return IPVersion.IPv6;
+ } else {
+ throw new IllegalArgumentException("Unknown IP version for " + InetAddresses.toAddrString(address) + " of class " + address.getClass().getName());
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java
new file mode 100644
index 00000000000..9533b7240c4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java
new file mode 100644
index 00000000000..572182f7991
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
new file mode 100644
index 00000000000..007547aa41b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java
@@ -0,0 +1,16 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+public interface ChildProcess2 extends AutoCloseable {
+ void waitForTermination();
+ int exitCode();
+ String getOutput();
+
+ /** Close/cleanup any resources held. Must not throw an exception. */
+ @Override
+ void close();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java
new file mode 100644
index 00000000000..8574028b6d7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java
@@ -0,0 +1,139 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import java.util.logging.Level;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ChildProcess2Impl implements ChildProcess2 {
+ private static final Logger logger = Logger.getLogger(ChildProcess2Impl.class.getName());
+
+ private final CommandLine commandLine;
+ private final ProcessApi2 process;
+ private final Path outputPath;
+ private final Timer timer;
+
+ public ChildProcess2Impl(CommandLine commandLine,
+ ProcessApi2 process,
+ Path outputPath,
+ Timer timer) {
+ this.commandLine = commandLine;
+ this.process = process;
+ this.outputPath = outputPath;
+ this.timer = timer;
+ }
+
+ @Override
+ public void waitForTermination() {
+ Duration timeoutDuration = commandLine.getTimeout();
+ Instant timeout = timer.currentTime().plus(timeoutDuration);
+ long maxOutputBytes = commandLine.getMaxOutputBytes();
+
+ // How frequently do we want to wake up and check the output file size?
+ final Duration pollInterval = Duration.ofSeconds(10);
+
+ boolean hasTerminated = false;
+ while (!hasTerminated) {
+ Instant now = timer.currentTime();
+ long sleepPeriodMillis = pollInterval.toMillis();
+ if (now.plusMillis(sleepPeriodMillis).isAfter(timeout)) {
+ sleepPeriodMillis = Duration.between(now, timeout).toMillis();
+
+ if (sleepPeriodMillis <= 0) {
+ gracefullyKill();
+ throw new TimeoutChildProcessException(
+ timeoutDuration, commandLine.toString(), getOutput());
+ }
+ }
+
+ try {
+ hasTerminated = process.waitFor(sleepPeriodMillis, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // Ignore, just loop around.
+ continue;
+ }
+
+ // Always check output file size to ensure we don't load too much into memory.
+ long sizeInBytes = uncheck(() -> Files.size(outputPath));
+ if (sizeInBytes > maxOutputBytes) {
+ gracefullyKill();
+ throw new LargeOutputChildProcessException(
+ sizeInBytes, commandLine.toString(), getOutput());
+ }
+ }
+ }
+
+ @Override
+ public int exitCode() {
+ return process.exitValue();
+ }
+
+ @Override
+ public String getOutput() {
+ byte[] bytes = uncheck(() -> Files.readAllBytes(outputPath));
+ return new String(bytes, commandLine.getOutputEncoding());
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (commandLine.getOutputFile().isEmpty())
+ Files.delete(outputPath);
+ } catch (Throwable t) {
+ logger.log(Level.WARNING, "Failed to delete " + outputPath, t);
+ }
+ }
+
+ Path getOutputPath() {
+ return outputPath;
+ }
+
+ private void gracefullyKill() {
+ process.destroy();
+
+ Duration maxWaitAfterSigTerm = commandLine.getSigTermGracePeriod();
+ Instant timeout = timer.currentTime().plus(maxWaitAfterSigTerm);
+ if (!waitForTermination(timeout)) {
+ process.destroyForcibly();
+
+ // If waiting for the process now takes a long time, it's probably a kernel issue
+ // or huge core is getting dumped.
+ Duration maxWaitAfterSigKill = commandLine.getSigKillGracePeriod();
+ if (!waitForTermination(timer.currentTime().plus(maxWaitAfterSigKill))) {
+ throw new UnkillableChildProcessException(
+ maxWaitAfterSigTerm,
+ maxWaitAfterSigKill,
+ commandLine.toString(),
+ getOutput());
+ }
+ }
+ }
+
+ /** @return true if process terminated, false on timeout. */
+ private boolean waitForTermination(Instant timeout) {
+ while (true) {
+ long waitDurationMillis = Duration.between(timer.currentTime(), timeout).toMillis();
+ if (waitDurationMillis <= 0) {
+ return false;
+ }
+
+ try {
+ return process.waitFor(waitDurationMillis, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
new file mode 100644
index 00000000000..9a0c08a8596
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
@@ -0,0 +1,47 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.text.internal.SnippetGenerator;
+
+/**
+ * Base class for child process related exceptions, with a util to build an error message
+ * that includes a large part of the output.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public abstract class ChildProcessException extends RuntimeException {
+ private static final SnippetGenerator snippetGenerator = new SnippetGenerator();
+
+ /**
+ * An exception with a message of the following format:
+ * Command 'COMMANDLINE' PROBLEM: stdout/stderr: 'OUTPUT'
+ *
+ * If the output of the terminated command is too large it will be sampled.
+ *
+ * @param problem E.g. "terminated with exit code 1"
+ * @param commandLine The command that failed in a concise (e.g. shell-like) format
+ * @param possiblyHugeOutput The output of the command
+ */
+ protected ChildProcessException(String problem, String commandLine, String possiblyHugeOutput) {
+ super(makeSnippet(problem, commandLine, possiblyHugeOutput));
+ }
+
+ protected ChildProcessException(RuntimeException cause,
+ String problem,
+ String commandLine,
+ String possiblyHugeOutput) {
+ super(makeSnippet(problem, commandLine, possiblyHugeOutput), cause);
+ }
+
+ private static String makeSnippet(String problem, String commandLine, String possiblyHugeOutput) {
+ return "Command '" +
+ commandLine +
+ "' " +
+ problem +
+ ": stdout/stderr: '" +
+ snippetGenerator.makeSnippet(possiblyHugeOutput, 500) +
+ "'";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java
new file mode 100644
index 00000000000..2d1fe1f24bd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java
@@ -0,0 +1,15 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * The child process terminated with a non-zero exit code.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class ChildProcessFailureException extends ChildProcessException {
+ ChildProcessFailureException(int exitCode, String commandLine, String possiblyHugeOutput) {
+ super("terminated with exit code " + exitCode, commandLine, possiblyHugeOutput);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
new file mode 100644
index 00000000000..516b50dc601
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java
@@ -0,0 +1,382 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * A CommandLine is used to specify and execute a shell-like program in a child process,
+ * and capture its output.
+ *
+ * @author hakonhall
+ */
+public class CommandLine {
+ private static final Logger logger = Logger.getLogger(CommandLine.class.getName());
+ private static final Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=!@%/+:.,_-]+$");
+
+ /** The default timeout. See setTimeout() for details. */
+ public static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(10);
+
+ /** The default maximum number of output bytes. See setMaxOutputBytes() for details. */
+ public static final long DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024 * 1024; // 1 Gb
+
+ /**
+ * The default grace period after SIGTERM has been sent during a graceful kill.
+ * See setSigTermGracePeriod for details.
+ */
+ public static final Duration DEFAULT_SIGTERM_GRACE_PERIOD = Duration.ofMinutes(1);
+
+ /**
+ * The default grace period after SIGKILL has been sent during a graceful kill.
+ * See setSigKillGracePeriod for details.
+ */
+ public static final Duration DEFAULT_SIGKILL_GRACE_PERIOD = Duration.ofMinutes(30);
+
+ private final List<String> arguments = new ArrayList<>();
+ private final Set<Integer> censoredArgumentIndices = new HashSet<>();
+ private final TreeMap<String, String> environment = new TreeMap<>();
+ private final TaskContext taskContext;
+ private final ProcessFactory processFactory;
+
+ private boolean redirectStderrToStdoutInsteadOfDiscard = true;
+ private boolean executeSilentlyCalled = false;
+ private Optional<Path> outputFile = Optional.empty();
+ private Charset outputEncoding = StandardCharsets.UTF_8;
+ private Duration timeout = DEFAULT_TIMEOUT;
+ private long maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES;
+ private Duration sigTermGracePeriod = DEFAULT_SIGTERM_GRACE_PERIOD;
+ private Duration sigKillGracePeriod = DEFAULT_SIGKILL_GRACE_PERIOD;
+ private Predicate<Integer> successfulExitCodePredicate = code -> code == 0;
+ private boolean waitForTermination = true;
+
+ public CommandLine(TaskContext taskContext, ProcessFactory processFactory) {
+ this.taskContext = taskContext;
+ this.processFactory = processFactory;
+ }
+
+ /** Add arguments to the command. The first argument in the first call to add() is the program. */
+ public CommandLine add(String... arguments) { return add(List.of(arguments)); }
+
+ /** Add arguments to the command. The first argument in the first call to add() is the program. */
+ public CommandLine add(Collection<String> arguments) {
+ this.arguments.addAll(arguments);
+ return this;
+ }
+
+ /** Add arguments by splitting arguments by space. */
+ public CommandLine addTokens(String arguments) {
+ return add(arguments.split("\\s+"));
+ }
+
+ /** Set an environment variable, overriding any existing. */
+ public CommandLine setEnvironmentVariable(String name, String value) {
+ if (name.indexOf('=') != -1) {
+ throw new IllegalArgumentException("name contains '=': " + name);
+ }
+ Objects.requireNonNull(value, "cannot set environment variable to null");
+
+ environment.put(name, value);
+ return this;
+ }
+
+ public CommandLine removeEnvironmentVariable(String name) {
+ if (name.indexOf('=') != -1) {
+ throw new IllegalArgumentException("name contains '=': " + name);
+ }
+ environment.put(name, null);
+ return this;
+ }
+
+ /** Censor (prevent logging of) the last argument added to this */
+ public CommandLine censorArgument() {
+ censoredArgumentIndices.add(arguments.size() - 1);
+ return this;
+ }
+
+ /**
+ * Execute a shell-like program in a child process:
+ * - the program is recorded and logged as modifying the system, but see executeSilently().
+ * - the program's stderr is redirected to stdout, but see discardStderr().
+ * - the program's output is assumed to be UTF-8, but see setOutputEncoding().
+ * - the program must terminate with exit code 0, but see ignoreExitCode().
+ * - the output of the program will be accessible in the returned CommandResult.
+ *
+ * Footnote 1: As a safety measure the size of the output is capped, and the program is
+ * only allowed to execute up to a timeout. The defaults are set high so you typically do
+ * not have to worry about reaching these limits, but otherwise see setMaxOutputBytes()
+ * and setTimeout(), respectively.
+ *
+ * Footnote 2: If the child process is forced to be killed due to footnote 1, then
+ * setSigTermGracePeriod() and setSigKillGracePeriod() can be used to tweak how much time
+ * is given to the program to shut down. Again, the defaults should be reasonable.
+ */
+ public CommandResult execute() {
+ taskContext.recordSystemModification(logger, "Executing command: " + toString());
+ return doExecute();
+ }
+
+ /**
+ * Same as execute(), except it will not record the program as modifying the system.
+ *
+ * If the program is later found to have modified the system, or otherwise worthy of
+ * a record, call recordSilentExecutionAsSystemModification().
+ */
+ public CommandResult executeSilently() {
+ executeSilentlyCalled = true;
+ return doExecute();
+ }
+
+ public static class Options {
+ private boolean silent = false;
+
+ public Options() {}
+
+ /** Invoke {@link #executeSilently()} instead of {@link #execute()} (default). */
+ public Options setSilent(boolean silent) {
+ this.silent = silent;
+ return this;
+ }
+ }
+
+ /** Convenience method to bundle up a bunch of calls on this into an options object. */
+ public CommandResult execute(Options options) {
+ return options.silent ? executeSilently() : execute();
+ }
+
+ /**
+ * Record an already executed executeSilently() as having modified the system.
+ * For instance with YUM it is not known until after a 'yum install' whether it
+ * modified the system.
+ */
+ public void recordSilentExecutionAsSystemModification() {
+ if (!executeSilentlyCalled) {
+ throw new IllegalStateException("executeSilently has not been called");
+ }
+ // Disallow multiple consecutive calls to this method without an intervening call
+ // to executeSilently().
+ executeSilentlyCalled = false;
+
+ taskContext.recordSystemModification(logger, "Executed command: " + toString());
+ }
+
+ /**
+ * The first argument of the command specifies the program and is either the program's
+ * filename (in case the environment variable PATH will be used to search for the program
+ * file) or a path with the last component being the program's filename.
+ *
+ * @return The filename of the program.
+ */
+ public String programName() {
+ if (arguments.isEmpty()) {
+ throw new IllegalStateException(
+ "The program name cannot be determined yet as no arguments have been given");
+ }
+ String path = arguments.get(0);
+ int lastIndex = path.lastIndexOf('/');
+ if (lastIndex == -1) {
+ return path;
+ } else {
+ return path.substring(lastIndex + 1);
+ }
+ }
+
+ /** Returns a shell-like representation of the command. */
+ @Override
+ public String toString() {
+ return toString(true);
+ }
+
+ String toString(boolean censor) {
+ var command = new StringBuilder();
+
+ if (!environment.isEmpty()) {
+ // Pretend environment is propagated through the env program for display purposes
+ command.append(environment.entrySet().stream()
+ .map(entry -> {
+ if (entry.getValue() == null) {
+ return "-u " + maybeEscapeArgument(entry.getKey());
+ } else {
+ return maybeEscapeArgument(entry.getKey() + "=" + entry.getValue());
+ }
+ })
+ .collect(Collectors.joining(" ", "env ", " ")));
+ }
+
+ for (int i = 0; i < arguments.size(); i++) {
+ if (censor && censoredArgumentIndices.contains(i)) {
+ command.append("<censored>");
+ } else {
+ command.append(maybeEscapeArgument(arguments.get(i)));
+ }
+ if (i < arguments.size() - 1) {
+ command.append(" ");
+ }
+ }
+
+ // Note: Both of these cannot be confused with an argument since they would
+ // require escaping.
+ command.append(redirectStderrToStdoutInsteadOfDiscard ? " 2>&1" : " 2>/dev/null");
+
+ return command.toString();
+ }
+
+
+ /**
+ * By default, stderr is redirected to stderr. This method will instead discard stderr.
+ */
+ public CommandLine discardStderr() {
+ this.redirectStderrToStdoutInsteadOfDiscard = false;
+ return this;
+ }
+
+ /**
+ * By default, a non-zero exit code will cause the command execution to fail. This method
+ * will instead ignore the exit code.
+ */
+ public CommandLine ignoreExitCode() {
+ this.successfulExitCodePredicate = code -> true;
+ return this;
+ }
+
+ /**
+ * By default, a non-zero exit code causes the command execution to fail. This method
+ * will override that predicate.
+ */
+ public CommandLine setSuccessfulExitCodePredicate(Predicate<Integer> successPredicate) {
+ successfulExitCodePredicate = successPredicate;
+ return this;
+ }
+
+ /**
+ * By default, the output of the command is parsed as UTF-8. This method will set a
+ * different encoding.
+ */
+ public CommandLine setOutputEncoding(Charset outputEncoding) {
+ this.outputEncoding = outputEncoding;
+ return this;
+ }
+
+ /**
+ * By default, the output of the command is piped to a temporary file, which is deleted
+ * when execution ends. This method will cause output to be piped to the given path
+ * instead, and the file will not be removed.
+ */
+ public CommandLine setOutputFile(Path outputFile) {
+ this.outputFile = Optional.of(outputFile);
+ return this;
+ }
+
+ /**
+ * By default, the command will be gracefully killed after DEFAULT_TIMEOUT. This method
+ * overrides that default.
+ */
+ public CommandLine setTimeout(Duration timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ /**
+ * By default, the command will be gracefully killed if it ever outputs more than
+ * DEFAULT_MAX_OUTPUT_BYTES. This method overrides that default.
+ */
+ public CommandLine setMaxOutputBytes(long maxOutputBytes) {
+ this.maxOutputBytes = maxOutputBytes;
+ return this;
+ }
+
+ /**
+ * By default, if the program needs to be gracefully killed it will wait up to
+ * DEFAULT_SIGTERM_GRACE_PERIOD for the program to exit after it has been killed with
+ * the SIGTERM signal.
+ */
+ public CommandLine setSigTermGracePeriod(Duration period) {
+ this.sigTermGracePeriod = period;
+ return this;
+ }
+
+ public CommandLine setSigKillGracePeriod(Duration period) {
+ this.sigKillGracePeriod = period;
+ return this;
+ }
+
+ /**
+ * WARNING: This will leave the child as a zombie process until this process dies.
+ * I.e. only use this just before or a limited number of times per host admin restart.
+ */
+ public CommandLine doNotWaitForTermination() {
+ this.waitForTermination = false;
+ return this;
+ }
+
+ public List<String> getArguments() { return Collections.unmodifiableList(arguments); }
+
+ /** Returns a copy of the environment overrides. A null value means the environment variable should be removed. */
+ public TreeMap<String, String> getEnvironmentOverrides() { return new TreeMap<>(environment); }
+
+ // Accessor fields necessary for classes in this package. Could be public if necessary.
+ boolean getRedirectStderrToStdoutInsteadOfDiscard() { return redirectStderrToStdoutInsteadOfDiscard; }
+ Predicate<Integer> getSuccessfulExitCodePredicate() { return successfulExitCodePredicate; }
+ Optional<Path> getOutputFile() { return outputFile; }
+ Charset getOutputEncoding() { return outputEncoding; }
+ Duration getTimeout() { return timeout; }
+ long getMaxOutputBytes() { return maxOutputBytes; }
+ Duration getSigTermGracePeriod() { return sigTermGracePeriod; }
+ Duration getSigKillGracePeriod() { return sigKillGracePeriod; }
+
+ private CommandResult doExecute() {
+ try (ChildProcess2 child = processFactory.spawn(this)) {
+ if (!waitForTermination) {
+ return new CommandResult(this, 0, "");
+ }
+
+ child.waitForTermination();
+ int exitCode = child.exitCode();
+ if (!successfulExitCodePredicate.test(exitCode)) {
+ throw new ChildProcessFailureException(exitCode, toString(), child.getOutput());
+ }
+
+ String output = child.getOutput();
+ return new CommandResult(this, exitCode, output);
+ }
+ }
+
+ private static String maybeEscapeArgument(String argument) {
+ if (UNESCAPED_ARGUMENT_PATTERN.matcher(argument).matches()) {
+ return argument;
+ } else {
+ return escapeArgument(argument);
+ }
+ }
+
+ private static String escapeArgument(String argument) {
+ StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10);
+
+ for (int i = 0; i < argument.length(); ++i) {
+ char c = argument.charAt(i);
+ switch (c) {
+ case '"', '\\' -> doubleQuoteEscaped.append("\\").append(c);
+ default -> doubleQuoteEscaped.append(c);
+ }
+ }
+
+ return "\"" + doubleQuoteEscaped + "\"";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java
new file mode 100644
index 00000000000..c4f3229792b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java
@@ -0,0 +1,92 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A CommandResult is the result of the execution of a CommandLine.
+ *
+ * @author hakonhall
+ */
+public class CommandResult {
+ private static final Pattern NEWLINE = Pattern.compile("\\n");
+
+ private final CommandLine commandLine;
+ private final int exitCode;
+ private final String output;
+
+ public CommandResult(CommandLine commandLine, int exitCode, String output) {
+ this.commandLine = commandLine;
+ this.exitCode = exitCode;
+ this.output = output;
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ /** Returns the output with leading and trailing white-space removed. */
+ public String getOutput() { return output.trim(); }
+
+ public String getUntrimmedOutput() { return output; }
+
+ /** Returns the output lines of the command, omitting trailing empty lines. */
+ public List<String> getOutputLines() {
+ return getOutputLinesStream().toList();
+ }
+
+ /** Returns the output lines as a stream, omitting trailing empty lines. */
+ public Stream<String> getOutputLinesStream() {
+ if (output.isEmpty()) {
+ // For some reason an empty string => one-element list.
+ return Stream.empty();
+ }
+
+ // For some reason this removes trailing empty elements, but that's OK with us.
+ return NEWLINE.splitAsStream(output);
+ }
+
+ /**
+ * Map this CommandResult to an instance of type R.
+ *
+ * If a RuntimeException is thrown by the mapper, it is wrapped in an
+ * UnexpectedOutputException that includes a snippet of the output in the message.
+ *
+ * This method is intended to be used as part of the verification of the output.
+ */
+ public <R> R map(Function<CommandResult, R> mapper) {
+ try {
+ return mapper.apply(this);
+ } catch (RuntimeException e) {
+ throw new UnexpectedOutputException(e, "Failed to map output", commandLine.toString(), output);
+ }
+ }
+
+ /**
+ * Map the output to an instance of type R according to mapper, wrapping any
+ * RuntimeException in UnexpectedOutputException w/output snippet. See map() for details.
+ */
+ public <R> R mapOutput(Function<String, R> mapper) { return map(result -> mapper.apply(result.getOutput())); }
+
+ /**
+ * Map each output line to an instance of type R according to mapper, wrapping any
+ * RuntimeException in UnexpectedOutputException w/output snippet. See map() for details.
+ */
+ public <R> List<R> mapEachLine(Function<String, R> mapper) {
+ return map(result -> result.getOutputLinesStream().map(mapper).toList());
+ }
+
+ /**
+ * Convenience method for getting the CommandLine, whose execution resulted in
+ * this CommandResult instance.
+ *
+ * Warning: the CommandLine is mutable and may be changed by the caller of the execution
+ * through other references! This is just a convenience method for getting that instance.
+ */
+ public CommandLine getCommandLine() { return commandLine; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java
new file mode 100644
index 00000000000..440928b5762
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java
@@ -0,0 +1,15 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * Exception thrown if the output of the child process is larger than the maximum limit.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class LargeOutputChildProcessException extends ChildProcessException {
+ LargeOutputChildProcessException(long maxFileSize, String commandLine, String possiblyHugeOutput) {
+ super("output more than " + maxFileSize + " bytes", commandLine, possiblyHugeOutput);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java
new file mode 100644
index 00000000000..006f1373e0f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java
@@ -0,0 +1,17 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Process abstraction.
+ *
+ * @author hakonhall
+ */
+public interface ProcessApi2 {
+ boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException;
+ int exitValue();
+ void destroy();
+ void destroyForcibly();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java
new file mode 100644
index 00000000000..0e4bc799007
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java
@@ -0,0 +1,36 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessApi2Impl implements ProcessApi2 {
+ private final Process process;
+
+ ProcessApi2Impl(Process process) {
+ this.process = process;
+ }
+
+ @Override
+ public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException {
+ return process.waitFor(timeout, unit);
+ }
+
+ @Override
+ public int exitValue() {
+ return process.exitValue();
+ }
+
+ @Override
+ public void destroy() {
+ process.destroy();
+ }
+
+ @Override
+ public void destroyForcibly() {
+ process.destroyForcibly();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java
new file mode 100644
index 00000000000..c09d6b543c3
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java
@@ -0,0 +1,10 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+public interface ProcessFactory {
+ ChildProcess2 spawn(CommandLine commandLine);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java
new file mode 100644
index 00000000000..f4bef260ec0
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java
@@ -0,0 +1,106 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessFactoryImpl implements ProcessFactory {
+ private static final Logger logger = Logger.getLogger(ProcessFactoryImpl.class.getName());
+ private static final File DEV_NULL = new File("/dev/null");
+
+ private final ProcessStarter processStarter;
+ private final Timer timer;
+
+ ProcessFactoryImpl(ProcessStarter processStarter, Timer timer) {
+ this.processStarter = processStarter;
+ this.timer = timer;
+ }
+
+ @Override
+ public ChildProcess2Impl spawn(CommandLine commandLine) {
+ List<String> arguments = commandLine.getArguments();
+ if (arguments.isEmpty()) {
+ throw new IllegalArgumentException("No arguments specified - missing program to spawn");
+ }
+
+ ProcessBuilder processBuilder = new ProcessBuilder(arguments);
+
+ for (var entry : commandLine.getEnvironmentOverrides().entrySet()) {
+ if (entry.getValue() == null) {
+ processBuilder.environment().remove(entry.getKey());
+ } else {
+ processBuilder.environment().put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ if (commandLine.getRedirectStderrToStdoutInsteadOfDiscard()) {
+ processBuilder.redirectErrorStream(true);
+ } else {
+ processBuilder.redirectError(ProcessBuilder.Redirect.to(DEV_NULL));
+ }
+
+ // The output is redirected to a file (temporary or user-defined) because:
+ // - We could read continuously from process.getInputStream, but that may block
+ // indefinitely with a faulty program.
+ // - If we don't read continuously from process.getInputStream, then because
+ // the underlying channel may be a pipe, the child may be stopped because the pipe
+ // is full.
+ // - To honor the timeout, no API can be used that may end up blocking indefinitely.
+ //
+ // Therefore, we redirect the output to a file and use waitFor w/timeout. This also
+ // has the benefit of allowing for inspection of the file during execution, and
+ // allowing the inspection of the file if it e.g. gets too large to hold in-memory.
+
+ FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute(
+ PosixFilePermissions.fromString("rw-------"));
+
+ Path outputFile = commandLine.getOutputFile()
+ .map(file -> {
+ uncheck(() -> Files.deleteIfExists(file));
+ uncheck(() -> Files.createFile(file, fileAttribute));
+ return file;
+ })
+ .orElseGet(() -> {
+ String temporaryFilePrefix =
+ ProcessFactoryImpl.class.getSimpleName() + "-" + commandLine.programName() + "-";
+
+ return uncheck(() -> Files.createTempFile(
+ temporaryFilePrefix,
+ ".out",
+ fileAttribute));
+ });
+
+ try {
+ processBuilder.redirectOutput(outputFile.toFile());
+ ProcessApi2 process = processStarter.start(processBuilder);
+ return new ChildProcess2Impl(commandLine, process, outputFile, timer);
+ } catch (RuntimeException | Error throwable) {
+ try {
+ if (commandLine.getOutputFile().isEmpty())
+ Files.delete(outputFile);
+ } catch (IOException ioException) {
+ logger.log(Level.WARNING, "Failed to delete temporary file at " +
+ outputFile, ioException);
+ }
+ throw throwable;
+ }
+
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java
new file mode 100644
index 00000000000..fc78b5d3e72
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java
@@ -0,0 +1,10 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+public interface ProcessStarter {
+ ProcessApi2 start(ProcessBuilder processBuilder);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java
new file mode 100644
index 00000000000..644e5876eb7
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.logging.Level;
+
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+public class ProcessStarterImpl implements ProcessStarter {
+ private static final Logger logger = Logger.getLogger(ProcessStarterImpl.class.getName());
+
+ @Override
+ public ProcessApi2 start(ProcessBuilder processBuilder) {
+ if (logger.isLoggable(Level.FINE)) {
+ logger.log(Level.FINE, "Spawning process: " + processBuilder.command());
+ }
+
+ Process process = uncheck(processBuilder::start);
+ return new ProcessApi2Impl(process);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java
new file mode 100644
index 00000000000..1cf6b533d5a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java
@@ -0,0 +1,14 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+/**
+ * A Terminal is a light-weight terminal-like interface for executing shell-like programs.
+ *
+ * @author hakonhall
+ */
+public interface Terminal {
+ CommandLine newCommandLine(TaskContext taskContext);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java
new file mode 100644
index 00000000000..e13e30d9c75
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+/**
+ * @author hakonhall
+ */
+public class TerminalImpl implements Terminal {
+ private final ProcessFactory processFactory;
+
+ public TerminalImpl(Timer timer) {
+ this(new ProcessFactoryImpl(new ProcessStarterImpl(), timer));
+ }
+
+ /** For testing. */
+ public TerminalImpl(ProcessFactory processFactory) {
+ this.processFactory = processFactory;
+ }
+
+ @Override
+ public CommandLine newCommandLine(TaskContext taskContext) {
+ return new CommandLine(taskContext, processFactory);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java
new file mode 100644
index 00000000000..8490bc01f56
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public class TestChildProcess2 implements ChildProcess2 {
+ private final int exitCode;
+ private final String output;
+ private Optional<RuntimeException> exceptionToThrowInWaitForTermination = Optional.empty();
+ private boolean closeCalled = false;
+
+ public TestChildProcess2(int exitCode, String output) {
+ this.exitCode = exitCode;
+ this.output = output;
+ }
+
+ public void throwInWaitForTermination(RuntimeException e) {
+ this.exceptionToThrowInWaitForTermination = Optional.of(e);
+ }
+
+ @Override
+ public void waitForTermination() {
+ if (exceptionToThrowInWaitForTermination.isPresent()) {
+ throw exceptionToThrowInWaitForTermination.get();
+ }
+ }
+
+ @Override
+ public int exitCode() {
+ return exitCode;
+ }
+
+ @Override
+ public String getOutput() {
+ return output;
+ }
+
+ @Override
+ public void close() {
+ if (closeCalled) {
+ throw new IllegalStateException("close already called");
+ }
+ closeCalled = true;
+ }
+
+ public boolean closeCalled() {
+ return closeCalled;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java
new file mode 100644
index 00000000000..4e831dc2865
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java
@@ -0,0 +1,113 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * @author hakonhall
+ */
+public class TestProcessFactory implements ProcessFactory {
+ private static class SpawnCall {
+ private final String commandDescription;
+ private final Function<CommandLine, ChildProcess2> callback;
+
+ private SpawnCall(String commandDescription,
+ Function<CommandLine, ChildProcess2> callback) {
+ this.commandDescription = commandDescription;
+ this.callback = callback;
+ }
+ }
+ private final List<SpawnCall> expectedSpawnCalls = new ArrayList<>();
+ private final List<CommandLine> spawnCommandLines = new ArrayList<>();
+
+ private boolean muteVerifyAllCommandsExecuted = false;
+
+ /** Forward call to spawn() to callback. */
+ public TestProcessFactory interceptSpawn(String commandDescription,
+ Function<CommandLine, ChildProcess2> callback) {
+ expectedSpawnCalls.add(new SpawnCall(commandDescription, callback));
+ return this;
+ }
+
+ // Convenience method for the caller to avoid having to create a TestChildProcess2 instance.
+ public TestProcessFactory expectSpawn(String commandLineString, TestChildProcess2 toReturn) {
+ int commandIndex = expectedSpawnCalls.size();
+ return interceptSpawn(
+ commandLineString,
+ commandLine -> defaultSpawn(commandLine, commandLineString, toReturn, commandIndex));
+ }
+
+ // Convenience method for the caller to avoid having to create a TestChildProcess2 instance.
+ public TestProcessFactory expectSpawn(String commandLine, int exitCode, String output) {
+ return expectSpawn(commandLine, new TestChildProcess2(exitCode, output));
+ }
+
+ /** Ignore the CommandLine passed to spawn(), just return successfully with the given output. */
+ public TestProcessFactory ignoreSpawn(String output) {
+ return interceptSpawn(
+ "[call index " + expectedSpawnCalls.size() + "]",
+ commandLine -> new TestChildProcess2(0, output));
+ }
+
+ public TestProcessFactory ignoreSpawn() {
+ return ignoreSpawn("");
+ }
+
+ public void verifyAllCommandsExecuted() {
+ if (muteVerifyAllCommandsExecuted) return;
+
+ if (spawnCommandLines.size() < expectedSpawnCalls.size()) {
+ int missingCommandIndex = spawnCommandLines.size();
+ throw new IllegalStateException("Command #" + missingCommandIndex +
+ " never executed: " +
+ expectedSpawnCalls.get(missingCommandIndex).commandDescription +
+ "\nExpected commands:\n" + getExpectedCommandLines() +
+ "\nActual commands:\n" + spawnCommandLines);
+ }
+ }
+
+ /**
+ * WARNING: CommandLine is mutable, and e.g. reusing a CommandLine for the next call
+ * would make the CommandLine in this list no longer reflect the original CommandLine.
+ */
+ public List<CommandLine> getMutableCommandLines() {
+ return spawnCommandLines;
+ }
+
+ @Override
+ public ChildProcess2 spawn(CommandLine commandLine) {
+ String commandLineString = commandLine.toString(false);
+ if (spawnCommandLines.size() + 1 > expectedSpawnCalls.size()) {
+ throw new IllegalStateException("Too many invocations: " + commandLineString);
+ }
+ spawnCommandLines.add(commandLine);
+
+ return expectedSpawnCalls.get(spawnCommandLines.size() - 1).callback.apply(commandLine);
+ }
+
+ private ChildProcess2 defaultSpawn(CommandLine commandLine,
+ String expectedCommandLineString,
+ ChildProcess2 toReturn,
+ int commandSequenceNumber) {
+ String actualCommandLineString = commandLine.toString(false);
+ if (!Objects.equals(actualCommandLineString, expectedCommandLineString)) {
+ muteVerifyAllCommandsExecuted = true;
+ throw new IllegalArgumentException("Expected command #" + commandSequenceNumber + " to be: \n" +
+ " \"" + expectedCommandLineString + "\"\n" +
+ "but got:\n" +
+ " \"" + actualCommandLineString + "\"");
+ }
+
+ return toReturn;
+ }
+
+ private List<String> getExpectedCommandLines() {
+ return expectedSpawnCalls.stream()
+ .map(spawnCall -> spawnCall.commandDescription)
+ .toList();
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java
new file mode 100644
index 00000000000..bf231b7c35b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java
@@ -0,0 +1,67 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.util.function.Function;
+
+/**
+ * @author hakonhall
+ */
+public class TestTerminal implements Terminal {
+ private final TerminalImpl realTerminal;
+ private final TestProcessFactory testProcessFactory = new TestProcessFactory();
+
+ public TestTerminal() {
+ this.realTerminal = new TerminalImpl(testProcessFactory);
+ }
+
+ /** Get the TestProcessFactory the terminal was started with. */
+ public TestProcessFactory getTestProcessFactory() { return testProcessFactory; }
+
+ /** Forward call to spawn() to callback. */
+ public TestTerminal interceptCommand(String commandDescription,
+ Function<CommandLine, ChildProcess2> callback) {
+ testProcessFactory.interceptSpawn(commandDescription, callback);
+ return this;
+ }
+
+ /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+ public TestTerminal expectCommand(String commandLine, TestChildProcess2 toReturn) {
+ testProcessFactory.expectSpawn(commandLine, toReturn);
+ return this;
+ }
+
+ /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+ public TestTerminal expectCommand(String commandLine, int exitCode, String output) {
+ testProcessFactory.expectSpawn(commandLine, new TestChildProcess2(exitCode, output));
+ return this;
+ }
+
+ /** Verifies command line matches commandLine, and returns successfully with output "". */
+ public TestTerminal expectCommand(String commandLine) {
+ expectCommand(commandLine, 0, "");
+ return this;
+ }
+
+ /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+ public TestTerminal ignoreCommand(String output) {
+ testProcessFactory.ignoreSpawn(output);
+ return this;
+ }
+
+ /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */
+ public TestTerminal ignoreCommand() {
+ testProcessFactory.ignoreSpawn();
+ return this;
+ }
+
+ public void verifyAllCommandsExecuted() {
+ testProcessFactory.verifyAllCommandsExecuted();
+ }
+
+ @Override
+ public CommandLine newCommandLine(TaskContext taskContext) {
+ return realTerminal.newCommandLine(taskContext);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java
new file mode 100644
index 00000000000..c4c59073de8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java
@@ -0,0 +1,18 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.time.Duration;
+
+/**
+ * Exception thrown when a child process has taken too long to terminate, in case it has been
+ * forcibly killed.
+ *
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class TimeoutChildProcessException extends ChildProcessException {
+ TimeoutChildProcessException(Duration timeout, String commandLine, String possiblyHugeOutput) {
+ super("timed out after " + timeout, commandLine, possiblyHugeOutput);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java
new file mode 100644
index 00000000000..1829df96601
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+/**
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class UnexpectedOutputException extends ChildProcessException {
+ /**
+ * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$"
+ */
+ public UnexpectedOutputException(String problem, String commandLine, String possiblyHugeOutput) {
+ super("output was not of the expected format: " + problem, commandLine, possiblyHugeOutput);
+ }
+
+ /**
+ * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$"
+ */
+ public UnexpectedOutputException(RuntimeException cause,
+ String problem,
+ String commandLine,
+ String possiblyHugeOutput) {
+ super(cause, "output was not of the expected format: " + problem, commandLine, possiblyHugeOutput);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java
new file mode 100644
index 00000000000..1b847380b47
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java
@@ -0,0 +1,21 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+@SuppressWarnings("serial")
+public class UnkillableChildProcessException extends ChildProcessException {
+ public UnkillableChildProcessException(Duration waitForSigTerm,
+ Duration waitForSigKill,
+ String commandLine,
+ String possiblyHugeOutput) {
+ super("did not terminate even after SIGTERM, +" + waitForSigTerm +
+ ", SIGKILL, and +" + waitForSigKill,
+ commandLine,
+ possiblyHugeOutput);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
new file mode 100644
index 00000000000..d03eb80af50
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java
new file mode 100644
index 00000000000..55c7b23b1e8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java
@@ -0,0 +1,227 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
+
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Control the systemd system and service manager
+ *
+ * @author hakonhall
+ */
+public class SystemCtl {
+
+ // Valid systemd property names from looking at a couple of services.
+ private static final Pattern PROPERTY_NAME_PATTERN = Pattern.compile("^[a-zA-Z]+$");
+
+ // Last line of `systemctl list-unit-files <unit>` prints '0 unit files listed.'
+ private static final Pattern UNIT_FILES_LISTED_PATTERN = Pattern.compile("([0-9]+) unit files listed\\.");
+
+ private static final Pattern ACTIVE_STATE_PROPERTY_PATTERN = createPropertyPattern("ActiveState");
+
+ private final Terminal terminal;
+ private boolean useSudo = false;
+
+ private static Pattern createPropertyPattern(String propertyName) {
+ if (!PROPERTY_NAME_PATTERN.matcher(propertyName).matches()) {
+ throw new IllegalArgumentException("Property name does not match " + PROPERTY_NAME_PATTERN);
+ }
+
+ // Make ^ and $ match beginning and end of lines.
+ String regex = String.format("(?md)^%s=(.*)$", propertyName);
+
+ return Pattern.compile(regex);
+ }
+
+ public SystemCtl(Terminal terminal) {
+ this.terminal = terminal;
+ }
+
+ /** Call all commands through sudo */
+ public SystemCtl withSudo() {
+ this.useSudo = true;
+ return this;
+ }
+
+ /** Returns whether this is configured to use sudo */
+ public boolean useSudo() {
+ return useSudo;
+ }
+
+ public void daemonReload(TaskContext taskContext) {
+ newCommandLine(taskContext).add("systemctl", "daemon-reload")
+ .execute();
+ }
+
+ public SystemCtlEnable enable(String unit) { return new SystemCtlEnable(unit); }
+ public SystemCtlDisable disable(String unit) { return new SystemCtlDisable(unit); }
+ public SystemCtlStart start(String unit) { return new SystemCtlStart(unit); }
+ public SystemCtlStop stop(String unit) { return new SystemCtlStop(unit); }
+ public SystemCtlRestart restart(String unit) { return new SystemCtlRestart(unit); }
+ public SystemCtlReload reload(String unit) { return new SystemCtlReload(unit); }
+
+ public boolean serviceExists(TaskContext context, String unit) {
+ return newCommandLine(context)
+ .add("systemctl", "list-unit-files", unit + ".service").executeSilently()
+ .mapOutput(output -> {
+ // Last line of the form: "1 unit files listed."
+ Matcher matcher = UNIT_FILES_LISTED_PATTERN.matcher(output);
+ if (!matcher.find()) {
+ throw new IllegalArgumentException();
+ }
+
+ return !matcher.group(1).equals("0");
+ });
+ }
+
+ /** Returns true if the unit exists and is active (i.e. running). unit is e.g. "docker". */
+ public boolean isActive(TaskContext context, String unit) {
+ return newCommandLine(context)
+ .add("systemctl", "--quiet", "is-active", unit + ".service")
+ .ignoreExitCode()
+ .executeSilently()
+ .map(CommandResult::getExitCode) == 0;
+ }
+
+ public String getServiceProperty(TaskContext context, String unit, String property) {
+ return newCommandLine(context)
+ .add("systemctl", "show", "--property", property, "--value", unit + ".service")
+ .executeSilently()
+ .getOutput();
+ }
+
+ private CommandLine newCommandLine(TaskContext context) {
+ var commandLine = terminal.newCommandLine(context);
+ if (useSudo) {
+ commandLine.add("sudo");
+ }
+ return commandLine;
+ }
+
+ public class SystemCtlEnable extends SystemCtlCommand {
+ private SystemCtlEnable(String unit) {
+ super("enable", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ return isUnitEnabled(context);
+ }
+ }
+
+ public class SystemCtlDisable extends SystemCtlCommand {
+ private SystemCtlDisable(String unit) {
+ super("disable", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ return !isUnitEnabled(context);
+ }
+ }
+
+ public class SystemCtlStart extends SystemCtlCommand {
+ private SystemCtlStart(String unit) {
+ super("start", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN);
+ return Objects.equals(activeState, "active");
+ }
+ }
+
+ public class SystemCtlStop extends SystemCtlCommand {
+ private SystemCtlStop(String unit) {
+ super("stop", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN);
+ return Objects.equals(activeState, "inactive");
+ }
+ }
+
+ public class SystemCtlRestart extends SystemCtlCommand {
+ private SystemCtlRestart(String unit) {
+ super("restart", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ return false;
+ }
+ }
+
+ public class SystemCtlReload extends SystemCtlCommand {
+ private SystemCtlReload(String unit) {
+ super("reload", unit);
+ }
+
+ protected boolean isAlreadyConverged(TaskContext context) {
+ return false;
+ }
+ }
+
+ public abstract class SystemCtlCommand {
+
+ private final String command;
+ private final String unit;
+
+ private SystemCtlCommand(String command, String unit) {
+ this.command = command;
+ this.unit = unit;
+ }
+
+ protected abstract boolean isAlreadyConverged(TaskContext context);
+
+ public boolean converge(TaskContext context) {
+ if (isAlreadyConverged(context)) {
+ return false;
+ }
+ newCommandLine(context).add("systemctl", command, unit)
+ .execute();
+ return true;
+ }
+
+ /** Returns true if unit is enabled */
+ boolean isUnitEnabled(TaskContext context) {
+ return newCommandLine(context).add("systemctl", "--quiet", "is-enabled", unit)
+ .ignoreExitCode()
+ .executeSilently()
+ .map(CommandResult::getExitCode) == 0;
+ }
+
+ /**
+ * @param propertyPattern Pattern to match the output of systemctl show command with
+ * exactly 1 group. The matchng group must exist.
+ * @return The matched group from the 'systemctl show' output.
+ */
+ String getSystemCtlProperty(TaskContext context, Pattern propertyPattern) {
+ return newCommandLine(context).add("systemctl", "show", unit)
+ .executeSilently()
+ .mapOutput(output -> extractProperty(output, propertyPattern));
+ }
+ }
+
+
+ /**
+ * Find the systemd property value of the property (given by propertyPattern)
+ * matching the 'systemctl show' output (given by showProcess).
+ */
+ private static String extractProperty(String showOutput, Pattern propertyPattern) {
+ Matcher matcher = propertyPattern.matcher(showOutput);
+ if (!matcher.find()) {
+ throw new IllegalArgumentException("Pattern '" + propertyPattern +
+ "' didn't match output");
+ } else if (matcher.groupCount() != 1) {
+ throw new IllegalArgumentException("Property pattern must have exactly 1 group");
+ }
+
+ return matcher.group(1);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java
new file mode 100644
index 00000000000..32da4f455c1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java
@@ -0,0 +1,91 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+
+import java.util.function.Consumer;
+
+/**
+ * A {@link SystemCtl} tester that simplifies testing interaction with systemd units.
+ *
+ * @author mpolden
+ */
+public class SystemCtlTester extends SystemCtl {
+
+ private final TestTerminal terminal;
+
+ public SystemCtlTester(TestTerminal terminal) {
+ super(terminal);
+ this.terminal = terminal;
+ }
+
+ public Expectation expectServiceExists(String unit) {
+ return new Expectation(wantedReturn ->
+ expectCommand("systemctl list-unit-files " + unit + ".service 2>&1", 0, (wantedReturn ? 1 : 0) + " unit files listed."));
+ }
+
+ public Expectation expectIsActive(String unit) {
+ return new Expectation(wantedReturn -> {
+ expectCommand("systemctl --quiet is-active " + unit + ".service 2>&1", wantedReturn ? 0 : 1, "");
+ });
+ }
+
+ public Expectation expectEnable(String unit) { return forChangeEnabledState(unit, true); }
+ public Expectation expectDisable(String unit) { return forChangeEnabledState(unit, false); }
+ public Expectation expectStart(String unit) { return forChangeRunningState(unit, true); }
+ public Expectation expectStop(String unit) { return forChangeRunningState(unit, false); }
+
+ public SystemCtlTester expectRestart(String unit) {
+ expectCommand("systemctl restart " + unit + " 2>&1", 0, "");
+ return this;
+ }
+
+ public SystemCtlTester expectReload(String unit) {
+ expectCommand("systemctl reload " + unit + " 2>&1", 0, "");
+ return this;
+ }
+
+ public SystemCtlTester expectDaemonReload() {
+ expectCommand("systemctl daemon-reload 2>&1", 0, "");
+ return this;
+ }
+
+ public SystemCtlTester expectGetServiceProperty(String unit, String property, String output) {
+ expectCommand("systemctl show --property " + property + " --value " + unit + ".service 2>&1", 0, output);
+ return this;
+ }
+
+ private void expectCommand(String command, int exitCode, String output) {
+ terminal.expectCommand((useSudo() ? "sudo " : "") + command, exitCode, output);
+ }
+
+ private Expectation forChangeEnabledState(String unit, boolean enable) {
+ return new Expectation(wantedReturn -> {
+ expectCommand("systemctl --quiet is-enabled " + unit + " 2>&1", enable != wantedReturn ? 0 : 1, "");
+ if (wantedReturn)
+ expectCommand("systemctl " + (enable ? "enable" : "disable") + " " + unit + " 2>&1", 0, "");
+ });
+ }
+
+ private Expectation forChangeRunningState(String unit, boolean start) {
+ return new Expectation(wantedReturn -> {
+ expectCommand("systemctl show " + unit + " 2>&1", 0, "ActiveState=" + (start != wantedReturn ? "active" : "inactive"));
+ if (wantedReturn)
+ expectCommand("systemctl " + (start ? "start" : "stop") + " " + unit + " 2>&1", 0, "");
+ });
+ }
+
+ public class Expectation {
+ private final Consumer<Boolean> converger;
+ public Expectation(Consumer<Boolean> converger) {
+ this.converger = converger;
+ }
+
+ /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */
+ public SystemCtlTester andReturn(boolean value) {
+ converger.accept(value);
+ return SystemCtlTester.this;
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java
new file mode 100644
index 00000000000..465cec3c026
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java
new file mode 100644
index 00000000000..2d907f79e2d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+
+/**
+ * @author hakonhall
+ */
+public class BadTemplateException extends TemplateException {
+ public BadTemplateException(Cursor location, String message) {
+ super(message + " at " + location.calculateLocation().lineAndColumnText());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java
new file mode 100644
index 00000000000..3ebac3322b4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java
@@ -0,0 +1,32 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import java.util.Locale;
+
+/**
+ * Public methods common to both Template and ListElement.
+ *
+ * @author hakonhall
+ */
+public interface Form {
+ /** Set the value of a variable, e.g. %{=color}. */
+ Template set(String name, String value);
+
+ /** Set the value of a variable and/or if-condition. */
+ default Template set(String name, boolean value) { return set(name, Boolean.toString(value)); }
+
+ default Template set(String name, int value) { return set(name, Integer.toString(value)); }
+ default Template set(String name, long value) { return set(name, Long.toString(value)); }
+
+ default Template set(String name, String format, Object first, Object... rest) {
+ var args = new Object[1 + rest.length];
+ args[0] = first;
+ System.arraycopy(rest, 0, args, 1, rest.length);
+ var value = String.format(Locale.US, format, args);
+
+ return set(name, value);
+ }
+
+ /** Add an instance of a list section after any previously added (for the given name) */
+ ListElement add(String name);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java
new file mode 100644
index 00000000000..d00b66c9b24
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java
@@ -0,0 +1,69 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+class IfSection extends Section {
+ private final boolean negated;
+ private final String name;
+ private final Cursor nameOffset;
+ private final SectionList ifSections;
+ private final Optional<SectionList> elseSections;
+
+ IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset,
+ SectionList ifSections, Optional<SectionList> elseSections) {
+ super("if", range);
+ this.negated = negated;
+ this.name = name;
+ this.nameOffset = nameOffset;
+ this.ifSections = ifSections;
+ this.elseSections = elseSections;
+ }
+
+ String name() { return name; }
+ Cursor nameOffset() { return nameOffset; }
+
+ @Override
+ void appendTo(StringBuilder buffer) {
+ Optional<String> stringValue = template().getVariableValue(name);
+ if (stringValue.isEmpty())
+ throw new TemplateNameNotSetException(name, nameOffset);
+
+ final boolean value;
+ if (stringValue.get().equals("true")) {
+ value = true;
+ } else if (stringValue.get().equals("false")) {
+ value = false;
+ } else {
+ throw new NotBooleanValueTemplateException(name);
+ }
+
+ boolean condition = negated ? !value : value;
+ if (condition) {
+ ifSections.sections().forEach(section -> section.appendTo(buffer));
+ } else if (elseSections.isPresent()) {
+ elseSections.get().sections().forEach(section -> section.appendTo(buffer));
+ }
+ }
+
+ @Override
+ void appendCopyTo(SectionList sectionList) {
+ SectionList ifSectionCopy = new SectionList(ifSections.range().start(), sectionList.templateBuilder());
+ ifSections.sections().forEach(section -> section.appendCopyTo(ifSectionCopy));
+
+ Optional<SectionList> elseSectionCopy = elseSections.map(elseSections2 -> {
+ SectionList elseSectionCopy2 = new SectionList(elseSections2.range().start(),
+ sectionList.templateBuilder());
+ elseSections2.sections().forEach(section -> section.appendCopyTo(elseSectionCopy2));
+ return elseSectionCopy2;
+ });
+
+ sectionList.appendIfSection(negated, name, nameOffset, range().end(), ifSectionCopy, elseSectionCopy);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java
new file mode 100644
index 00000000000..e8b96d4a6b8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java
@@ -0,0 +1,17 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+/**
+ * @author hakonhall
+ */
+public class ListElement implements Form {
+ private final Template template;
+
+ ListElement(Template template) { this.template = template; }
+
+ @Override
+ public Template set(String name, String value) { return template.set(name, value); }
+
+ @Override
+ public ListElement add(String name) { return new ListElement(template.addElement(name)); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java
new file mode 100644
index 00000000000..512518c3a42
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java
@@ -0,0 +1,61 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author hakonhall
+ */
+class ListSection extends Section {
+ private final String name;
+ private final Cursor nameOffset;
+ private final Template body;
+ private final List<Template> elements = new ArrayList<>();
+
+ ListSection(CursorRange range, String name, Cursor nameOffset, Template body) {
+ super("list", range);
+ this.name = name;
+ this.nameOffset = new Cursor(nameOffset);
+ this.body = body;
+ }
+
+ String name() { return name; }
+ Cursor nameOffset() { return new Cursor(nameOffset); }
+
+ @Override
+ void setTemplate(Template template) {
+ super.setTemplate(template);
+ body.setParent(template);
+ }
+
+ Template add() {
+ Template element = body.snapshot();
+ element.setParent(template());
+ elements.add(element);
+ return element;
+ }
+
+ @Override
+ void appendTo(StringBuilder buffer) {
+ elements.forEach(template -> template.appendTo(buffer));
+ }
+
+ @Override
+ void appendCopyTo(SectionList sectionList) {
+ // Optimization: Reuse body in copy, since it is only used for copying.
+
+ ListSection newSection = sectionList.appendListSection(name, nameOffset, range().end(), body);
+
+ elements.stream()
+ .map(template -> {
+ Template templateCopy = template.snapshot();
+ templateCopy.setParent(template());
+ return templateCopy;
+ })
+ .forEach(newSection.elements::add);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java
new file mode 100644
index 00000000000..50c07fd1e7e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+/**
+ * Represents a template literal section
+ *
+ * @see Template
+ * @author hakonhall
+ */
+class LiteralSection extends Section {
+ LiteralSection(CursorRange range) {
+ super("literal", range);
+ }
+
+ @Override
+ void appendTo(StringBuilder buffer) {
+ range().appendTo(buffer);
+ }
+
+ @Override
+ void appendCopyTo(SectionList sectionList) {
+ sectionList.appendLiteralSection(range().end());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java
new file mode 100644
index 00000000000..d9e7cdb4ccd
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java
@@ -0,0 +1,14 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+/**
+ * @author hakonhall
+ */
+public class NameAlreadyExistsTemplateException extends TemplateException {
+ public NameAlreadyExistsTemplateException(String name, Section first, Section second) {
+ super("The name '" + name + "' of the " + second.type() + " section at " +
+ second.range().start().calculateLocation().lineAndColumnText() +
+ " is in conflict with the identically named " + first.type() + " section at " +
+ first.range().start().calculateLocation().lineAndColumnText());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java
new file mode 100644
index 00000000000..a8020cc92d1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+/**
+ * @author hakonhall
+ */
+public class NoSuchNameTemplateException extends TemplateException {
+ public NoSuchNameTemplateException(CursorRange range, String name) {
+ super("No such element '" + name + "' in the " + describeSection(range));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java
new file mode 100644
index 00000000000..34879514cd1
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java
@@ -0,0 +1,11 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+/**
+ * @author hakonhall
+ */
+public class NotBooleanValueTemplateException extends TemplateException {
+ public NotBooleanValueTemplateException(String name) {
+ super(name + " was set to a non-boolean value: must be true or false");
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java
new file mode 100644
index 00000000000..640baae98ac
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java
@@ -0,0 +1,35 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.util.Objects;
+
+/**
+ * A section of a template text.
+ *
+ * @see Template
+ * @author hakonhall
+ */
+abstract class Section {
+ private final String type;
+ private final CursorRange range;
+ private Template template;
+
+ protected Section(String type, CursorRange range) {
+ this.type = type;
+ this.range = range;
+ }
+
+ void setTemplate(Template template) { this.template = template; }
+
+ /** Guaranteed to return non-null after TemplateBuilder::build() returns. */
+ protected Template template() { return Objects.requireNonNull(template); }
+
+ protected String type() { return type; }
+ protected CursorRange range() { return range; }
+
+ abstract void appendTo(StringBuilder buffer);
+
+ abstract void appendCopyTo(SectionList sectionList);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java
new file mode 100644
index 00000000000..5a2f5ededc2
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java
@@ -0,0 +1,69 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A mutable list of sections at the same level that can be used to build a template, e.g. the if-body.
+ *
+ * @author hakonhall
+ */
+class SectionList {
+ private final Cursor start;
+ private final Cursor end;
+ private final TemplateBuilder templateBuilder;
+
+ private final List<Section> sections = new ArrayList<>();
+
+ SectionList(Cursor start, TemplateBuilder templateBuilder) {
+ this.start = new Cursor(start);
+ this.end = new Cursor(start);
+ this.templateBuilder = templateBuilder;
+ }
+
+ CursorRange range() { return new CursorRange(start, end); }
+ TemplateBuilder templateBuilder() { return templateBuilder; }
+ List<Section> sections() { return List.copyOf(sections); }
+
+ void appendLiteralSection(Cursor end) {
+ CursorRange range = verifyAndUpdateEnd(end);
+ var section = new LiteralSection(range);
+ templateBuilder.addLiteralSection(section);
+ sections.add(section);
+ }
+
+ VariableSection appendVariableSection(String name, Cursor nameOffset, Cursor end) {
+ CursorRange range = verifyAndUpdateEnd(end);
+ var section = new VariableSection(range, name, nameOffset);
+ templateBuilder.addVariableSection(section);
+ sections.add(section);
+ return section;
+ }
+
+ void appendIfSection(boolean negated, String name, Cursor nameOffset, Cursor end,
+ SectionList ifSections, Optional<SectionList> elseSections) {
+ CursorRange range = verifyAndUpdateEnd(end);
+ var section = new IfSection(range, negated, name, nameOffset, ifSections, elseSections);
+ templateBuilder.addIfSection(section);
+ sections.add(section);
+ }
+
+ ListSection appendListSection(String name, Cursor nameOffset, Cursor end, Template body) {
+ CursorRange range = verifyAndUpdateEnd(end);
+ var section = new ListSection(range, name, nameOffset, body);
+ templateBuilder.addListSection(section);
+ sections.add(section);
+ return section;
+ }
+
+ private CursorRange verifyAndUpdateEnd(Cursor newEnd) {
+ var range = new CursorRange(this.end, newEnd);
+ this.end.set(newEnd);
+ return range;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java
new file mode 100644
index 00000000000..818da2d3403
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java
@@ -0,0 +1,114 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The Java representation of a template text.
+ *
+ * <p>A template is a sequence of literal text and dynamic sections defined by %{...} directives:</p>
+ *
+ * <pre>
+ * template: section*
+ * section: literal | variable | list
+ * literal: plain text not containing %{
+ * variable: %{=id}
+ * if: %{if [!]id}template[%{else}template]%{end}
+ * list: %{list id}template%{end}
+ * id: a valid Java identifier
+ * </pre>
+ *
+ * <p>Other directive delimiters than "%{" and "}" may be used, see {@link TemplateDescriptor}.</p>
+ *
+ * <p>Fill the template with variable values ({@link #set(String, String) set()}, set if conditions
+ * ({@link #set(String, boolean)}), add list elements ({@link #add(String) add()}, etc, and finally
+ * render it as a String ({@link #render()}).</p>
+ *
+ * <p>To reuse a template, create the template and work on snapshots of that ({@link #snapshot()}).</p>
+ *
+ * @see TemplateDescriptor
+ * @author hakonhall
+ */
+public class Template implements Form {
+ private Template parent = null;
+ private final CursorRange range;
+ private final List<Section> sections;
+
+ private final Map<String, String> values = new HashMap<>();
+ private final Map<String, ListSection> lists;
+
+ public static Template at(Path path) { return at(path, new TemplateDescriptor()); }
+ public static Template at(Path path, TemplateDescriptor descriptor) {
+ String content = new UnixPath(path).readUtf8File();
+ return Template.from(content, descriptor);
+ }
+
+ public static Template from(String text) { return from(text, new TemplateDescriptor()); }
+ public static Template from(String text, TemplateDescriptor descriptor) {
+ return TemplateParser.parse(text, descriptor).template();
+ }
+
+ Template(CursorRange range, List<Section> sections, Map<String, ListSection> lists) {
+ this.range = new CursorRange(range);
+ this.sections = List.copyOf(sections);
+ this.lists = Map.copyOf(lists);
+ }
+
+ /** Set the value of a variable, e.g. %{=color}. */
+ @Override
+ public Template set(String name, String value) {
+ values.put(name, value);
+ return this;
+ }
+
+ @Override
+ public ListElement add(String name) { return new ListElement(addElement(name)); }
+
+ public String render() {
+ var buffer = new StringBuilder((int) (range.length() * 1.2 + 128));
+ appendTo(buffer);
+ return buffer.toString();
+ }
+
+ public void appendTo(StringBuilder buffer) { sections.forEach(section -> section.appendTo(buffer)); }
+
+ /** Returns a deep copy of this. No changes to this affects the returned template, and vice versa. */
+ public Template snapshot() {
+ var builder = new TemplateBuilder(range.start());
+ sections.forEach(section -> section.appendCopyTo(builder.topLevelSectionList()));
+ Template template = builder.build();
+ values.forEach(template::set);
+ return template;
+ }
+
+ public FileWriter getFileWriterTo(Path path) {
+ String content = render();
+ return new FileWriter(path, () -> content);
+ }
+
+ /** Must be called (if there is a parent) before any other method. */
+ void setParent(Template parent) { this.parent = parent; }
+
+ Template addElement(String name) {
+ var section = lists.get(name);
+ if (section == null) {
+ throw new NoSuchNameTemplateException(range, name);
+ }
+ return section.add();
+ }
+
+ Optional<String> getVariableValue(String name) {
+ String value = values.get(name);
+ if (value != null) return Optional.of(value);
+ if (parent != null) return parent.getVariableValue(name);
+ return Optional.empty();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java
new file mode 100644
index 00000000000..05b3cce52cc
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java
@@ -0,0 +1,76 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author hakonhall
+ */
+class TemplateBuilder {
+ /** The top-level section list in this template. */
+ private final SectionList sectionList;
+ private final List<Section> allSections = new ArrayList<>();
+ private final Map<String, VariableSection> sampleVariables = new HashMap<>();
+ private final Map<String, IfSection> sampleIfSections = new HashMap<>();
+ private final Map<String, ListSection> lists = new HashMap<>();
+
+ TemplateBuilder(Cursor start) {
+ this.sectionList = new SectionList(start, this);
+ }
+
+ SectionList topLevelSectionList() { return sectionList; }
+
+ void addLiteralSection(LiteralSection section) {
+ allSections.add(section);
+ }
+
+ void addVariableSection(VariableSection section) {
+ // It's OK if the same name is used in an if-directive (as long as the value is boolean,
+ // determined when set on a template).
+
+ ListSection existing = lists.get(section.name());
+ if (existing != null)
+ throw new NameAlreadyExistsTemplateException(section.name(), existing, section);
+
+ sampleVariables.put(section.name(), section);
+ allSections.add(section);
+ }
+
+ void addIfSection(IfSection section) {
+ // It's OK if the same name is used in a variable section (as long as the value is boolean,
+ // determined when set on a template).
+
+ ListSection list = lists.get(section.name());
+ if (list != null)
+ throw new NameAlreadyExistsTemplateException(section.name(), list, section);
+
+ sampleIfSections.put(section.name(), section);
+ allSections.add(section);
+ }
+
+ void addListSection(ListSection section) {
+ VariableSection variableSection = sampleVariables.get(section.name());
+ if (variableSection != null)
+ throw new NameAlreadyExistsTemplateException(section.name(), variableSection, section);
+
+ IfSection ifSection = sampleIfSections.get(section.name());
+ if (ifSection != null)
+ throw new NameAlreadyExistsTemplateException(section.name(), ifSection, section);
+
+ ListSection previous = lists.put(section.name(), section);
+ if (previous != null)
+ throw new NameAlreadyExistsTemplateException(section.name(), previous, section);
+ allSections.add(section);
+ }
+
+ Template build() {
+ var template = new Template(sectionList.range(), sectionList.sections(), lists);
+ allSections.forEach(section -> section.setTemplate(template));
+ return template;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java
new file mode 100644
index 00000000000..657e30de084
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java
@@ -0,0 +1,42 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+/**
+ * Specifies the how to interpret a template text.
+ *
+ * @author hakonhall
+ */
+public class TemplateDescriptor {
+
+ private String startDelimiter = "%{";
+ private String endDelimiter = "}";
+ private boolean removeNewline = true;
+
+ public TemplateDescriptor() {}
+
+ public TemplateDescriptor(TemplateDescriptor that) {
+ this.startDelimiter = that.startDelimiter;
+ this.endDelimiter = that.endDelimiter;
+ this.removeNewline = that.removeNewline;
+ }
+
+ /** Use these delimiters instead of the standard "%{" and "}" to start and end a template directive. */
+ public TemplateDescriptor setDelimiters(String startDelimiter, String endDelimiter) {
+ this.startDelimiter = Token.verifyDelimiter(startDelimiter);
+ this.endDelimiter = Token.verifyDelimiter(endDelimiter);
+ return this;
+ }
+
+ /**
+ * Whether to remove a newline that immediately follows a non-variable directive. The opposite
+ * effect can be achieved by preceding the end delimiter with a "-" char, e.g. %{if foo-}.
+ */
+ public TemplateDescriptor setRemoveNewline(boolean removeNewline) {
+ this.removeNewline = removeNewline;
+ return this;
+ }
+
+ public String startDelimiter() { return startDelimiter; }
+ public String endDelimiter() { return endDelimiter; }
+ public boolean removeNewline() { return removeNewline; }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java
new file mode 100644
index 00000000000..61d13abe4ca
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java
@@ -0,0 +1,18 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+/**
+ * @author hakonhall
+ */
+public class TemplateException extends RuntimeException {
+ public TemplateException(String message) { super(message); }
+
+ protected static String describeSection(CursorRange range) {
+ var startLocation = range.start().calculateLocation();
+ var endLocation = range.end().calculateLocation();
+ return "template section starting at line " + startLocation.line() + " and column " + startLocation.column() +
+ ", and ending at line " + endLocation.line() + " and column " + endLocation.column();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java
new file mode 100644
index 00000000000..eda5e553576
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+
+/**
+ * @author hakonhall
+ */
+public class TemplateNameNotSetException extends TemplateException {
+ public TemplateNameNotSetException(String name, Cursor nameOffset) {
+ super("Variable at " + nameOffset.calculateLocation().lineAndColumnText() + " has not been set: " + name);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java
new file mode 100644
index 00000000000..814197e80ea
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java
@@ -0,0 +1,156 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+
+import java.util.EnumSet;
+import java.util.Optional;
+
+/**
+ * Parses a template String, see {@link Template} for details.
+ *
+ * @author hakonhall
+ */
+class TemplateParser {
+ private final TemplateDescriptor descriptor;
+ private final Cursor start;
+ private final Cursor current;
+ private final TemplateBuilder templateBuilder;
+
+ static TemplateParser parse(String text, TemplateDescriptor descriptor) {
+ return parse(new TemplateDescriptor(descriptor), new Cursor(text), EnumSet.of(Sentinel.EOT));
+ }
+
+ private static TemplateParser parse(TemplateDescriptor descriptor, Cursor start, EnumSet<Sentinel> sentinel) {
+ var parser = new TemplateParser(descriptor, start);
+ parser.parse(parser.templateBuilder.topLevelSectionList(), sentinel);
+ return parser;
+ }
+
+ private enum Sentinel { ELSE, END, EOT }
+
+ private TemplateParser(TemplateDescriptor descriptor, Cursor start) {
+ this.descriptor = descriptor;
+ this.start = new Cursor(start);
+ this.current = new Cursor(start);
+ this.templateBuilder = new TemplateBuilder(start);
+ }
+
+ Template template() { return templateBuilder.build(); }
+
+ private Sentinel parse(SectionList sectionList, EnumSet<Sentinel> sentinels) {
+ do {
+ current.advanceTo(descriptor.startDelimiter());
+ if (!current.equals(start)) {
+ sectionList.appendLiteralSection(current);
+ }
+
+ if (current.eot()) {
+ if (!sentinels.contains(Sentinel.EOT)) {
+ throw new BadTemplateException(start, "Missing end directive for section started");
+ }
+ return Sentinel.EOT;
+ }
+
+ Optional<Sentinel> sentinel = parseSection(sectionList, sentinels);
+ if (sentinel.isPresent()) return sentinel.get();
+ } while (true);
+ }
+
+ private Optional<Sentinel> parseSection(SectionList sectionList, EnumSet<Sentinel> sentinels) {
+ current.skip(descriptor.startDelimiter());
+
+ if (current.skip(Token.VARIABLE_DIRECTIVE_CHAR)) {
+ parseVariableSection(sectionList);
+ } else {
+ var startOfType = new Cursor(current);
+ String type = skipId().orElseThrow(() -> new BadTemplateException(current, "Missing section name"));
+
+ switch (type) {
+ case "else" -> {
+ if (!sentinels.contains(Sentinel.ELSE))
+ throw new BadTemplateException(startOfType, "Stray 'else'");
+ parseEndDirective();
+ return Optional.of(Sentinel.ELSE);
+ }
+ case "end" -> {
+ if (!sentinels.contains(Sentinel.END))
+ throw new BadTemplateException(startOfType, "Stray 'end'");
+ parseEndDirective();
+ return Optional.of(Sentinel.END);
+ }
+ case "if" -> parseIfSection(sectionList);
+ case "list" -> parseListSection(sectionList);
+ default -> throw new BadTemplateException(startOfType, "Unknown section '" + type + "'");
+ }
+ }
+
+ return Optional.empty();
+ }
+
+ private void parseVariableSection(SectionList sectionList) {
+ var nameStart = new Cursor(current);
+ String name = parseId();
+ parseEndDelimiter(false);
+ sectionList.appendVariableSection(name, nameStart, current);
+ }
+
+ private void parseEndDirective() {
+ parseEndDelimiter(true);
+ }
+
+ private void parseListSection(SectionList sectionList) {
+ skipRequiredWhitespaces();
+ var startOfName = new Cursor(current);
+ String name = parseId();
+ parseEndDelimiter(true);
+
+ TemplateParser bodyParser = parse(descriptor, current, EnumSet.of(Sentinel.END));
+ current.set(bodyParser.current);
+
+ sectionList.appendListSection(name, startOfName, current, bodyParser.templateBuilder.build());
+ }
+
+ private void parseIfSection(SectionList sectionList) {
+ skipRequiredWhitespaces();
+ boolean negated = current.skip(Token.NEGATE_CHAR);
+ current.skipWhitespaces();
+ var startOfName = new Cursor(current);
+ String name = parseId();
+ parseEndDelimiter(true);
+
+ SectionList ifSectionList = new SectionList(current, templateBuilder);
+ Sentinel ifSentinel = parse(ifSectionList, EnumSet.of(Sentinel.ELSE, Sentinel.END));
+
+ Optional<SectionList> elseSectionList = Optional.empty();
+ if (ifSentinel == Sentinel.ELSE) {
+ elseSectionList = Optional.of(new SectionList(current, templateBuilder));
+ parse(elseSectionList.get(), EnumSet.of(Sentinel.END));
+ }
+
+ sectionList.appendIfSection(negated, name, startOfName, current, ifSectionList, elseSectionList);
+ }
+
+ private void skipRequiredWhitespaces() {
+ if (!current.skipWhitespaces()) {
+ throw new BadTemplateException(current, "Expected whitespace");
+ }
+ }
+
+ private String parseId() {
+ return skipId().orElseThrow(() -> new BadTemplateException(current, "Expected identifier"));
+ }
+
+ private Optional<String> skipId() { return Token.skipId(current); }
+
+ private void parseEndDelimiter(boolean allowSkipNewline) {
+ boolean removeNewlineCharPresent = current.skip(Token.REMOVE_NEWLINE_CHAR);
+
+ if (!current.skip(descriptor.endDelimiter()))
+ throw new BadTemplateException(current, "Expected section end (" + descriptor.endDelimiter() + ")");
+
+ // The presence of the remove-newline-char means the opposite behavior is wanted.
+ if (allowSkipNewline && (removeNewlineCharPresent != descriptor.removeNewline()))
+ current.skip('\n');
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java
new file mode 100644
index 00000000000..61e9b27372c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java
@@ -0,0 +1,60 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+class Token {
+ static final char NEGATE_CHAR = '!';
+ static final char REMOVE_NEWLINE_CHAR = '-';
+ static final char VARIABLE_DIRECTIVE_CHAR = '=';
+
+ static Optional<String> skipId(Cursor cursor) {
+ if (cursor.eot() || !isIdStart(cursor.getChar())) return Optional.empty();
+
+ Cursor start = new Cursor(cursor);
+ cursor.increment();
+
+ while (!cursor.eot() && isIdPart(cursor.getChar()))
+ cursor.increment();
+
+ return Optional.of(new CursorRange(start, cursor).string());
+ }
+
+ /** A delimiter either starts a directive (e.g. %{) or ends it (e.g. }). */
+ static String verifyDelimiter(String delimiter) {
+ if (!isAsciiToken(delimiter)) {
+ throw new IllegalArgumentException("Invalid delimiter: '" + delimiter + "'");
+ }
+ return delimiter;
+ }
+
+ /** Returns true for a non-empty string with only ASCII token characters. */
+ private static boolean isAsciiToken(String string) {
+ if (string.isEmpty()) return false;
+ for (char c : string.toCharArray()) {
+ if (!isAsciiTokenChar(c)) return false;
+ }
+ return true;
+ }
+
+ /** Returns true if char is a printable ASCII character except space (isgraph(3)). */
+ private static boolean isAsciiTokenChar(char c) {
+ // 0x1F unit separator
+ // 0x20 space
+ // 0x21 !
+ // ...
+ // 0x7E ~
+ // 0x7F del
+ return 0x20 < c && c < 0x7F;
+ }
+
+ // Our identifiers are equivalent to a Java identifiers.
+ private static boolean isIdStart(char c) { return Character.isJavaIdentifierStart(c); }
+ private static boolean isIdPart(char c) { return Character.isJavaIdentifierPart(c); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java
new file mode 100644
index 00000000000..1423e9774af
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java
@@ -0,0 +1,37 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor;
+import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange;
+
+/**
+ * Represents a template variable section
+ *
+ * @see Template
+ * @author hakonhall
+ */
+class VariableSection extends Section {
+ private final String name;
+ private final Cursor nameOffset;
+
+ VariableSection(CursorRange range, String name, Cursor nameOffset) {
+ super("variable", range);
+ this.name = name;
+ this.nameOffset = nameOffset;
+ }
+
+ String name() { return name; }
+ Cursor nameOffset() { return new Cursor(nameOffset); }
+
+ @Override
+ void appendTo(StringBuilder buffer) {
+ String value = template().getVariableValue(name)
+ .orElseThrow(() -> new TemplateNameNotSetException(name, nameOffset));
+ buffer.append(value);
+ }
+
+ @Override
+ void appendCopyTo(SectionList sectionList) {
+ sectionList.appendVariableSection(name, nameOffset, range().end());
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java
new file mode 100644
index 00000000000..0618b0c09e5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java
new file mode 100644
index 00000000000..f6f2005dcab
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java
@@ -0,0 +1,165 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.text;
+
+import java.util.Objects;
+
+/**
+ * Cursor is a mutable offset into a fixed String, and useful for String parsing.
+ *
+ * @author hakonhall
+ */
+// @Mutable
+public class Cursor {
+ private final String text;
+ private int offset;
+ private TextLocation locationCache;
+
+ /** Creates a pointer to the first char of {@code text}, which is EOT if {@code text} is empty. */
+ public Cursor(String text) { this(text, 0, new TextLocation()); }
+
+ public Cursor(Cursor that) { this(that.text, that.offset, that.locationCache); }
+
+ private Cursor(String text, int offset, TextLocation location) {
+ this.text = Objects.requireNonNull(text);
+ this.offset = offset;
+ this.locationCache = Objects.requireNonNull(location);
+ }
+
+ /** Returns the substring of {@code text} starting at {@link #offset()} (to EOT). */
+ @Override
+ public String toString() { return text.substring(offset); }
+
+ public String fullText() { return text; }
+ public int offset() { return offset; }
+ public boolean bot() { return offset == 0; }
+ public boolean eot() { return offset == text.length(); }
+ public boolean startsWith(char c) { return offset < text.length() && text.charAt(offset) == c; }
+ public boolean startsWith(String prefix) { return text.startsWith(prefix, offset); }
+
+ /** @throws IndexOutOfBoundsException if {@link #eot()}. */
+ public char getChar() { return text.charAt(offset); }
+
+ /** The number of chars between pointer and EOT. */
+ public int length() { return text.length() - offset; }
+
+ /** Calculate the current text location in O(length(text)). */
+ public TextLocation calculateLocation() {
+ if (offset < locationCache.offset()) {
+ locationCache = new TextLocation();
+ } else if (offset == locationCache.offset()) {
+ return locationCache;
+ }
+
+ int lineIndex = locationCache.lineIndex();
+ int columnIndex = locationCache.columnIndex();
+ for (int i = locationCache.offset(); i < offset; ++i) {
+ if (text.charAt(i) == '\n') {
+ ++lineIndex;
+ columnIndex = 0;
+ } else {
+ ++columnIndex;
+ }
+ }
+
+ locationCache = new TextLocation(offset, lineIndex, columnIndex);
+ return locationCache;
+ }
+
+ public void set(Cursor that) {
+ if (that.text != text) {
+ throw new IllegalArgumentException("'that' doesn't refer to the same text");
+ }
+
+ this.offset = that.offset;
+ }
+
+ /** Advance substring.length() if this startsWith the substring, returning true if so. */
+ public boolean skip(String substring) {
+ if (startsWith(substring)) {
+ offset += substring.length();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public boolean skip(char c) {
+ if (startsWith(c)) {
+ ++offset;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /** If the current char is a whitespace, skip it and return true. */
+ public boolean skipWhitespace() {
+ if (!eot() && Character.isWhitespace(getChar())) {
+ ++offset;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /** Returns true if at least one whitespace was skipped. */
+ public boolean skipWhitespaces() {
+ if (skipWhitespace()) {
+ while (skipWhitespace())
+ ++offset;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /** Return false if eot(), otherwise advance to the next char and return true. */
+ public boolean increment() {
+ if (eot()) return false;
+ ++offset;
+ return true;
+ }
+
+ /**
+ * Advance {@code distance} chars until bot() or eot() is reached (distance may be negative),
+ * and return true if this cursor moved the full distance.
+ */
+ public boolean advance(int distance) {
+ int newOffset = offset + distance;
+ if (newOffset < 0) {
+ this.offset = 0;
+ return false;
+ } else if (newOffset > text.length()) {
+ this.offset = text.length();
+ return false;
+ } else {
+ this.offset = newOffset;
+ return true;
+ }
+ }
+
+ /** Advance pointer until start of needle is found (and return true), or EOT is reached (and return false). */
+ public boolean advanceTo(String needle) {
+ int index = text.indexOf(needle, offset);
+ if (index == -1) {
+ offset = text.length();
+ return false; // and eot() is true
+ } else {
+ offset = index;
+ return true; // and eot() is false
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Cursor cursor = (Cursor) o;
+ return offset == cursor.offset && text.equals(cursor.text);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(text, offset);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java
new file mode 100644
index 00000000000..b70aabbb1ec
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java
@@ -0,0 +1,38 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.text;
+
+/**
+ * A start- and end- offset in an underlying String.
+ *
+ * @author hakonhall
+ */
+public class CursorRange {
+ private final Cursor start;
+ private final Cursor end;
+
+ @SuppressWarnings("StringEquality")
+ public CursorRange(Cursor start, Cursor end) {
+ if (start.fullText() != end.fullText()) {
+ throw new IllegalArgumentException("start and end points to different texts");
+ }
+
+ if (start.offset() > end.offset()) {
+ throw new IllegalArgumentException("start offset " + start.offset() +
+ " is beyond end offset " + end.offset());
+ }
+
+ this.start = new Cursor(start);
+ this.end = new Cursor(end);
+ }
+
+ public CursorRange(CursorRange that) {
+ this.start = new Cursor(that.start);
+ this.end = new Cursor(that.end);
+ }
+
+ public Cursor start() { return new Cursor(start); }
+ public Cursor end() { return new Cursor(end); }
+ public int length() { return end.offset() - start.offset(); }
+ public String string() { return start.fullText().substring(start.offset(), end.offset()); }
+ public void appendTo(StringBuilder buffer) { buffer.append(start.fullText(), start.offset(), end.offset()); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java
new file mode 100644
index 00000000000..3d47a782103
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java
@@ -0,0 +1,30 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.text;
+
+/**
+ * The location within an implied multi-line String.
+ *
+ * @author hakonhall
+ */
+//@Immutable
+public class TextLocation {
+ private final int offset;
+ private final int lineIndex;
+ private final int columnIndex;
+
+ public TextLocation() { this(0, 0, 0); }
+
+ public TextLocation(int offset, int lineIndex, int columnIndex) {
+ this.offset = offset;
+ this.lineIndex = lineIndex;
+ this.columnIndex = columnIndex;
+ }
+
+ public int offset() { return offset; }
+ public int lineIndex() { return lineIndex; }
+ public int line() { return lineIndex + 1; }
+ public int columnIndex() { return columnIndex; }
+ public int column() { return columnIndex + 1; }
+
+ public String lineAndColumnText() { return "line " + line() + " and column " + column(); }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java
new file mode 100644
index 00000000000..efda4639579
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.text;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
new file mode 100644
index 00000000000..de89ad1489d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -0,0 +1,85 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
+
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.DeleteVersionLockYumCommand;
+import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.GenericYumCommand;
+import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.InstallFixedYumCommand;
+
+/**
+ * @author hakonhall
+ */
+public class Yum {
+
+ private final Terminal terminal;
+
+ public Yum(Terminal terminal) {
+ this.terminal = terminal;
+ }
+
+ public Optional<YumPackageName> queryInstalled(TaskContext context, String packageName) {
+ return YumCommand.queryInstalled(terminal, context, YumPackageName.fromString(packageName));
+ }
+
+ /** Lock and install, or if necessary downgrade, a package to a given version. */
+ public InstallFixedYumCommand installFixedVersion(YumPackageName yumPackage) {
+ return new InstallFixedYumCommand(terminal, yumPackage);
+ }
+
+ public GenericYumCommand install(YumPackageName... packages) {
+ return new GenericYumCommand(terminal, GenericYumCommand.CommandType.install, List.of(packages));
+ }
+
+ public GenericYumCommand install(String package1, String... packages) {
+ return install(toYumPackageNameArray(package1, packages));
+ }
+
+ public GenericYumCommand install(List<String> packages) {
+ return install(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+ }
+
+
+ public GenericYumCommand upgrade(YumPackageName... packages) {
+ return new GenericYumCommand(terminal, GenericYumCommand.CommandType.upgrade, List.of(packages));
+ }
+
+ public GenericYumCommand upgrade(String package1, String... packages) {
+ return upgrade(toYumPackageNameArray(package1, packages));
+ }
+
+ public GenericYumCommand upgrade(List<String> packages) {
+ return upgrade(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+ }
+
+
+ public GenericYumCommand remove(YumPackageName... packages) {
+ return new GenericYumCommand(terminal, GenericYumCommand.CommandType.remove, List.of(packages));
+ }
+
+ public GenericYumCommand remove(String package1, String... packages) {
+ return remove(toYumPackageNameArray(package1, packages));
+ }
+
+ public GenericYumCommand remove(List<String> packages) {
+ return remove(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+ }
+
+ public YumCommand.DeleteVersionLockYumCommand deleteVersionLock(YumPackageName yumPackage) {
+ return new DeleteVersionLockYumCommand(terminal, yumPackage);
+ }
+
+ static YumPackageName[] toYumPackageNameArray(String package1, String... packages) {
+ YumPackageName[] array = new YumPackageName[1 + packages.length];
+ array[0] = YumPackageName.fromString(package1);
+ for (int i = 0; i < packages.length; ++i) {
+ array[1 + i] = YumPackageName.fromString(packages[i]);
+ }
+ return array;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
new file mode 100644
index 00000000000..64b637fdd5b
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
@@ -0,0 +1,305 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+/**
+ * @author freva
+ */
+public abstract class YumCommand<T extends YumCommand<T>> {
+
+ // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find()
+ public static final Pattern INSTALL_NOOP_PATTERN = Pattern.compile("(?dm)^Nothing to do\\.?$");
+ public static final Pattern UPGRADE_NOOP_PATTERN = Pattern.compile("(?dm)^No packages marked for update$");
+ public static final Pattern REMOVE_NOOP_PATTERN = Pattern.compile("(?dm)^No [pP]ackages marked for removal\\.?$");
+
+ // WARNING: These must be in the same order as the supplier below
+ private static final String RPM_QUERYFORMAT = Stream.of("NAME", "EPOCH", "VERSION", "RELEASE", "ARCH")
+ .map(formatter -> "%{" + formatter + "}")
+ .collect(Collectors.joining("\\n"));
+ private static final Function<YumPackageName.Builder, List<Function<String, YumPackageName.Builder>>>
+ PACKAGE_NAME_BUILDERS_GENERATOR = builder -> List.of(
+ builder::setName, builder::setEpoch, builder::setVersion, builder::setRelease, builder::setArchitecture);
+
+ private List<String> disabledRepos = List.of();
+ private List<String> enabledRepos = List.of();
+ private final Terminal terminal;
+
+ protected YumCommand(Terminal terminal) {
+ this.terminal = terminal;
+ }
+
+ /** Disables the given repos for this command */
+ public T disableRepo(String... repo) {
+ disabledRepos = List.of(repo);
+ return getThis();
+ }
+
+ /** Enables the given repos for this command */
+ public T enableRepo(String... repo) {
+ enabledRepos = List.of(repo);
+ return getThis();
+ }
+
+ protected abstract T getThis(); // Hack to get around unchecked cast warning
+
+ protected void addParametersToCommandLine(CommandLine commandLine) {
+ commandLine.add("--assumeyes");
+ disabledRepos.forEach(repo -> commandLine.add("--disablerepo=" + repo));
+ enabledRepos.forEach(repo -> commandLine.add("--enablerepo=" + repo));
+ }
+
+ public abstract boolean converge(TaskContext context);
+
+ public static class GenericYumCommand extends YumCommand<GenericYumCommand> {
+ private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$");
+
+ private final Terminal terminal;
+ private final CommandType yumCommand;
+ private final List<YumPackageName> packages;
+ private final List<String> options = new ArrayList<>();
+
+ GenericYumCommand(Terminal terminal, CommandType yumCommand, List<YumPackageName> packages) {
+ super(terminal);
+ this.terminal = terminal;
+ this.yumCommand = yumCommand;
+ this.packages = packages;
+
+ switch (yumCommand) {
+ case install: {
+ if (packages.size() > 1) options.add("skip_missing_names_on_install=False");
+ break;
+ }
+ case upgrade: {
+ if (packages.size() > 1) options.add("skip_missing_names_on_update=False");
+ break;
+ }
+ case remove: break;
+ default: throw new IllegalArgumentException("Unknown yum command: " + yumCommand);
+ }
+
+ if (packages.isEmpty() && yumCommand != CommandType.upgrade)
+ throw new IllegalArgumentException("No packages specified");
+ }
+
+ @Override
+ protected void addParametersToCommandLine(CommandLine commandLine) {
+ super.addParametersToCommandLine(commandLine);
+ options.forEach(option -> commandLine.add("--setopt", option));
+ }
+
+ @Override
+ public boolean converge(TaskContext context) {
+ if (yumCommand == CommandType.install)
+ if (packages.stream().allMatch(pkg -> isInstalled(context, pkg))) return false;
+ if (yumCommand == CommandType.remove)
+ if (packages.stream().noneMatch(pkg -> isInstalled(context, pkg))) return false;
+
+ CommandLine commandLine = terminal.newCommandLine(context);
+ commandLine.add("yum", yumCommand.name());
+ addParametersToCommandLine(commandLine);
+ commandLine.add(packages.stream().map(pkg -> pkg.toName()).toList());
+
+ // There's no way to figure out whether a yum command would have been a no-op.
+ // Therefore, run the command and parse the output to decide.
+ boolean modifiedSystem = commandLine
+ .executeSilently()
+ .mapOutput(this::packageChanged);
+
+ if (modifiedSystem) {
+ commandLine.recordSilentExecutionAsSystemModification();
+ }
+
+ return modifiedSystem;
+ }
+
+ private boolean packageChanged(String output) {
+ Matcher unknownPackageMatcher = UNKNOWN_PACKAGE_PATTERN.matcher(output);
+ if (unknownPackageMatcher.find()) {
+ throw new IllegalArgumentException("Unknown package: " + unknownPackageMatcher.group(1));
+ }
+
+ return yumCommand.outputNoopPatterns.stream().noneMatch(pattern -> pattern.matcher(output).find());
+ }
+
+ protected GenericYumCommand getThis() { return this; }
+
+ enum CommandType {
+ install(INSTALL_NOOP_PATTERN), remove(REMOVE_NOOP_PATTERN), upgrade(INSTALL_NOOP_PATTERN, UPGRADE_NOOP_PATTERN);
+
+ private final List<Pattern> outputNoopPatterns;
+ CommandType(Pattern... outputNoopPatterns) {
+ this.outputNoopPatterns = List.of(outputNoopPatterns);
+ }
+ }
+ }
+
+
+ public static class InstallFixedYumCommand extends YumCommand<InstallFixedYumCommand> {
+ // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find()
+ private static final Pattern CHECKING_FOR_UPDATE_PATTERN =
+ Pattern.compile("(?dm)^Package matching [^ ]+ already installed\\. Checking for update\\.$");
+
+ private final Terminal terminal;
+ private final YumPackageName yumPackage;
+
+ InstallFixedYumCommand(Terminal terminal, YumPackageName yumPackage) {
+ super(terminal);
+ this.terminal = terminal;
+ this.yumPackage = yumPackage;
+ }
+
+ @Override
+ public boolean converge(TaskContext context) {
+ String targetVersionLockName = yumPackage.toVersionLockName();
+
+ boolean alreadyLocked = false;
+ Optional<String> versionLock = versionLockExists(context, terminal, yumPackage);
+ if (versionLock.isPresent()) {
+ if (versionLock.get().equals(targetVersionLockName)) {
+ alreadyLocked = true;
+ } else {
+ YumCommand.deleteVersionLock(context, terminal, versionLock.get());
+ }
+ }
+
+ boolean modified = false;
+
+ if (!alreadyLocked) {
+ CommandLine commandLine = terminal.newCommandLine(context).add("yum", "versionlock", "add");
+ // If the targetVersionLockName refers to a package in a by-default-disabled repo,
+ // we must enable the repo unless targetVersionLockName is already installed.
+ // The other versionlock commands (list, delete) does not require --enablerepo.
+ addParametersToCommandLine(commandLine);
+ commandLine.add(targetVersionLockName).execute();
+ modified = true;
+ }
+
+ // The following 3 things may happen with yum install:
+ // 1. The package is installed or upgraded to the target version, in case we'd return
+ // true from converge()
+ // 2. The package is already installed at target version, in case
+ // "Nothing to do" is printed in the last line and we may return false from converge()
+ // 3. The package is already installed but at a later version than the target version,
+ // in case the last 2 lines of the output is:
+ // - "Package matching yakl-client-0.10-654.el7.x86_64 already installed. Checking for update."
+ // - "Nothing to do"
+ // And in case we need to downgrade and return true from converge()
+
+ var installCommand = terminal.newCommandLine(context).add("yum", "install");
+ addParametersToCommandLine(installCommand);
+ installCommand.add(yumPackage.toName());
+
+ String output = installCommand.executeSilently().getUntrimmedOutput();
+
+ if (INSTALL_NOOP_PATTERN.matcher(output).find()) {
+ if (CHECKING_FOR_UPDATE_PATTERN.matcher(output).find()) {
+ // case 3.
+ var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade");
+ addParametersToCommandLine(upgradeCommand);
+ upgradeCommand.add(yumPackage.toName()).execute();
+ modified = true;
+ } else {
+ // case 2.
+ }
+ } else {
+ // case 1.
+ installCommand.recordSilentExecutionAsSystemModification();
+ modified = true;
+ }
+
+ return modified;
+ }
+
+ protected InstallFixedYumCommand getThis() { return this; }
+ }
+
+ public static class DeleteVersionLockYumCommand extends YumCommand<DeleteVersionLockYumCommand> {
+ private final Terminal terminal;
+ private final YumPackageName yumPackage;
+
+ DeleteVersionLockYumCommand(Terminal terminal, YumPackageName yumPackage) {
+ super(terminal);
+ this.terminal = terminal;
+ this.yumPackage = yumPackage;
+ }
+
+ @Override
+ public boolean converge(TaskContext context) {
+ return deleteVersionLock(context, terminal, yumPackage.toName());
+ }
+
+ protected DeleteVersionLockYumCommand getThis() { return this; }
+ }
+
+ protected boolean isInstalled(TaskContext context, YumPackageName yumPackage) {
+ return queryInstalled(terminal, context, yumPackage).map(yumPackage::isSubsetOf).orElse(false);
+ }
+
+ static Optional<YumPackageName> queryInstalled(Terminal terminal, TaskContext context, YumPackageName yumPackage) {
+ String packageName = yumPackage.toName();
+ CommandResult commandResult = terminal.newCommandLine(context)
+ .add("rpm", "-q", packageName, "--queryformat", RPM_QUERYFORMAT)
+ .ignoreExitCode()
+ .executeSilently();
+
+ if (commandResult.getExitCode() != 0) return Optional.empty();
+
+ YumPackageName.Builder builder = new YumPackageName.Builder();
+ List<Function<String, YumPackageName.Builder>> builders = PACKAGE_NAME_BUILDERS_GENERATOR.apply(builder);
+ List<Optional<String>> lines = commandResult.mapEachLine(line -> Optional.of(line).filter(s -> !"(none)".equals(s)));
+ if (lines.size() % builders.size() != 0) throw new IllegalStateException(String.format("Unexpected response from rpm, expected %d lines, got '%s'", builders.size(), commandResult.getOutput()));
+ if (lines.size() > builders.size()) throw new IllegalArgumentException("Found multiple installed packages for '" + packageName + "'. Version is required to match package exactly");
+
+ IntStream.range(0, builders.size()).forEach(i -> lines.get(i).ifPresent(builders.get(i)::apply));
+ if (builder.epoch().isEmpty()) builder.setEpoch("0");
+
+ return Optional.of(builder.build());
+ }
+
+ private static Optional<String> versionLockExists(TaskContext context, Terminal terminal, YumPackageName yumPackage) {
+
+ List<String> command = new ArrayList<>(4);
+ command.add("yum");
+ command.add("versionlock");
+ command.add("list");
+
+ return terminal
+ .newCommandLine(context)
+ .add(command)
+ .executeSilently()
+ .getOutputLinesStream()
+ .map(YumPackageName::parseString)
+ .filter(Optional::isPresent) // removes garbage first lines, even with --quiet
+ .map(Optional::get)
+ // Ignore lines for other packages
+ .filter(packageName -> packageName.getName().equals(yumPackage.getName()))
+ // If existing lock doesn't exactly match the full package name,
+ // it means it's locked to another version and we must remove that lock.
+ .map(YumPackageName::toVersionLockName)
+ .findFirst();
+ }
+
+ private static boolean deleteVersionLock(TaskContext context, Terminal terminal, String wildcardEntry) {
+ // Idempotent command, gives exit code 0 also when versionlock does not exist
+ terminal.newCommandLine(context)
+ .add("yum", "versionlock", "delete", wildcardEntry)
+ .execute()
+ .getOutputLinesStream();
+ return true;
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
new file mode 100644
index 00000000000..7fac5c57d06
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
@@ -0,0 +1,245 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.google.common.base.Strings;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * A YUM/DNF package name.
+ *
+ * <p>From yum(8): YUM package names are used with install, update, remove, list, info etc
+ * with any of the following as well as globs of any of the following, with any of the
+ * following as well as globs of any of the following:
+ *
+ * <ol>
+ * <li>name
+ * <li>name.arch
+ * <li>name-ver
+ * <li>name-ver-rel
+ * <li>name-ver-rel.arch
+ * <li>name-epoch:ver-rel.arch
+ * <li>epoch:name-ver-rel.arch
+ * </ol>
+ *
+ * <p>However this specification is terribly ambiguous. This class allows constructing
+ * a package name from its components, which is beneficial because with certain YUM
+ * commands that needs to canonicalize names (e.g. versionlock).
+ *
+ * @author hakonhall
+ */
+public class YumPackageName {
+
+ private enum Architecture { noarch, x86_64, i386, i586, i686 }
+
+ private static final String ARCHITECTURES_OR =
+ Arrays.stream(Architecture.values()).map(Architecture::name).collect(Collectors.joining("|"));
+ private static final Pattern ARCHITECTURE_PATTERN = Pattern.compile("\\.(" + ARCHITECTURES_OR + "|\\*)$");
+ private static final Pattern EPOCH_PATTERN = Pattern.compile("^((.+)-)?([0-9]+)$");
+ private static final Pattern NAME_VER_REL_PATTERN =
+ Pattern.compile("^(.+?)?" + // name
+ "-([+()a-z0-9._]*[0-9]+\\.[a-z0-9._]*)" + // ver: contains at least one digit and dot
+ "(?:-([+()a-z0-9._]*[0-9][a-z0-9._]*))?$"); // rel: contains at least one digit
+ private static final Pattern NAME_PATTERN = Pattern.compile("^[+()a-zA-Z0-9._-]+$");
+
+ private final Optional<String> epoch;
+ private final String name;
+ private final Optional<String> version;
+ private final Optional<String> release;
+ private final Optional<String> architecture;
+
+ public static class Builder {
+ private Optional<String> epoch = Optional.empty();
+ private String name;
+ private Optional<String> version = Optional.empty();
+ private Optional<String> release = Optional.empty();
+ private Optional<String> architecture = Optional.empty();
+
+ public Builder() { }
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ public Builder(YumPackageName packageName) {
+ epoch = packageName.epoch;
+ name = packageName.name;
+ version = packageName.version;
+ release = packageName.release;
+ architecture = packageName.architecture;
+ }
+
+ public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; }
+ public Builder setName(String name) { this.name = name; return this; }
+ public Builder setVersion(String version) { this.version = Optional.of(version); return this; }
+ public Builder setRelease(String release) { this.release = Optional.of(release); return this; }
+ public Builder setArchitecture(String architecture) { this.architecture = Optional.of(architecture); return this; }
+
+ public Optional<String> epoch() { return epoch; }
+ public String name() { return name; }
+ public Optional<String> version() { return version; }
+ public Optional<String> release() { return release; }
+ public Optional<String> architecture() { return architecture; }
+
+ public YumPackageName build() { return new YumPackageName(epoch, name, version, release, architecture); }
+ }
+
+ /** @see Builder */
+ private YumPackageName(Optional<String> epoch,
+ String name,
+ Optional<String> version,
+ Optional<String> release,
+ Optional<String> architecture) {
+ if (Strings.isNullOrEmpty(name))
+ throw new IllegalArgumentException("name cannot be null or empty");
+ this.epoch = epoch;
+ this.name = name;
+ this.version = version;
+ this.release = release;
+ this.architecture = architecture;
+ }
+
+ /**
+ * Parse the string specification of a YUM package.
+ *
+ * <p>The following formats are supported:
+ *
+ * <ol>
+ * <li>name
+ * <li>name.arch
+ * <li>name-ver
+ * <li>name-ver-rel
+ * <li>name-ver-rel.arch
+ * <li>name-epoch:ver-rel.arch
+ * <li>epoch:name-ver-rel.arch
+ * </ol>
+ *
+ * @throws IllegalArgumentException if spec does not specify a package name.
+ * @see #parseString(String)
+ */
+ public static YumPackageName fromString(String packageSpec) {
+ String spec = packageSpec;
+ Optional<String> epoch = Optional.empty();
+ String name = null;
+
+ // Parse epoch and remove it from spec
+ int epochColon = spec.indexOf(':');
+ if (epochColon >= 0) {
+ Matcher epochMatcher = EPOCH_PATTERN.matcher(spec.substring(0, epochColon));
+ if (!epochMatcher.find()) {
+ throw new IllegalArgumentException("Unexpected epoch format: " + packageSpec);
+ }
+
+ name = epochMatcher.group(2);
+ epoch = Optional.of(epochMatcher.group(3));
+
+ spec = epochColon == 0
+ ? spec.substring(epochColon + 1)
+ : spec.substring(0, epochColon - 1) + spec.substring(epochColon + 1);
+ }
+
+ // Parse architecture and remove it from spec
+ Optional<String> architecture = Optional.empty();
+ Matcher architectureMatcher = ARCHITECTURE_PATTERN.matcher(spec);
+ if (architectureMatcher.find()) {
+ architecture = Optional.of(architectureMatcher.group(1));
+ spec = spec.substring(0, architectureMatcher.start());
+ }
+
+ // Parse name, version and release and remove the latter two from spec
+ Optional<String> version = Optional.empty();
+ Optional<String> release = Optional.empty();
+ Matcher matcher = NAME_VER_REL_PATTERN.matcher(spec);
+ if (matcher.find()) {
+ spec = matcher.group(1);
+ if (spec == null) {
+ if (name == null) {
+ throw new IllegalArgumentException("No package name was found: " + packageSpec);
+ }
+ spec = name; // makes spec hold the package name in all cases below.
+ }
+
+ version = Optional.of(matcher.group(2));
+ release = Optional.ofNullable(matcher.group(3));
+ }
+
+ if (!NAME_PATTERN.matcher(spec).find()) {
+ throw new IllegalArgumentException("Bad package name in " + packageSpec + ": '" + spec + "'");
+ }
+ name = spec;
+
+ return new YumPackageName(epoch, name, version, release, architecture);
+ }
+
+ /** See {@link #fromString(String)}. */
+ public static Optional<YumPackageName> parseString(final String packageSpec) {
+ try {
+ return Optional.of(fromString(packageSpec));
+ } catch (IllegalArgumentException e) {
+ return Optional.empty();
+ }
+ }
+
+ public Optional<String> getEpoch() { return epoch; }
+ public String getName() { return name; }
+ public Optional<String> getVersion() { return version; }
+ public Optional<String> getRelease() { return release; }
+ public Optional<String> getArchitecture() { return architecture; }
+
+ /** Return package name, omitting components that are not specified. */
+ public String toName() {
+ StringBuilder builder = new StringBuilder();
+ char nextDelimiter;
+ builder.append(name);
+ // Fully versioned package names must always include epoch in Yum 4
+ epoch.or(() -> Optional.of("0").filter(v -> version.isPresent()))
+ .ifPresent(ep -> builder.append('-').append(ep));
+ nextDelimiter = ':';
+ version.ifPresent(s -> builder.append(nextDelimiter).append(s));
+ release.ifPresent(s -> builder.append('-').append(s));
+ architecture.ifPresent(arch -> builder.append('.').append(arch));
+ return builder.toString();
+ }
+
+ /**
+ * The package name output by 'yum versionlock list'. Can also be used with 'add' and 'delete'.
+ *
+ * @throws IllegalStateException if any field required for the version lock spec is missing
+ */
+ public String toVersionLockName() {
+ YumPackageName lockSpec = new Builder(this).setArchitecture("*").build();
+ if (lockSpec.getVersion().isEmpty()) throw new IllegalStateException("Version is missing for YUM package " + name);
+ if (lockSpec.getRelease().isEmpty()) throw new IllegalStateException("Release is missing for YUM package " + name);
+ return lockSpec.toName();
+ }
+
+ public boolean isSubsetOf(YumPackageName other) {
+ return Objects.equals(name, other.name) &&
+ (epoch.isEmpty() || Objects.equals(epoch, other.epoch)) &&
+ (version.isEmpty() || Objects.equals(version, other.version)) &&
+ (release.isEmpty() || Objects.equals(release, other.release)) &&
+ (architecture.isEmpty() || Objects.equals(architecture, other.architecture));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ YumPackageName that = (YumPackageName) o;
+ return Objects.equals(epoch, that.epoch) &&
+ Objects.equals(name, that.name) &&
+ Objects.equals(version, that.version) &&
+ Objects.equals(release, that.release) &&
+ Objects.equals(architecture, that.architecture);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(epoch, name, version, release, architecture);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
new file mode 100644
index 00000000000..d4fc670e43a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
@@ -0,0 +1,178 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A {@link Yum} tester that simplifies testing interaction with yum.
+ *
+ * @author freva
+ */
+public class YumTester extends Yum {
+
+ private final TestTerminal terminal;
+
+ public YumTester(TestTerminal terminal) {
+ super(terminal);
+ this.terminal = terminal;
+ }
+
+ public GenericYumCommandExpectation expectInstall(String... packages) {
+ return new GenericYumCommandExpectation(CommandType.install, packages);
+ }
+
+ public GenericYumCommandExpectation expectUpdate(String... packages) {
+ return new GenericYumCommandExpectation(CommandType.upgrade, packages);
+ }
+
+ public GenericYumCommandExpectation expectRemove(String... packages) {
+ return new GenericYumCommandExpectation(CommandType.remove, packages);
+ }
+
+ public InstallFixedCommandExpectation expectInstallFixedVersion(String yumPackage) {
+ return new InstallFixedCommandExpectation(yumPackage);
+ }
+
+ public DeleteVersionLockCommandExpectation expectDeleteVersionLock(String yumPackage) {
+ return new DeleteVersionLockCommandExpectation(yumPackage);
+ }
+
+ public QueryInstalledExpectation expectQueryInstalled(String packageName) {
+ return new QueryInstalledExpectation(packageName);
+ }
+
+
+ public class GenericYumCommandExpectation {
+ private final CommandType commandType;
+ protected final List<YumPackageName> packages;
+ private List<String> disableRepos = List.of();
+ private List<String> enableRepos = List.of();
+
+ private GenericYumCommandExpectation(CommandType commandType, String... packages) {
+ this.commandType = commandType;
+ this.packages = Stream.of(packages).map(YumPackageName::fromString).toList();
+ }
+
+ public GenericYumCommandExpectation withDisableRepo(String... repo) {
+ this.disableRepos = List.of(repo);
+ return this;
+ }
+
+ public GenericYumCommandExpectation withEnableRepo(String... repo) {
+ this.enableRepos = List.of(repo);
+ return this;
+ }
+
+ /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */
+ public YumTester andReturn(boolean value) {
+ if (value) return execute("Success");
+ return switch (commandType) {
+ case deleteVersionLock, installFixed, install -> execute("Nothing to do");
+ case upgrade -> execute("No packages marked for update");
+ case remove -> execute("No Packages marked for removal");
+ default -> throw new IllegalArgumentException("Unknown command type: " + commandType);
+ };
+ }
+
+ private YumTester execute(String output) {
+ if (commandType == CommandType.install)
+ terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(1, "Not installed"));
+ if (commandType == CommandType.remove) { // Pretend the first package is installed so we can continue to yum commands
+ YumPackageName pkg = packages.get(0);
+ terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(0, String.join("\n",
+ pkg.getName(),
+ pkg.getEpoch().orElse("(none)"),
+ pkg.getVersion().orElse("1.2.3"),
+ pkg.getRelease().orElse("1"),
+ pkg.getArchitecture().orElse("(none)"))));
+ }
+
+ StringBuilder cmd = new StringBuilder();
+ cmd.append("yum ").append(commandType.command);
+ if (commandType != CommandType.deleteVersionLock) {
+ cmd.append(" --assumeyes");
+ // mimic maybeEscapeArgument() in CommandLine
+ disableRepos.forEach(repo -> {
+ if (repo.equals("*")) cmd.append(" \"--disablerepo=").append(repo).append("\"");
+ else cmd.append(" --disablerepo=").append(repo);
+ });
+ enableRepos.forEach(repo -> cmd.append(" --enablerepo=").append(repo));
+ }
+ if (commandType == CommandType.install && packages.size() > 1)
+ cmd.append(" --setopt skip_missing_names_on_install=False");
+ if (commandType == CommandType.upgrade && packages.size() > 1)
+ cmd.append(" --setopt skip_missing_names_on_update=False");
+ packages.forEach(pkg -> {
+ String name = pkg.toName();
+ if (name.contains("(") || name.contains(")")) { // Ugly hack to handle implicit quoting done in com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine
+ name = "\"" + name + "\"";
+ }
+ cmd.append(" ").append(name);
+ });
+ cmd.append(" 2>&1");
+
+ terminal.expectCommand(cmd.toString(), 0, output);
+ return YumTester.this;
+ }
+ }
+
+ public class InstallFixedCommandExpectation extends GenericYumCommandExpectation {
+
+ private InstallFixedCommandExpectation(String yumPackage) {
+ super(CommandType.installFixed, yumPackage);
+ }
+
+ @Override
+ public YumTester andReturn(boolean value) {
+ terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName());
+ return super.andReturn(value);
+ }
+
+ }
+
+ public class DeleteVersionLockCommandExpectation extends GenericYumCommandExpectation {
+
+ private DeleteVersionLockCommandExpectation(String yumPackage) {
+ super(CommandType.deleteVersionLock, yumPackage);
+ }
+
+ }
+
+ public class QueryInstalledExpectation {
+ private final String packageName;
+
+ public QueryInstalledExpectation(String packageName) {
+ this.packageName = packageName;
+ }
+
+ /** Package name to return or null if package is not installed */
+ public YumTester andReturn(YumPackageName yumPackage) {
+ TestChildProcess2 process = new TestChildProcess2(
+ yumPackage == null ? 1 : 0,
+ yumPackage == null ? "not installed" : String.join("\n",
+ yumPackage.getName(),
+ yumPackage.getEpoch().orElse("(none)"),
+ yumPackage.getVersion().orElseThrow(() -> new IllegalArgumentException("Version must be set")),
+ yumPackage.getRelease().orElseThrow(() -> new IllegalArgumentException("Release must be set")),
+ yumPackage.getArchitecture().orElse("(none)")));
+
+ terminal.expectCommand("rpm -q " + packageName + " --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", process);
+ return YumTester.this;
+ }
+ }
+
+ private enum CommandType {
+ install("install"), upgrade("upgrade"), remove("remove"), installFixed("install"), deleteVersionLock("versionlock delete");
+
+ private final String command;
+ CommandType(String command) {
+ this.command = command;
+ }
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java
new file mode 100644
index 00000000000..89a39151974
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java
new file mode 100644
index 00000000000..34d0f555661
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java
@@ -0,0 +1,30 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.wireguard;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.WireguardKeyWithTimestamp;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress;
+
+import java.util.List;
+
+/**
+ * A wireguard peer. Sorted by hostname. IP addresses are sorted by version, IPv6 first.
+ * The public key should always be non-null.
+ *
+ * @author gjoranv
+ */
+public record WireguardPeer(HostName hostname,
+ List<VersionedIpAddress> ipAddresses,
+ WireguardKeyWithTimestamp keyWithTimestamp) implements Comparable<WireguardPeer> {
+
+ public WireguardPeer {
+ if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value());
+ ipAddresses = ipAddresses.stream().sorted().toList();
+ }
+
+ @Override
+ public int compareTo(WireguardPeer o) {
+ return hostname.value().compareTo(o.hostname.value());
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java
new file mode 100644
index 00000000000..fb5a055915a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.wireguard;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/sh/node-admin.sh b/node-admin/src/main/sh/node-admin.sh
new file mode 100755
index 00000000000..a2ce59e57a8
--- /dev/null
+++ b/node-admin/src/main/sh/node-admin.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ empty_if_start_slash=${mypath%%/*}
+ if [ "${empty_if_start_slash}" ]; then
+ mypath=$(pwd)/${mypath}
+ fi
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findhost () {
+ if [ "${VESPA_HOSTNAME}" = "" ]; then
+ VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1
+ fi
+ validate="${VESPA_HOME}/bin/vespa-validate-hostname"
+ if [ -f "$validate" ]; then
+ "$validate" "${VESPA_HOSTNAME}" || exit 1
+ fi
+ export VESPA_HOSTNAME
+}
+
+findroot
+findhost
+
+ROOT=${VESPA_HOME%/}
+export ROOT
+
+# END environment bootstrap section
+
+Usage() {
+ cat <<EOF
+Usage: ${0##*/} [start|stop]
+Manage standalone node admin
+EOF
+
+ exit 1
+}
+
+Start() {
+ "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@"
+}
+
+Stop() {
+ "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@"
+}
+
+if (( $# == 0 )); then
+ Usage
+fi
+
+command="$1"
+shift
+
+case "$command" in
+ start) Start "$@" ;;
+ stop) Stop "$@" ;;
+ restart)
+ Stop "$@"
+ Start "$@"
+ ;;
+ *) Usage ;;
+esac
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java
new file mode 100644
index 00000000000..d3982af14e4
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java
@@ -0,0 +1,162 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Device;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author freva
+ */
+public class CgroupTest {
+
+ private static final ContainerId containerId = new ContainerId("4aec78cc");
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Cgroup containerCgroup = Cgroup.root(fileSystem).resolveContainer(containerId);
+ private final CpuController containerCpu = containerCgroup.cpu();
+ private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build();
+ private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories();
+
+ @Test
+ public void updates_cpu_quota_and_period() {
+ assertEquals(Optional.empty(), containerCgroup.cpu().readMax());
+
+ cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n");
+ assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax());
+
+ cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n");
+ assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax());
+
+ containerCgroup.cpu().updateMax(context, 456, 123456);
+
+ assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456));
+ assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax());
+ assertEquals("654 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File());
+
+ assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456));
+ assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax());
+ assertEquals("max 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File());
+ }
+
+ @Test
+ public void updates_cpu_shares() {
+ assertEquals(Optional.empty(), containerCgroup.cpu().readShares());
+
+ cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n");
+ assertEquals(Optional.of(2), containerCgroup.cpu().readShares());
+
+ assertFalse(containerCgroup.cpu().updateShares(context, 2));
+
+ assertTrue(containerCgroup.cpu().updateShares(context, 12345));
+ assertEquals(Optional.of(12323), containerCgroup.cpu().readShares());
+ }
+
+ @Test
+ public void reads_cpu_stats() {
+ cgroupRoot.resolve("cpu.stat").writeUtf8File("""
+ usage_usec 17794243
+ user_usec 16099205
+ system_usec 1695038
+ nr_periods 12465
+ nr_throttled 25
+ throttled_usec 14256
+ """);
+
+ assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L,
+ TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats());
+ }
+
+ @Test
+ public void reads_memory_metrics() {
+ cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n");
+ assertEquals(2525093888L, containerCgroup.memory().readCurrent().value());
+
+ cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n");
+ assertEquals(4322885632L, containerCgroup.memory().readMax().value());
+
+ cgroupRoot.resolve("memory.stat").writeUtf8File("""
+ anon 3481600
+ file 69206016
+ kernel_stack 73728
+ slab 3552304
+ percpu 262336
+ sock 73728
+ shmem 8380416
+ file_mapped 1081344
+ file_dirty 135168
+ slab_reclaimable 1424320
+ """);
+ var stats = containerCgroup.memory().readStat();
+ assertEquals(69206016L, stats.file().value());
+ assertEquals(3481600L, stats.anon().value());
+ assertEquals(3552304L, stats.slab().value());
+ assertEquals(73728L, stats.sock().value());
+ assertEquals(1424320L, stats.slabReclaimable().value());
+ }
+
+ @Test
+ public void shares_to_weight_and_back_is_stable() {
+ for (int i = 2; i <= 262144; i++) {
+ int originalShares = i; // Must be effectively final to use in lambda :(
+ int roundTripShares = weightToShares(sharesToWeight(i));
+ int diff = i - roundTripShares;
+ assertTrue(diff >= 0 && diff <= 27, // ~26.2 shares / weight
+ () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff);
+ }
+ }
+
+ @Test
+ void reads_io_max() {
+ assertEquals(Optional.empty(), containerCgroup.io().readMax());
+
+ cgroupRoot.resolve("io.max").writeUtf8File("");
+ assertEquals(Optional.of(Map.of()), containerCgroup.io().readMax());
+
+ cgroupRoot.resolve("io.max").writeUtf8File("""
+ 253:1 rbps=11 wbps=max riops=22 wiops=33
+ 253:0 rbps=max wbps=44 riops=max wiops=55
+ """);
+ assertEquals(Map.of(new Device(253, 1), new Max(Size.from(11), Size.max(), Size.from(22), Size.from(33)),
+ new Device(253, 0), new Max(Size.max(), Size.from(44), Size.max(), Size.from(55))),
+ containerCgroup.io().readMax().orElseThrow());
+ }
+
+ @Test
+ void writes_io_max() {
+ Device device = new Device(253, 0);
+ Max initial = new Max(Size.max(), Size.from(44), Size.max(), Size.from(55));
+ assertTrue(containerCgroup.io().updateMax(context, device, initial));
+ assertEquals("253:0 rbps=max wbps=44 riops=max wiops=55\n", cgroupRoot.resolve("io.max").readUtf8File());
+
+ cgroupRoot.resolve("io.max").writeUtf8File("""
+ 253:1 rbps=11 wbps=max riops=22 wiops=33
+ 253:0 rbps=max wbps=44 riops=max wiops=55
+ """);
+ assertFalse(containerCgroup.io().updateMax(context, device, initial));
+
+ cgroupRoot.resolve("io.max").writeUtf8File("");
+ assertFalse(containerCgroup.io().updateMax(context, device, Max.UNLIMITED));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java
new file mode 100644
index 00000000000..cb828394249
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java
@@ -0,0 +1,19 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.cgroup;
+
+import org.junit.jupiter.api.Test;
+
+import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+class IoControllerTest {
+
+ @Test
+ void parse_io_max() {
+ assertEquals(Max.UNLIMITED, Max.fromString(""));
+ assertEquals(new Max(Size.from(1), Size.max(), Size.max(), Size.max()), Max.fromString("rbps=1 wiops=max"));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java
new file mode 100644
index 00000000000..910fd8e670a
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java
@@ -0,0 +1,194 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.entity.BasicHttpEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicStatusLine;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Basic testing of retry logic.
+ *
+ * @author dybis
+ */
+public class ConfigServerApiImplTest {
+
+ private static final int FAIL_RETURN_CODE = 100000;
+ private static final int TIMEOUT_RETURN_CODE = 100001;
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class TestPojo {
+ @JsonProperty("foo")
+ String foo;
+ @JsonProperty("error-code")
+ Integer errorCode;
+ }
+
+ private final String uri1 = "http://host1:666";
+ private final String uri2 = "http://host2:666";
+ private final List<URI> configServers = List.of(URI.create(uri1), URI.create(uri2));
+ private final StringBuilder mockLog = new StringBuilder();
+
+ private ConfigServerApiImpl configServerApi;
+ private int mockReturnCode = 200;
+
+ @BeforeEach
+ public void initExecutor() throws IOException {
+ CloseableHttpClient httpMock = mock(CloseableHttpClient.class);
+ when(httpMock.execute(any())).thenAnswer(invocationOnMock -> {
+ HttpGet get = (HttpGet) invocationOnMock.getArguments()[0];
+ mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append(" ");
+
+ switch (mockReturnCode) {
+ case FAIL_RETURN_CODE -> throw new RuntimeException("FAIL");
+ case TIMEOUT_RETURN_CODE -> throw new SocketTimeoutException("read timed out");
+ }
+
+ BasicStatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, mockReturnCode, null);
+ BasicHttpEntity entity = new BasicHttpEntity();
+ String returnMessage = "{\"foo\":\"bar\", \"no\":3, \"error-code\": " + mockReturnCode + "}";
+ InputStream stream = new ByteArrayInputStream(returnMessage.getBytes(StandardCharsets.UTF_8));
+ entity.setContent(stream);
+
+ CloseableHttpResponse response = mock(CloseableHttpResponse.class);
+ when(response.getEntity()).thenReturn(entity);
+ when(response.getStatusLine()).thenReturn(statusLine);
+
+ return response;
+ });
+ configServerApi = ConfigServerApiImpl.createForTestingWithClient(configServers, httpMock);
+ }
+
+ @Test
+ void testBasicParsingSingleServer() {
+ TestPojo answer = configServerApi.get("/path", TestPojo.class);
+ assertEquals(answer.foo, "bar");
+ assertLogStringContainsGETForAHost();
+ }
+
+ @Test
+ void testBasicFailure() {
+ assertThrows(HttpException.class, () -> {
+ // Server is returning 400, no retries.
+ mockReturnCode = 400;
+
+ TestPojo testPojo = configServerApi.get("/path", TestPojo.class);
+ assertEquals(testPojo.errorCode.intValue(), mockReturnCode);
+ assertLogStringContainsGETForAHost();
+ });
+ }
+
+ @Test
+ void testBasicSuccessWithNoRetries() {
+ // Server is returning 201, no retries.
+ mockReturnCode = 201;
+
+ TestPojo testPojo = configServerApi.get("/path", TestPojo.class);
+ assertEquals(testPojo.errorCode.intValue(), mockReturnCode);
+ assertLogStringContainsGETForAHost();
+ }
+
+ @Test
+ void testBasicSuccessWithCustomTimeouts() {
+ mockReturnCode = TIMEOUT_RETURN_CODE;
+
+ var params = new ConfigServerApi.Params<TestPojo>();
+ params.setConnectionTimeout(Duration.ofSeconds(3));
+
+ try {
+ configServerApi.get("/path", TestPojo.class, params);
+ fail();
+ } catch (ConnectionException e) {
+ assertNotNull(e.getCause());
+ assertEquals("read timed out", e.getCause().getMessage());
+ }
+ }
+
+ @Test
+ void testRetries() {
+ // Client is throwing exception, should be retries.
+ mockReturnCode = FAIL_RETURN_CODE;
+ try {
+ configServerApi.get("/path", TestPojo.class);
+ fail("Expected failure");
+ } catch (Exception e) {
+ // ignore
+ }
+
+ List<String> log = List.of(mockLog.toString().split(" "));
+ assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path")));
+ }
+
+ @Test
+ void testNoRetriesOnBadHttpResponseCode() {
+ // Client is throwing exception, should be retries.
+ mockReturnCode = 503;
+ try {
+ configServerApi.get("/path", TestPojo.class);
+ fail("Expected failure");
+ } catch (Exception e) {
+ // ignore
+ }
+
+ assertLogStringContainsGETForAHost();
+ }
+
+ @Test
+ void testForbidden() {
+ mockReturnCode = 403;
+ try {
+ configServerApi.get("/path", TestPojo.class);
+ fail("Expected exception");
+ } catch (HttpException.ForbiddenException e) {
+ // ignore
+ }
+ assertLogStringContainsGETForAHost();
+ }
+
+ @Test
+ void testNotFound() {
+ // Server is returning 404, special exception is thrown.
+ mockReturnCode = 404;
+ try {
+ configServerApi.get("/path", TestPojo.class);
+ fail("Expected exception");
+ } catch (HttpException.NotFoundException e) {
+ // ignore
+ }
+ assertLogStringContainsGETForAHost();
+ }
+
+ @Test
+ void testConflict() {
+ // Server is returning 409, no exception is thrown.
+ mockReturnCode = 409;
+ configServerApi.get("/path", TestPojo.class);
+ assertLogStringContainsGETForAHost();
+ }
+
+ private void assertLogStringContainsGETForAHost() {
+ String logString = mockLog.toString();
+ assertTrue((logString.equals("GET http://host1:666/path ") || logString.equals("GET http://host2:666/path ")),
+ "log does not contain expected entries:" + logString);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
new file mode 100644
index 00000000000..430da856cfa
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
@@ -0,0 +1,151 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.cores;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.test.json.JsonTestHelper;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException;
+import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+class CoresTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final ConfigServerApi configServerApi = mock(ConfigServerApi.class);
+ private final Cores cores = new CoresImpl(configServerApi);
+ private final HostName hostname = HostName.of("foo.com");
+ private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0";
+ private final CoreDumpMetadata metadata = new CoreDumpMetadata()
+ .setType(CoreDumpMetadata.Type.OOM)
+ .setCreated(Instant.ofEpochMilli(12345678))
+ .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64")
+ .setCpuMicrocodeVersion("0x1000065")
+ .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813"))
+ .setDecryptionToken("987def")
+ .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8"))
+ .setBinPath("/usr/bin/java")
+ .setVespaVersion("8.68.8")
+ .setBacktraceAllThreads(List.of("Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...",
+ "Debugger attached successfully.",
+ " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)",
+ " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)"))
+ .setBacktrace(List.of("Example", "of", "backtrace"));
+
+ @Test
+ void reportOK() {
+ var oKResponse = new StandardConfigServerResponse();
+ oKResponse.message = "OK";
+ when(configServerApi.post(any(), any(), any())).thenReturn(oKResponse);
+
+ cores.report(hostname, id, metadata);
+
+ var pathCaptor = ArgumentCaptor.forClass(String.class);
+ var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(configServerApi, times(1)).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any());
+
+ assertEquals("/cores/v1/report/" + hostname + "/" + id, pathCaptor.getValue());
+
+ assertEquals("""
+ {
+ "backtrace": [
+ "Example",
+ "of",
+ "backtrace"
+ ],
+ "backtrace_all_threads": [
+ "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...",
+ "Debugger attached successfully.",
+ " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)",
+ " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)"
+ ],
+ "bin_path": "/usr/bin/java",
+ "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
+ "cpu_microcode_version": "0x1000065",
+ "created": 12345678,
+ "decryption_token": "987def",
+ "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
+ "kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
+ "type": "OOM",
+ "vespa_version": "8.68.8"
+ }""",
+ JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue()))));
+ }
+
+ @Test
+ void reportFails() {
+ var response = new StandardConfigServerResponse();
+ response.errorCode = "503";
+ response.message = "error detail";
+ when(configServerApi.post(any(), any(), any())).thenReturn(response);
+
+ assertThrows(ConfigServerException.class,
+ () -> cores.report(hostname, "abcde-1234", metadata),
+ "Failed to report core dump at Optional[/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813]: error detail 503");
+
+ var pathCaptor = ArgumentCaptor.forClass(String.class);
+ var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any());
+ }
+
+ @Test
+ void serialization() {
+ Path path = fileSystem.getPath("/foo.json");
+ ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata);
+ request.save(path);
+ assertEquals("""
+ {
+ "backtrace": [
+ "Example",
+ "of",
+ "backtrace"
+ ],
+ "backtrace_all_threads": [
+ "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...",
+ "Debugger attached successfully.",
+ " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)",
+ " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)"
+ ],
+ "bin_path": "/usr/bin/java",
+ "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
+ "cpu_microcode_version": "0x1000065",
+ "created": 12345678,
+ "decryption_token": "987def",
+ "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
+ "kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
+ "type": "OOM",
+ "vespa_version": "8.68.8"
+ }""",
+ JsonTestHelper.normalize(new UnixPath(path).readUtf8File()));
+
+ Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path);
+ assertTrue(loaded.isPresent());
+ var meta = new CoreDumpMetadata();
+ loaded.get().populateMetadata(meta, fileSystem);
+ assertEquals(metadata, meta);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java
new file mode 100644
index 00000000000..664e25bc744
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java
@@ -0,0 +1,40 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.flags;
+
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.flags.json.wire.WireFlagData;
+import com.yahoo.vespa.flags.json.wire.WireFlagDataList;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class RealFlagRepositoryTest {
+ private final ConfigServerApi configServerApi = mock(ConfigServerApi.class);
+ private final RealFlagRepository repository = new RealFlagRepository(configServerApi);
+
+ @Test
+ void test() {
+ WireFlagDataList list = new WireFlagDataList();
+ list.flags = new ArrayList<>();
+ list.flags.add(new WireFlagData());
+ list.flags.get(0).id = "id1";
+
+ when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list);
+ Map<FlagId, FlagData> allFlagData = repository.getAllFlagData();
+ assertEquals(1, allFlagData.size());
+ assertTrue(allFlagData.containsKey(new FlagId("id1")));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java
new file mode 100644
index 00000000000..d91e9befab9
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java
@@ -0,0 +1,182 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ *
+ * @author smorgrav
+ */
+public class AclTest {
+
+ private static final Acl aclCommon = new Acl(
+ Set.of(1234, 453), Set.of(4321),
+ testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2", "fe80::3"),
+ Set.of());
+
+ private static final Acl aclWithoutPorts = new Acl(
+ Set.of(), Set.of(),
+ testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2"),
+ Set.of());
+
+ @Test
+ void no_trusted_ports() {
+ String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4));
+ assertEquals(
+ """
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p icmp -j ACCEPT
+ -A INPUT -s 192.1.2.2/32 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp-port-unreachable""",
+ listRulesIpv4);
+ }
+
+ @Test
+ void ipv4_rules() {
+ String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4));
+ assertEquals(
+ """
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT
+ -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT
+ -A INPUT -s 192.1.2.2/32 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp-port-unreachable""",
+ listRulesIpv4);
+ }
+
+ @Test
+ void ipv6_rules() {
+ String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6));
+ assertEquals(
+ """
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p ipv6-icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT
+ -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT
+ -A INPUT -s fb00::1/128 -j ACCEPT
+ -A INPUT -s fe80::2/128 -j ACCEPT
+ -A INPUT -s fe80::3/128 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", listRulesIpv6);
+ }
+
+ @Test
+ void ipv6_rules_stable_order() {
+ Acl aclCommonDifferentOrder = new Acl(
+ Set.of(453, 1234), Set.of(4321),
+ testNodes(Set.of(), "fe80::2", "192.1.2.2", "fb00::1", "fe80::3"),
+ Set.of());
+
+ for (IPVersion ipVersion : IPVersion.values()) {
+ assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion));
+ }
+ }
+
+ @Test
+ void trusted_networks() {
+ Acl acl = new Acl(Set.of(4080), Set.of(), testNodes(Set.of(), "127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32"));
+
+ assertEquals("""
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT
+ -A INPUT -s 127.0.0.1/32 -j ACCEPT
+ -A INPUT -s 10.0.0.0/24 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp-port-unreachable""",
+ String.join("\n", acl.toRules(IPVersion.IPv4)));
+
+ assertEquals("""
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p ipv6-icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT
+ -A INPUT -s 2001:db8::/32 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp6-port-unreachable""",
+ String.join("\n", acl.toRules(IPVersion.IPv6)));
+ }
+
+ @Test
+ void config_server_acl() {
+ Set<Acl.Node> testNodes = Stream.concat(testNodes(NodeType.config, Set.of(), "172.17.0.41", "172.17.0.42", "172.17.0.43").stream(),
+ testNodes(NodeType.tenant, Set.of(19070), "172.17.0.81", "172.17.0.82", "172.17.0.83").stream())
+ .collect(Collectors.toSet());
+ Acl acl = new Acl(Set.of(22, 4443), Set.of(), testNodes, Set.of());
+ assertEquals("""
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT
+ -A INPUT -s 172.17.0.41/32 -j ACCEPT
+ -A INPUT -s 172.17.0.42/32 -j ACCEPT
+ -A INPUT -s 172.17.0.43/32 -j ACCEPT
+ -A INPUT -s 172.17.0.81/32 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -s 172.17.0.82/32 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -s 172.17.0.83/32 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp-port-unreachable""",
+ String.join("\n", acl.toRules(IPVersion.IPv4)));
+
+ Set<Acl.Node> testNodes2 = Stream.concat(testNodes(NodeType.config, Set.of(), "2001:db8::41", "2001:db8::42", "2001:db8::43").stream(),
+ testNodes(NodeType.tenant, Set.of(19070), "2001:db8::81", "2001:db8::82", "2001:db8::83").stream())
+ .collect(Collectors.toSet());
+ Acl acl2 = new Acl(Set.of(22, 4443), Set.of(), testNodes2, Set.of());
+
+ assertEquals("""
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p ipv6-icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT
+ -A INPUT -s 2001:db8::41/128 -j ACCEPT
+ -A INPUT -s 2001:db8::42/128 -j ACCEPT
+ -A INPUT -s 2001:db8::43/128 -j ACCEPT
+ -A INPUT -s 2001:db8::81/128 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -s 2001:db8::82/128 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -s 2001:db8::83/128 -p tcp -m multiport --dports 19070 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp6-port-unreachable""",
+ String.join("\n", acl2.toRules(IPVersion.IPv6)));
+ }
+
+ private static Set<Acl.Node> testNodes(Set<Integer> ports, String... address) {
+ return testNodes(NodeType.tenant, ports, address);
+ }
+
+ private static Set<Acl.Node> testNodes(NodeType nodeType, Set<Integer> ports, String... address) {
+ return Arrays.stream(address)
+ .map(addr -> new Acl.Node("hostname", addr, ports))
+ .collect(Collectors.toUnmodifiableSet());
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java
new file mode 100644
index 00000000000..b236c223078
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import org.junit.jupiter.api.Test;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class NodeStateTest {
+
+ @Test
+ void is_equal_to_node_repository_states() {
+ Set<String> nodeRepositoryStates = Stream.of(Node.State.values()).map(Enum::name).collect(Collectors.toSet());
+ Set<String> nodeAdminStates = Stream.of(NodeState.values()).map(Enum::name).collect(Collectors.toSet());
+
+ assertEquals(nodeAdminStates, nodeRepositoryStates);
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
new file mode 100644
index 00000000000..4100b3cf102
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
@@ -0,0 +1,249 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+
+import com.yahoo.application.Networking;
+import com.yahoo.application.container.JDisc;
+import com.yahoo.config.provision.CloudAccount;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.config.provision.WireguardKeyWithTimestamp;
+import com.yahoo.config.provision.host.FlavorOverrides;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress;
+import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer;
+import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler;
+import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository
+ * which already contains some data.
+ *
+ * @author dybdahl
+ */
+public class RealNodeRepositoryTest {
+
+ private static final double delta = 0.00000001;
+ private JDisc container;
+ private NodeRepository nodeRepositoryApi;
+
+ private int findRandomOpenPort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ socket.setReuseAddress(true);
+ return socket.getLocalPort();
+ }
+ }
+
+ /**
+ * Starts NodeRepository with
+ * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors}
+ * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository}
+ * {@link NodesV2ApiHandler}
+ * These classes define some test data that is used in these tests.
+ */
+ @BeforeEach
+ public void startContainer() throws Exception {
+ Exception lastException = null;
+
+ // This tries to bind a random open port for the node-repo mock, which is a race condition, so try
+ // a few times before giving up
+ for (int i = 0; i < 3; i++) {
+ try {
+ int port = findRandomOpenPort();
+ container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.from("123456789012")), Networking.enable);
+ ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting(
+ List.of(URI.create("http://127.0.0.1:" + port)));
+ waitForJdiscContainerToServe(configServerApi);
+ return;
+ } catch (RuntimeException e) {
+ lastException = e;
+ }
+ }
+ throw new RuntimeException("Failed to bind a port in three attempts, giving up", lastException);
+ }
+
+ private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException {
+ Instant start = Instant.now();
+ nodeRepositoryApi = new RealNodeRepository(configServerApi);
+ while (Instant.now().minusSeconds(120).isBefore(start)) {
+ try {
+ nodeRepositoryApi.getNodes("foobar");
+ return;
+ } catch (Exception e) {
+ Thread.sleep(100);
+ }
+ }
+ throw new RuntimeException("Could not get answer from container.");
+ }
+
+ @AfterEach
+ public void stopContainer() {
+ if (container != null) {
+ container.close();
+ }
+ }
+
+ @Test
+ void testGetContainersToRunApi() {
+ String dockerHostHostname = "dockerhost1.yahoo.com";
+
+ List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname);
+ assertEquals(1, containersToRun.size());
+ NodeSpec node = containersToRun.get(0);
+ assertEquals("host4.yahoo.com", node.hostname());
+ assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get());
+ assertEquals(NodeState.active, node.state());
+ assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get());
+ assertEquals(Long.valueOf(0), node.currentRestartGeneration().get());
+ assertEquals(1, node.vcpu(), delta);
+ assertEquals(4, node.memoryGb(), delta);
+ assertEquals(100, node.diskGb(), delta);
+ }
+
+ @Test
+ void testGetContainer() {
+ String hostname = "host4.yahoo.com";
+ Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname);
+ assertTrue(node.isPresent());
+ assertEquals(hostname, node.get().hostname());
+ assertEquals(CloudAccount.from("123456789012"), node.get().cloudAccount());
+ }
+
+ @Test
+ void testGetContainerForNonExistingNode() {
+ String hostname = "host-that-does-not-exist";
+ Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname);
+ assertFalse(node.isPresent());
+ }
+
+ @Test
+ void testUpdateNodeAttributes() {
+ var hostname = "host4.yahoo.com";
+ var dockerImage = "registry.example.com/repo/image-1:6.2.3";
+ var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c=");
+ var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository
+ var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp);
+
+ nodeRepositoryApi.updateNodeAttributes(
+ hostname,
+ new NodeAttributes()
+ .withRestartGeneration(1)
+ .withDockerImage(DockerImage.fromString(dockerImage))
+ .withWireguardPubkey(wireguardKey));
+
+ NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow();
+ assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow());
+ assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString());
+ assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow());
+ }
+
+ @Test
+ void testMarkAsReady() {
+ nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.dirty);
+ nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.ready);
+
+ try {
+ nodeRepositoryApi.setNodeState("host4.yahoo.com", NodeState.ready);
+ fail("Should not be allowed to be marked ready as it is not registered as provisioned, dirty, failed or parked");
+ } catch (RuntimeException ignored) {
+ // expected
+ }
+
+ try {
+ nodeRepositoryApi.setNodeState("host101.yahoo.com", NodeState.ready);
+ fail("Expected failure because host101 does not exist");
+ } catch (RuntimeException ignored) {
+ // expected
+ }
+ }
+
+ @Test
+ void testAddNodes() {
+ AddNode host = AddNode.forHost("host123.domain.tld",
+ "id1",
+ "default",
+ Optional.of(FlavorOverrides.ofDisk(123)),
+ NodeType.confighost,
+ Set.of("::1"), Set.of("::2", "::3"));
+
+ NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local);
+ AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3"));
+
+ assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent());
+ nodeRepositoryApi.addNodes(List.of(host, node));
+
+ NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow();
+ assertEquals("id1", hostSpec.id());
+ assertEquals("default", hostSpec.flavor());
+ assertEquals(123, hostSpec.diskGb(), 0);
+ assertEquals(NodeType.confighost, hostSpec.type());
+ assertEquals(NodeResources.Architecture.x86_64, hostSpec.resources().architecture());
+
+ NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow();
+ assertEquals(nodeResources, nodeSpec.resources());
+ assertEquals(NodeType.config, nodeSpec.type());
+ }
+
+ @Test
+ void wireguard_peer_config_can_be_retrieved_for_configservers_and_exclave_nodes() {
+
+ //// Configservers ////
+
+ List<WireguardPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers();
+
+ // cfg2 does not have a wg public key, so should not be included
+ assertEquals(1, cfgPeers.size());
+
+ assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com",
+ "::201:1",
+ "lololololololololololololololololololololoo=",
+ 456L);
+
+ //// Exclave nodes ////
+
+ List<WireguardPeer> exclavePeers = nodeRepositoryApi.getExclavePeers();
+
+ // host3 does not have a wg public key, so should not be included
+ assertEquals(1, exclavePeers.size());
+
+ assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com",
+ "::101:1",
+ "000011112222333344445555666677778888999900c=",
+ 123L);
+ }
+
+ private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6,
+ String publicKey, long keyTimestamp) {
+ assertEquals(hostname, peer.hostname().value());
+ assertEquals(1, peer.ipAddresses().size());
+ assertIp(peer.ipAddresses().get(0), ipv6, 6);
+ var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey),
+ Instant.ofEpochMilli(keyTimestamp));
+ assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp());
+ }
+
+ private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) {
+ assertEquals(expectedIp, ip.asString());
+ assertEquals(expectedVersion, ip.version().version());
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java
new file mode 100644
index 00000000000..8de5986739e
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java
@@ -0,0 +1,72 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.test.json.JsonTestHelper;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author hakonhall
+ */
+public class NodeRepositoryNodeTest {
+ private static final ObjectMapper mapper = new ObjectMapper();
+ private final NodeRepositoryNode node = new NodeRepositoryNode();
+ private final NodeAttributes attributes = new NodeAttributes();
+
+
+ /**
+ * Test both how NodeRepositoryNode serialize, and the serialization of an empty NodeRepositoryNode
+ * patched with a NodeAttributes, as they work in tandem:
+ * NodeAttributes -> NodeRepositoryNode -> JSON.
+ */
+ @Test
+ void testReportsSerialization() {
+ // Make sure we don't accidentally patch with "reports": null, as that actually means removing all reports.
+ assertEquals(JsonInclude.Include.NON_NULL, NodeRepositoryNode.class.getAnnotation(JsonInclude.class).value());
+
+ // Absent report and unmodified attributes => nothing about reports in JSON
+ node.reports = null;
+ assertNodeAndAttributes("{}");
+
+ // Make sure we're able to patch with a null report value ("reportId": null), as that means removing the report.
+ node.reports = new HashMap<>();
+ node.reports.put("rid", null);
+ attributes.withReportRemoved("rid");
+ assertNodeAndAttributes("{\"reports\": {\"rid\": null}}");
+
+ // Add ridTwo report to node
+ ObjectNode reportJson = mapper.createObjectNode();
+ reportJson.set(BaseReport.CREATED_FIELD, mapper.valueToTree(3));
+ reportJson.set(BaseReport.DESCRIPTION_FIELD, mapper.valueToTree("desc"));
+ node.reports.put("ridTwo", reportJson);
+
+ // Add ridTwo report to attributes
+ BaseReport reportTwo = new BaseReport(3L, "desc", null);
+ attributes.withReport("ridTwo", reportTwo.toJsonNode());
+
+ // Verify node serializes to expected, as well as attributes patched on node.
+ assertNodeAndAttributes("{\"reports\": {\"rid\": null, \"ridTwo\": {\"createdMillis\": 3, \"description\": \"desc\"}}}");
+ }
+
+ private void assertNodeAndAttributes(String expectedJson) {
+ assertNodeJson(node, expectedJson);
+ assertNodeJson(RealNodeRepository.nodeRepositoryNodeFromNodeAttributes(attributes), expectedJson);
+ }
+
+ private void assertNodeJson(NodeRepositoryNode node, String json) {
+ JsonNode expected = uncheck(() -> mapper.readTree(json));
+ JsonNode actual = uncheck(() -> mapper.valueToTree(node));
+ JsonTestHelper.assertJsonEquals(actual, expected);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java
new file mode 100644
index 00000000000..69e79ec8720
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java
@@ -0,0 +1,73 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports;
+
+import com.yahoo.test.json.JsonTestHelper;
+import org.junit.jupiter.api.Test;
+
+import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.SOFT_FAIL;
+import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.UNSPECIFIED;
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author hakonhall
+ */
+public class BaseReportTest {
+ private static final String JSON_1 = "{\"createdMillis\": 1, \"description\": \"desc\"}";
+ private static final String JSON_2 = "{\"createdMillis\": 1, \"description\": \"desc\", \"type\": \"SOFT_FAIL\"}";
+
+ @Test
+ void testSerialization1() {
+ JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", SOFT_FAIL).toJsonNode(),
+ JSON_2);
+ JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", SOFT_FAIL).toJsonNode(),
+ "{\"description\": \"desc\", \"type\": \"SOFT_FAIL\"}");
+ JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, SOFT_FAIL).toJsonNode(),
+ "{\"createdMillis\": 1, \"type\": \"SOFT_FAIL\"}");
+ JsonTestHelper.assertJsonEquals(new BaseReport(null, null, SOFT_FAIL).toJsonNode(),
+ "{\"type\": \"SOFT_FAIL\"}");
+
+ JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", null).toJsonNode(),
+ JSON_1);
+ JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", null).toJsonNode(),
+ "{\"description\": \"desc\"}");
+ JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, null).toJsonNode(),
+ "{\"createdMillis\": 1}");
+ JsonTestHelper.assertJsonEquals(new BaseReport(null, null, null).toJsonNode(),
+ "{}");
+ }
+
+ @Test
+ void testShouldUpdate() {
+ BaseReport report = new BaseReport(1L, "desc", SOFT_FAIL);
+ assertFalse(report.updates(report));
+
+ // createdMillis is ignored
+ assertFalse(new BaseReport(1L, "desc", SOFT_FAIL).updates(report));
+ assertFalse(new BaseReport(2L, "desc", SOFT_FAIL).updates(report));
+ assertFalse(new BaseReport(null, "desc", SOFT_FAIL).updates(report));
+
+ // description is not ignored
+ assertTrue(new BaseReport(1L, "desc 2", SOFT_FAIL).updates(report));
+ assertTrue(new BaseReport(1L, null, SOFT_FAIL).updates(report));
+
+ // type is not ignored
+ assertTrue(new BaseReport(1L, "desc", null).updates(report));
+ assertTrue(new BaseReport(1L, "desc", BaseReport.Type.HARD_FAIL).updates(report));
+ }
+
+ @Test
+ void testJsonSerialization() {
+ BaseReport report = BaseReport.fromJson(JSON_2);
+ assertEquals(1L, (long) report.getCreatedMillisOrNull());
+ assertEquals("desc", report.getDescriptionOrNull());
+ assertEquals(SOFT_FAIL, report.getTypeOrNull());
+ JsonTestHelper.assertJsonEquals(report.toJson(), JSON_2);
+ }
+
+ @Test
+ void testUnspecifiedType() {
+ BaseReport report = new BaseReport(1L, "desc", null);
+ assertNull(report.getTypeOrNull());
+ assertEquals(UNSPECIFIED, report.getType());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java
new file mode 100644
index 00000000000..bb9c075ad74
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java
@@ -0,0 +1,172 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl;
+import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
+import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class OrchestratorImplTest {
+
+ private static final String hostName = "host123.yahoo.com";
+
+ private final ConfigServerApiImpl configServerApi = mock(ConfigServerApiImpl.class);
+ private final OrchestratorImpl orchestrator = new OrchestratorImpl(configServerApi);
+
+ @Test
+ void testSuspendCall() {
+ when(configServerApi.put(
+ eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"),
+ eq(Optional.empty()),
+ eq(UpdateHostResponse.class),
+ any()
+ )).thenReturn(new UpdateHostResponse(hostName, null));
+
+ orchestrator.suspend(hostName);
+ }
+
+ @Test
+ void testSuspendCallWithFailureReason() {
+ assertThrows(OrchestratorException.class, () -> {
+ when(configServerApi.put(
+ eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"),
+ eq(Optional.empty()),
+ eq(UpdateHostResponse.class),
+ any()
+ )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail")));
+
+ orchestrator.suspend(hostName);
+ });
+ }
+
+ @Test
+ void testSuspendCallWithNotFound() {
+ assertThrows(OrchestratorNotFoundException.class, () -> {
+ when(configServerApi.put(any(String.class), any(), any(), any()))
+ .thenThrow(new HttpException.NotFoundException("Not Found"));
+
+ orchestrator.suspend(hostName);
+ });
+ }
+
+ @Test
+ void testSuspendCallWithSomeOtherException() {
+ assertThrows(RuntimeException.class, () -> {
+ when(configServerApi.put(any(String.class), any(), any(), any()))
+ .thenThrow(new RuntimeException("Some parameter was wrong"));
+
+ orchestrator.suspend(hostName);
+ });
+ }
+
+
+ @Test
+ void testResumeCall() {
+ when(configServerApi.delete(
+ OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended",
+ UpdateHostResponse.class
+ )).thenReturn(new UpdateHostResponse(hostName, null));
+
+ orchestrator.resume(hostName);
+ }
+
+ @Test
+ void testResumeCallWithFailureReason() {
+ assertThrows(OrchestratorException.class, () -> {
+ when(configServerApi.delete(
+ OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended",
+ UpdateHostResponse.class
+ )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail")));
+
+ orchestrator.resume(hostName);
+ });
+ }
+
+ @Test
+ void testResumeCallWithNotFound() {
+ assertThrows(OrchestratorNotFoundException.class, () -> {
+ when(configServerApi.delete(
+ any(String.class),
+ any()
+ )).thenThrow(new HttpException.NotFoundException("Not Found"));
+
+ orchestrator.resume(hostName);
+ });
+ }
+
+ @Test
+ void testResumeCallWithSomeOtherException() {
+ assertThrows(RuntimeException.class, () -> {
+ when(configServerApi.put(any(String.class), any(), any(), any()))
+ .thenThrow(new RuntimeException("Some parameter was wrong"));
+
+ orchestrator.suspend(hostName);
+ });
+ }
+
+ @Test
+ void testBatchSuspendCall() {
+ String parentHostName = "host1.test.yahoo.com";
+ List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com");
+
+ when(configServerApi.put(
+ eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"),
+ eq(Optional.empty()),
+ eq(BatchOperationResult.class),
+ any()
+ )).thenReturn(BatchOperationResult.successResult());
+
+ orchestrator.suspend(parentHostName, hostNames);
+ }
+
+ @Test
+ void testBatchSuspendCallWithFailureReason() {
+ assertThrows(OrchestratorException.class, () -> {
+ String parentHostName = "host1.test.yahoo.com";
+ List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com");
+ String failureReason = "Failed to suspend";
+
+ when(configServerApi.put(
+ eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"),
+ eq(Optional.empty()),
+ eq(BatchOperationResult.class),
+ any()
+ )).thenReturn(new BatchOperationResult(failureReason));
+
+ orchestrator.suspend(parentHostName, hostNames);
+ });
+ }
+
+ @Test
+ void testBatchSuspendCallWithSomeException() {
+ assertThrows(RuntimeException.class, () -> {
+ String parentHostName = "host1.test.yahoo.com";
+ List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com");
+ String exceptionMessage = "Exception: Something crashed!";
+
+ when(configServerApi.put(
+ eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"),
+ eq(Optional.empty()),
+ eq(BatchOperationResult.class),
+ any()
+ )).thenThrow(new RuntimeException(exceptionMessage));
+
+ orchestrator.suspend(parentHostName, hostNames);
+ });
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java
new file mode 100644
index 00000000000..478e89cde34
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java
@@ -0,0 +1,54 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class HealthResponseTest {
+ @Test
+ void deserializationOfNormalResponse() throws Exception {
+ String jsonResponse = "{\n" +
+ " \"metrics\": {\n" +
+ " \"snapshot\": {\n" +
+ " \"from\": 1.523614569023E9,\n" +
+ " \"to\": 1.523614629023E9\n" +
+ " },\n" +
+ " \"values\": [\n" +
+ " {\n" +
+ " \"name\": \"requestsPerSecond\",\n" +
+ " \"values\": {\n" +
+ " \"count\": 121,\n" +
+ " \"rate\": 2.0166666666666666\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\": \"latencySeconds\",\n" +
+ " \"values\": {\n" +
+ " \"average\": 5.537190082644628E-4,\n" +
+ " \"count\": 121,\n" +
+ " \"last\": 0.001,\n" +
+ " \"max\": 0.001,\n" +
+ " \"min\": 0,\n" +
+ " \"rate\": 2.0166666666666666\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"status\": {\"code\": \"up\"},\n" +
+ " \"time\": 1523614629451\n" +
+ "}";
+
+ HealthResponse response = deserialize(jsonResponse);
+
+ assertEquals(response.status.code, "up");
+ }
+
+ private static HealthResponse deserialize(String json) throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+
+ return mapper.readValue(json, HealthResponse.class);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java
new file mode 100644
index 00000000000..733a105f047
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java
@@ -0,0 +1,39 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.state;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
+import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException;
+import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse;
+import org.junit.jupiter.api.Test;
+
+import java.net.ConnectException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class StateImplTest {
+ private final ConfigServerApi api = mock(ConfigServerApi.class);
+ private final StateImpl state = new StateImpl(api);
+
+ @Test
+ void testWhenUp() {
+ HealthResponse response = new HealthResponse();
+ response.status.code = "up";
+ when(api.get(any(), any())).thenReturn(response);
+
+ HealthCode code = state.getHealth();
+ assertEquals(HealthCode.UP, code);
+ }
+
+ @Test
+ void connectException() {
+ RuntimeException exception =
+ ConnectionException.handleException("Error: ", new ConnectException("connection refused"));
+ when(api.get(any(), any())).thenThrow(exception);
+
+ HealthCode code = state.getHealth();
+ assertEquals(HealthCode.DOWN, code);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java
new file mode 100644
index 00000000000..45b74368fd8
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java
@@ -0,0 +1,256 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.image.Image;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * @author mpolden
+ */
+public class ContainerEngineMock implements ContainerEngine {
+
+ private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>();
+ private final Map<String, ImageDownload> images = new ConcurrentHashMap<>();
+ private boolean asyncImageDownload = false;
+
+ private final TestTerminal terminal;
+
+ public ContainerEngineMock() {
+ this(null);
+ }
+
+ public ContainerEngineMock(TestTerminal terminal) {
+ this.terminal = terminal;
+ }
+
+ public ContainerEngineMock asyncImageDownload(boolean enabled) {
+ this.asyncImageDownload = enabled;
+ return this;
+ }
+
+ public ContainerEngineMock completeDownloadOf(DockerImage image) {
+ String imageId = image.asString();
+ ImageDownload download;
+ while ((download = images.get(imageId)) == null);
+ download.complete();
+ return this;
+ }
+
+ public ContainerEngineMock setImages(List<Image> images) {
+ this.images.clear();
+ for (var image : images) {
+ ImageDownload imageDownload = new ImageDownload(image);
+ imageDownload.complete();
+ this.images.put(image.id(), imageDownload);
+ }
+ return this;
+ }
+
+ public ContainerEngineMock addContainers(List<Container> containers) {
+ for (var container : containers) {
+ if (this.containers.containsKey(container.name())) {
+ throw new IllegalArgumentException("Container " + container.name() + " already exists");
+ }
+ this.containers.put(container.name(), container);
+ }
+ return this;
+ }
+
+ public ContainerEngineMock addContainer(Container container) {
+ return addContainers(List.of(container));
+ }
+
+ @Override
+ public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) {
+ addContainer(createContainer(context, PartialContainer.State.created, containerResources));
+ return new ContainerData() {
+ @Override
+ public void addFile(ContainerPath path, String data) {
+ throw new UnsupportedOperationException("addFile not implemented");
+ }
+
+ @Override
+ public void addFile(ContainerPath path, String data, String permissions) {
+ throw new UnsupportedOperationException("addFile not implemented");
+ }
+
+ @Override
+ public void addDirectory(ContainerPath path, String... permissions) {
+ throw new UnsupportedOperationException("addDirectory not implemented");
+ }
+
+ @Override
+ public void addSymlink(ContainerPath symlink, Path target) {
+ throw new UnsupportedOperationException("addSymlink not implemented");
+ }
+
+ @Override
+ public void converge(NodeAgentContext context) {
+ throw new UnsupportedOperationException("converge not implemented");
+ }
+ };
+ }
+
+ @Override
+ public void startContainer(NodeAgentContext context) {
+ Container container = requireContainer(context.containerName(), PartialContainer.State.created);
+ Container newContainer = createContainer(context, PartialContainer.State.running, container.resources());
+ containers.put(newContainer.name(), newContainer);
+ }
+
+ @Override
+ public void removeContainer(TaskContext context, PartialContainer container) {
+ requireContainer(container.name());
+ containers.remove(container.name());
+ }
+
+ @Override
+ public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) {
+ Container container = requireContainer(context.containerName());
+ containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(),
+ container.imageId(), container.image(),
+ container.labels(), container.pid(),
+ container.conmonPid(), container.hostname(),
+ containerResources, container.networks(),
+ container.managed()));
+ }
+
+ @Override
+ public Optional<Container> getContainer(NodeAgentContext context) {
+ return Optional.ofNullable(containers.get(context.containerName()));
+ }
+
+ @Override
+ public List<PartialContainer> listContainers(TaskContext context) {
+ return List.copyOf(containers.values());
+ }
+
+ @Override
+ public String networkInterface(NodeAgentContext context) {
+ return "eth0";
+ }
+
+ @Override
+ public CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command) {
+ if (terminal == null) {
+ return new CommandResult(null, 0, "");
+ }
+ return terminal.newCommandLine(context)
+ .add(command)
+ .executeSilently();
+ }
+
+ @Override
+ public CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) {
+ if (terminal == null) {
+ return new CommandResult(null, 0, "");
+ }
+ return terminal.newCommandLine(context).add(command).execute(options);
+ }
+
+ @Override
+ public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) {
+ String imageId = image.asString();
+ ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, List.of(imageId))));
+ if (!asyncImageDownload) {
+ imageDownload.complete();
+ }
+ imageDownload.awaitCompletion();
+ }
+
+ @Override
+ public boolean hasImage(TaskContext context, DockerImage image) {
+ ImageDownload download = images.get(image.asString());
+ return download != null && download.isComplete();
+ }
+
+ @Override
+ public void removeImage(TaskContext context, String id) {
+ images.remove(id);
+ }
+
+ @Override
+ public List<Image> listImages(TaskContext context) {
+ return images.values().stream()
+ .filter(ImageDownload::isComplete)
+ .map(ImageDownload::image)
+ .toList();
+ }
+
+ private Container requireContainer(ContainerName name) {
+ return requireContainer(name, null);
+ }
+
+ private Container requireContainer(ContainerName name, PartialContainer.State wantedState) {
+ Container container = containers.get(name);
+ if (container == null) throw new IllegalArgumentException("No such container: " + name);
+ if (wantedState != null && container.state() != wantedState) throw new IllegalArgumentException("Container is " + container.state() + ", wanted " + wantedState);
+ return container;
+ }
+
+ public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) {
+ return new Container(new ContainerId("id-of-" + context.containerName()),
+ context.containerName(),
+ Instant.EPOCH,
+ state,
+ "image-id",
+ context.node().wantedDockerImage().get(),
+ Map.of(),
+ 41,
+ 42,
+ context.hostname().value(),
+ containerResources,
+ List.of(),
+ true);
+ }
+
+ private static class ImageDownload {
+
+ private final Image image;
+ private final CountDownLatch done = new CountDownLatch(1);
+
+ ImageDownload(Image image) {
+ this.image = Objects.requireNonNull(image);
+ }
+
+ Image image() {
+ return image;
+ }
+
+ boolean isComplete() {
+ return done.getCount() == 0;
+ }
+
+ void complete() {
+ done.countDown();
+ }
+
+ void awaitCompletion() {
+ try {
+ done.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java
new file mode 100644
index 00000000000..f9f7a18597c
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author freva
+ */
+public class ContainerNameTest {
+ @Test
+ void testAlphanumericalContainerName() {
+ String name = "container123";
+ ContainerName containerName = new ContainerName(name);
+ assertEquals(containerName.asString(), name);
+ }
+
+ @Test
+ void testAlphanumericalWithDashContainerName() {
+ String name = "container-123";
+ ContainerName containerName = new ContainerName(name);
+ assertEquals(containerName.asString(), name);
+ }
+
+ @Test
+ void testContainerNameFromHostname() {
+ assertEquals(new ContainerName("container-123"), ContainerName.fromHostname("container-123.sub.domain.tld"));
+ }
+
+ @Test
+ void testAlphanumericalWithSlashContainerName() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new ContainerName("container/123");
+ });
+ }
+
+ @Test
+ void testEmptyContainerName() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new ContainerName("");
+ });
+ }
+
+ @Test
+ void testNullContainerName() {
+ assertThrows(NullPointerException.class, () -> {
+ new ContainerName(null);
+ });
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java
new file mode 100644
index 00000000000..a72e926a471
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java
@@ -0,0 +1,70 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author mpolden
+ */
+public class ContainerOperationsTest {
+
+ private final TestTaskContext context = new TestTaskContext();
+ private final ContainerEngineMock containerEngine = new ContainerEngineMock();
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final TestTimer timer = new TestTimer();
+ private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer);
+
+ @Test
+ void no_managed_containers_running() {
+ Container c1 = createContainer("c1", true);
+ Container c2 = createContainer("c2", false);
+
+ containerEngine.addContainer(c1);
+ assertFalse(containerOperations.noManagedContainersRunning(context));
+
+ containerEngine.removeContainer(context, c1);
+ assertTrue(containerOperations.noManagedContainersRunning(context));
+
+ containerEngine.addContainer(c2);
+ assertTrue(containerOperations.noManagedContainersRunning(context));
+ }
+
+ @Test
+ void retain_managed_containers() {
+ Container c1 = createContainer("c1", true);
+ Container c2 = createContainer("c2", true);
+ Container c3 = createContainer("c3", false);
+ containerEngine.addContainers(List.of(c1, c2, c3));
+
+ assertEquals(3, containerEngine.listContainers(context).size());
+ containerOperations.retainManagedContainers(context, Set.of(c1.name()));
+
+ assertEquals(List.of(c1.name(), c3.name()), containerEngine.listContainers(context).stream()
+ .map(PartialContainer::name)
+ .sorted()
+ .toList());
+ }
+
+ private Container createContainer(String name, boolean managed) {
+ return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running,
+ "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name,
+ ContainerResources.UNLIMITED, List.of(), managed);
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java
new file mode 100644
index 00000000000..cbc803b6105
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java
@@ -0,0 +1,49 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author freva
+ */
+public class ContainerResourcesTest {
+
+ @Test
+ void verify_unlimited() {
+ assertEquals(-1, ContainerResources.UNLIMITED.cpuQuota());
+ assertEquals(100_000, ContainerResources.UNLIMITED.cpuPeriod());
+ assertEquals(0, ContainerResources.UNLIMITED.cpuShares());
+ }
+
+ @Test
+ void validate_shares() {
+ new ContainerResources(0, 0, 0);
+ new ContainerResources(0, 2, 0);
+ new ContainerResources(0, 2048, 0);
+ new ContainerResources(0, 262_144, 0);
+
+ assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, -1, 0)); // Negative shares not allowed
+ assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 1, 0)); // 1 share not allowed
+ assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 262_145, 0));
+ }
+
+ @Test
+ void cpu_shares_scaling() {
+ ContainerResources resources = ContainerResources.from(5.3, 2.5, 0);
+ assertEquals(530_000, resources.cpuQuota());
+ assertEquals(100_000, resources.cpuPeriod());
+ assertEquals(80, resources.cpuShares());
+ }
+
+ private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) {
+ try {
+ runnable.run();
+ fail("Expected " + clazz);
+ } catch (Throwable e) {
+ if (!clazz.isInstance(e)) throw e;
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java
new file mode 100644
index 00000000000..8cd3d6529c5
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java
@@ -0,0 +1,147 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container;
+
+import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup;
+import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController;
+import com.yahoo.vespa.hosted.node.admin.cgroup.Size;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC;
+import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author mpolden
+ */
+public class ContainerStatsCollectorTest {
+
+ private final TestTerminal testTerminal = new TestTerminal();
+ private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal);
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Cgroup cgroup = mock(Cgroup.class, Answers.RETURNS_DEEP_STUBS);
+ private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build())
+ .fileSystem(TestFileSystem.create())
+ .build();
+ @Test
+ void collect() throws Exception {
+ ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24);
+ ContainerId containerId = new ContainerId("id1");
+ int containerPid = 42;
+ assertTrue(collector.collect(context, containerId, containerPid, "eth0").isEmpty(), "No stats found");
+
+ mockMemoryStats(containerId);
+ mockCpuStats(containerId);
+ mockNetworkStats(containerPid);
+
+ Optional<ContainerStats> stats = collector.collect(context, containerId, containerPid, "eth0");
+ assertTrue(stats.isPresent());
+ assertEquals(new ContainerStats.CpuStats(24, 6049374780000L, 691675615472L,
+ 262190000000L, 3L, 1L, 2L),
+ stats.get().cpuStats());
+ assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L),
+ stats.get().memoryStats());
+ assertEquals(Map.of("eth0", new ContainerStats.NetworkStats(22280813L, 4L, 3L,
+ 19859383L, 6L, 5L)),
+ stats.get().networks());
+ assertEquals(List.of(), stats.get().gpuStats());
+
+ mockGpuStats();
+ stats = collector.collect(context, containerId, containerPid, "eth0");
+ assertTrue(stats.isPresent());
+ assertEquals(List.of(new ContainerStats.GpuStats(0, 35, 16106127360L, 6144655360L),
+ new ContainerStats.GpuStats(1, 67, 32212254720L, 19314769920L)),
+ stats.get().gpuStats());
+ }
+
+ private void mockGpuStats() throws IOException {
+ Path devPath = fileSystem.getPath("/dev");
+ Files.createDirectories(devPath);
+ Files.createFile(devPath.resolve("nvidia0"));
+ testTerminal.expectCommand("nvidia-smi --query-gpu=index,utilization.gpu,memory.total,memory.free --format=csv,noheader,nounits 2>&1", 0,
+ """
+ 0, 35, 15360, 9500
+ 1, 67, 30720, 12300
+ """);
+ }
+
+ private void mockNetworkStats(int pid) {
+ UnixPath dev = new UnixPath(fileSystem.getPath("/proc/" + pid + "/net/dev"));
+ dev.createParents().writeUtf8File("Inter-| Receive | Transmit\n" +
+ " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" +
+ " lo: 36289258 149700 0 0 0 0 0 0 36289258 149700 0 0 0 0 0 0\n" +
+ " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n");
+ }
+
+ private void mockMemoryStats(ContainerId containerId) {
+ when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L));
+ when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L));
+ when(cgroup.resolveContainer(eq(containerId)).memory().readStat()).thenReturn(
+ new MemoryController.Stats(Size.from(470790144L), Size.from(0), Size.from(0), Size.from(0), Size.from(0)));
+ }
+
+ private void mockCpuStats(ContainerId containerId) throws IOException {
+ UnixPath proc = new UnixPath(fileSystem.getPath("/proc"));
+ proc.createDirectories();
+
+ when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of(
+ TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L,
+ TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L));
+
+ proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" +
+ "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" +
+ "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" +
+ "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" +
+ "cpu3 328107 50628 93406 23605135 44378 1048549 30199 0 0 0\n" +
+ "cpu4 267474 50404 99253 23606041 113094 1038572 26494 0 0 0\n" +
+ "cpu5 309584 50677 94284 23550372 132616 1033661 29436 0 0 0\n" +
+ "cpu6 477926 56888 121251 23367023 83121 1074930 28818 0 0 0\n" +
+ "cpu7 335335 29350 106130 23551107 95606 1066394 26156 0 0 0\n" +
+ "cpu8 323678 28629 99171 23586501 82183 1064708 25403 0 0 0\n" +
+ "cpu9 329805 27516 98538 23579458 89235 1061561 25140 0 0 0\n" +
+ "cpu10 291536 26455 93934 23642345 81282 1049736 25228 0 0 0\n" +
+ "cpu11 271103 25302 90630 23663641 85711 1048781 24291 0 0 0\n" +
+ "cpu12 323634 63392 100406 23465340 132684 1089157 28319 0 0 0\n" +
+ "cpu13 348085 49568 100772 23490388 114190 1079474 20948 0 0 0\n" +
+ "cpu14 310712 51208 90461 23547980 101601 1071940 26712 0 0 0\n" +
+ "cpu15 360405 52754 94620 23524878 79851 1062050 26836 0 0 0\n" +
+ "cpu16 367893 52141 98074 23541314 57500 1058968 25242 0 0 0\n" +
+ "cpu17 412756 51486 101592 23515056 47653 1044874 27467 0 0 0\n" +
+ "cpu18 287307 25478 106011 23599505 79848 1089812 23160 0 0 0\n" +
+ "cpu19 275001 24421 98338 23628694 79675 1084074 22083 0 0 0\n" +
+ "cpu20 288038 24805 94432 23629908 74735 1078501 21915 0 0 0\n" +
+ "cpu21 295373 25017 91344 23628585 75282 1071019 22026 0 0 0\n" +
+ "cpu22 326739 25588 90385 23608217 69186 1068494 21108 0 0 0\n" +
+ "cpu23 452284 24602 104397 23481583 72612 1052462 21985 0 0 0\n" +
+ "intr 6645352968 64 0 0 0 1481 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 4334106 1 6949071 5814662 5415344 6939471 6961483 6358810 5271953 6718644 0 126114 126114 126114 126114 126114 126114 126114 126114 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" +
+ "ctxt 2495530303\n" +
+ "btime 1611928223\n" +
+ "processes 4839481\n" +
+ "procs_running 4\n" +
+ "procs_blocked 0\n" +
+ "softirq 2202631388 4 20504999 46734 54405637 4330276 0 6951 1664780312 10130 458546345\n");
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java
new file mode 100644
index 00000000000..37db6895040
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java
@@ -0,0 +1,37 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class ContainerImageDownloaderTest {
+
+ @Test
+ @Timeout(5_000)
+ void test_download() {
+ ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true);
+ ContainerImageDownloader downloader = new ContainerImageDownloader(podman, new TestTimer());
+ TaskContext context = new TestTaskContext();
+ DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42");
+
+ assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download started");
+ assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download pending");
+ podman.completeDownloadOf(image);
+ boolean downloadCompleted;
+ while (!(downloadCompleted = downloader.get(context, image, () -> RegistryCredentials.none))) ;
+ assertTrue(downloadCompleted, "Download completed");
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java
new file mode 100644
index 00000000000..71312125cbc
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java
@@ -0,0 +1,184 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.image;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.container.Container;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerResources;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author freva
+ * @author mpolden
+ */
+public class ContainerImagePrunerTest {
+
+ private final Tester tester = new Tester();
+
+ @Test
+ void noImagesMeansNoUnusedImages() {
+ tester.withExistingImages()
+ .expectDeletedImages();
+ }
+
+ @Test
+ void singleImageWithoutContainersIsUnused() {
+ tester.withExistingImages(image("image-1"))
+ // Even though nothing is using the image, we will keep it for at least 1h
+ .expectDeletedImagesAfterMinutes(0)
+ .expectDeletedImagesAfterMinutes(30)
+ .expectDeletedImagesAfterMinutes(30, "image-1");
+ }
+
+ @Test
+ void singleImageWithContainerIsUsed() {
+ tester.withExistingImages(image("image-1"))
+ .withExistingContainers(container("container-1", "image-1"))
+ .expectDeletedImages();
+ }
+
+ @Test
+ void multipleUnusedImagesAreIdentified() {
+ tester.withExistingImages(image("image-1"), image("image-2"))
+ .expectDeletedImages("image-1", "image-2");
+ }
+
+ @Test
+ void unusedImagesWithMultipleTags() {
+ tester.withExistingImages(image("image-1", "vespa-6", "vespa-6.28", "vespa:latest"))
+ .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest");
+ }
+
+ @Test
+ void unusedImagesWithMultipleUntagged() {
+ tester.withExistingImages(image("image1", "<none>:<none>"),
+ image("image2", "<none>:<none>"))
+ .expectDeletedImages("image1", "image2");
+ }
+
+ @Test
+ void taggedImageWithNoContainersIsUnused() {
+ tester.withExistingImages(image("image-1", "vespa-6"))
+ .expectDeletedImages("vespa-6");
+ }
+
+ @Test
+ void reDownloadingImageIsNotImmediatelyDeleted() {
+ tester.withExistingImages(image("image"))
+ .expectDeletedImages("image") // After 1h we delete image
+ .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted
+ .expectDeletedImagesAfterMinutes(10)
+ .expectDeletedImages("image"); // 1h after re-download it is deleted again
+ }
+
+ @Test
+ void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() {
+ tester.withExistingImages(image("image", "my-tag"))
+ .expectDeletedImages("my-tag") // After 1h we delete image
+ .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted
+ .expectDeletedImagesAfterMinutes(10)
+ .expectDeletedImages("my-tag"); // 1h after re-download it is deleted again
+ }
+
+ /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */
+ @Test
+ void doesNotDeleteExcludedByIdImages() {
+ tester.withExistingImages(image("image-1"), image("image-2"))
+ // Normally, image-1 should also be deleted, but because we exclude image-1 only image-2 is deleted
+ .expectDeletedImages(List.of("image-1"), "image-2");
+ }
+
+ /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */
+ @Test
+ void doesNotDeleteExcludedByTagImages() {
+ tester.withExistingImages(image("image-1", "vespa:6.288.16"), image("image-2", "vespa:6.289.94"))
+ .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94");
+ }
+
+ @Test
+ void excludingNotDownloadedImageIsNoop() {
+ tester.withExistingImages(image("image-1", "vespa:6.288.16"),
+ image("image-2", "vespa:6.289.94"))
+ .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6");
+ }
+
+ private static Image image(String id, String... tags) {
+ return new Image(id, List.of(tags));
+ }
+
+ private static Container container(String name, String imageId) {
+ return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH,
+ Container.State.running, imageId, DockerImage.EMPTY, Map.of(),
+ 42, 43, name + ".example.com", ContainerResources.UNLIMITED,
+ List.of(), true);
+ }
+
+ private static class Tester {
+
+ private final ContainerEngineMock containerEngine = new ContainerEngineMock();
+ private final TaskContext context = new TestTaskContext();
+ private final TestTimer timer = new TestTimer();
+ private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, timer);
+ private final Map<String, Integer> removalCountByImageId = new HashMap<>();
+
+ private boolean initialized = false;
+
+ private Tester withExistingImages(Image... images) {
+ containerEngine.setImages(List.of(images));
+ return this;
+ }
+
+ private Tester withExistingContainers(Container... containers) {
+ containerEngine.addContainers(List.of(containers));
+ return this;
+ }
+
+ private Tester expectDeletedImages(String... imageIds) {
+ return expectDeletedImagesAfterMinutes(60, imageIds);
+ }
+
+ private Tester expectDeletedImages(List<String> excludedRefs, String... imageIds) {
+ return expectDeletedImagesAfterMinutes(60, excludedRefs, imageIds);
+ }
+
+ private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) {
+ return expectDeletedImagesAfterMinutes(minutesAfter, List.of(), imageIds);
+ }
+
+ private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) {
+ if (!initialized) {
+ // Run once with a very long expiry to initialize internal state of existing images
+ pruner.removeUnusedImages(context, List.of(), Duration.ofDays(999));
+ initialized = true;
+ }
+
+ timer.advance(Duration.ofMinutes(minutesAfter));
+
+ pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1));
+
+ List.of(imageIds)
+ .forEach(imageId -> {
+ int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1;
+ removalCountByImageId.put(imageId, newValue);
+
+ assertTrue(containerEngine.listImages(context).stream().noneMatch(image -> image.id().equals(imageId)),
+ "Image " + imageId + " removed");
+ });
+ return this;
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java
new file mode 100644
index 00000000000..8e23c7e54b6
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java
@@ -0,0 +1,99 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.container.metrics;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.APPLICATION_HOST;
+import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.DimensionType.DEFAULT;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class MetricsTest {
+ private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
+ private final Metrics metrics = new Metrics();
+
+ @Test
+ void testDefaultValue() {
+ metrics.declareCounter("some.name", hostDimension);
+
+ assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L);
+ }
+
+ @Test
+ void testSimpleIncrementMetric() {
+ Counter counter = metrics.declareCounter("a_counter.value", hostDimension);
+
+ counter.add(5);
+ counter.add(8);
+
+ Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension);
+ assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set");
+ assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8
+ }
+
+ @Test
+ void testSimpleGauge() {
+ Gauge gauge = metrics.declareGauge("test.gauge", hostDimension);
+
+ gauge.sample(42);
+ gauge.sample(-342.23);
+
+ Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension);
+ assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set");
+ assertEquals(latestMetrics.get("test.gauge"), -342.23);
+ }
+
+ @Test
+ void testRedeclaringSameGauge() {
+ Gauge gauge = metrics.declareGauge("test.gauge", hostDimension);
+ gauge.sample(42);
+
+ // Same as hostDimension, but new instance.
+ Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
+ Gauge newGauge = metrics.declareGauge("test.gauge", newDimension);
+ newGauge.sample(56);
+
+ assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.);
+ }
+
+ @Test
+ void testSameMetricNameButDifferentDimensions() {
+ Gauge gauge = metrics.declareGauge("test.gauge", hostDimension);
+ gauge.sample(42);
+
+ // Not the same as hostDimension.
+ Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build();
+ Gauge newGauge = metrics.declareGauge("test.gauge", newDimension);
+ newGauge.sample(56);
+
+ assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.);
+ assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.);
+ }
+
+ @Test
+ void testDeletingMetric() {
+ metrics.declareGauge("test.gauge", hostDimension);
+
+ Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build();
+ metrics.declareGauge("test.gauge", differentDimension);
+
+ assertEquals(2, metrics.getMetricsByType(DEFAULT).size());
+ metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT);
+ assertEquals(1, metrics.getMetricsByType(DEFAULT).size());
+ assertEquals(getMetricsForDimension(hostDimension).size(), 1);
+ assertEquals(getMetricsForDimension(differentDimension).size(), 0);
+ }
+
+ private Map<String, Number> getMetricsForDimension(Dimensions dimensions) {
+ return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT)
+ .getOrDefault(dimensions, Map.of())
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java
new file mode 100644
index 00000000000..de41da7329b
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author freva
+ */
+public class ContainerFailTest {
+
+ @Test
+ void test() {
+ DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image");
+ try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) {
+ ContainerName containerName = new ContainerName("host1");
+ String hostname = "host1.test.yahoo.com";
+ NodeSpec nodeSpec = NodeSpec.Builder
+ .testSpec(hostname)
+ .wantedDockerImage(dockerImage)
+ .currentDockerImage(dockerImage)
+ .build();
+ tester.addChildNodeRepositoryNode(nodeSpec);
+
+ NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build();
+
+ tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any());
+ tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+
+ tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get());
+
+ tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any());
+ tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any());
+ tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+
+ verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any());
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
new file mode 100644
index 00000000000..b4d85a5e974
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java
@@ -0,0 +1,182 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfo;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfoReader;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.nio.file.FileSystem;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author musum
+ */
+// Need to deconstruct nodeAdminStateUpdater
+public class ContainerTester implements AutoCloseable {
+
+ private static final Logger log = Logger.getLogger(ContainerTester.class.getName());
+ static final HostName HOST_HOSTNAME = HostName.of("host.test.yahoo.com");
+
+ private final Thread loopThread;
+ private final Phaser phaser = new Phaser(1);
+
+ private final ContainerEngineMock containerEngine = new ContainerEngineMock();
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final TestTimer timer = new TestTimer();
+ final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer));
+ final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
+ final Orchestrator orchestrator = mock(Orchestrator.class);
+ final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
+ final InOrder inOrder = Mockito.inOrder(containerOperations, nodeRepository, orchestrator, storageMaintainer);
+ final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+
+ final NodeAdminStateUpdater nodeAdminStateUpdater;
+ final NodeAdminImpl nodeAdmin;
+
+ private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED;
+
+
+ ContainerTester(List<DockerImage> images) {
+ images.forEach(image -> containerEngine.pullImage(null, image, RegistryCredentials.none));
+ when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty());
+
+ IPAddressesMock ipAddresses = new IPAddressesMock();
+ ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1");
+ ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::");
+ for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i);
+
+ NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build();
+ nodeRepository.updateNodeSpec(hostSpec);
+
+ Metrics metrics = new Metrics();
+ FileSystem fileSystem = TestFileSystem.create();
+ ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class);
+ when(procMeminfoReader.read()).thenReturn(new ProcMeminfo(1, 2));
+
+ NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) ->
+ new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none,
+ storageMaintainer, flagSource,
+ List.of(), Optional.empty(), Optional.empty(), timer, Duration.ofSeconds(-1),
+ VespaServiceDumper.DUMMY_INSTANCE, List.of()) {
+ @Override public void converge(NodeAgentContext context) {
+ super.converge(context);
+ phaser.arriveAndAwaitAdvance();
+ }
+ @Override public void stopForHostSuspension(NodeAgentContext context) {
+ super.stopForHostSuspension(context);
+ phaser.arriveAndAwaitAdvance();
+ }
+ @Override public void stopForRemoval(NodeAgentContext context) {
+ super.stopForRemoval(context);
+ phaser.arriveAndDeregister();
+ }
+ };
+ nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, timer, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader);
+ NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) ->
+ NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build();
+ nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator,
+ nodeAdmin, HOST_HOSTNAME);
+
+ loopThread = new Thread(() -> {
+ nodeAdminStateUpdater.start();
+ while ( ! phaser.isTerminated()) {
+ try {
+ nodeAdminStateUpdater.converge(wantedState);
+ } catch (RuntimeException e) {
+ log.info(e.getMessage());
+ }
+ }
+ nodeAdminStateUpdater.stop();
+ });
+ loopThread.start();
+ }
+
+ /** Adds a node to node-repository mock that is running on this host */
+ void addChildNodeRepositoryNode(NodeSpec nodeSpec) {
+ if (nodeSpec.wantedDockerImage().isPresent()) {
+ if (!containerEngine.hasImage(null, nodeSpec.wantedDockerImage().get())) {
+ throw new IllegalArgumentException("Want to use image " + nodeSpec.wantedDockerImage().get() +
+ ", but that image does not exist in the container engine");
+ }
+ }
+
+ if (nodeRepository.getOptionalNode(nodeSpec.hostname()).isEmpty())
+ phaser.register();
+
+ nodeRepository.updateNodeSpec(new NodeSpec.Builder(nodeSpec)
+ .parentHostname(HOST_HOSTNAME.value())
+ .build());
+ }
+
+ void setWantedState(NodeAdminStateUpdater.State wantedState) {
+ this.wantedState = wantedState;
+ }
+
+ <T> T inOrder(T t) {
+ waitSomeTicks();
+ return inOrder.verify(t);
+ }
+
+ void waitSomeTicks() {
+ try {
+ // 3 is enough for everyone! (Well, maybe not for all eternity ...)
+ for (int i = 0; i < 3; i++)
+ phaser.awaitAdvanceInterruptibly(phaser.arrive(), 1000, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException | TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static NodeAgentContext containerMatcher(ContainerName containerName) {
+ return argThat((ctx) -> ctx.containerName().equals(containerName));
+ }
+
+ @Override
+ public void close() {
+ phaser.forceTermination();
+ try {
+ loopThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java
new file mode 100644
index 00000000000..7e874bcd5a7
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+
+/**
+ * @author freva
+ */
+public class MultiContainerTest {
+
+ @Test
+ void test() {
+ DockerImage image1 = DockerImage.fromString("registry.example.com/repo/image1");
+ DockerImage image2 = DockerImage.fromString("registry.example.com/repo/image2");
+ try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) {
+ addAndWaitForNode(tester, "host1.test.yahoo.com", image1);
+ NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2);
+
+ tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build());
+
+ ContainerName host2 = new ContainerName("host2");
+ tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(host2), any());
+ tester.inOrder(tester.storageMaintainer).archiveNodeStorage(
+ argThat(context -> context.containerName().equals(host2)));
+ tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready));
+
+ addAndWaitForNode(tester, "host3.test.yahoo.com", image1);
+ }
+ }
+
+ private NodeAgentContext containerMatcher(ContainerName containerName) {
+ return argThat((ctx) -> ctx.containerName().equals(containerName));
+ }
+
+ private NodeSpec addAndWaitForNode(ContainerTester tester, String hostName, DockerImage dockerImage) {
+ NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build();
+ tester.addChildNodeRepositoryNode(nodeSpec);
+
+ ContainerName containerName = ContainerName.fromHostname(hostName);
+ tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any());
+ tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName));
+ tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any());
+
+ return nodeSpec;
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java
new file mode 100644
index 00000000000..da14c5aa47b
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java
@@ -0,0 +1,91 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NoSuchNodeException;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+
+/**
+ * Mock with some simple logic
+ *
+ * @author dybis
+ */
+public class NodeRepoMock implements NodeRepository {
+
+ private final Map<String, NodeSpec> nodeSpecByHostname = new ConcurrentHashMap<>();
+ private volatile Map<String, Acl> aclByHostname = Map.of();
+
+ @Override
+ public void addNodes(List<AddNode> nodes) { }
+
+ @Override
+ public List<NodeSpec> getNodes(String baseHostName) {
+ return nodeSpecByHostname.values().stream()
+ .filter(node -> baseHostName.equals(node.parentHostname().orElse(null)))
+ .toList();
+ }
+
+ @Override
+ public Optional<NodeSpec> getOptionalNode(String hostName) {
+ return Optional.ofNullable(nodeSpecByHostname.get(hostName));
+ }
+
+ @Override
+ public Map<String, Acl> getAcls(String hostname) {
+ return aclByHostname;
+ }
+
+ @Override
+ public List<WireguardPeer> getExclavePeers() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<WireguardPeer> getConfigserverPeers() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) {
+ updateNodeSpec(new NodeSpec.Builder(getNode(hostName))
+ .updateFromNodeAttributes(nodeAttributes)
+ .build());
+ }
+
+ @Override
+ public void setNodeState(String hostName, NodeState nodeState) {
+ updateNodeSpec(new NodeSpec.Builder(getNode(hostName))
+ .state(nodeState)
+ .build());
+ }
+
+ public void updateNodeSpec(NodeSpec nodeSpec) {
+ nodeSpecByHostname.put(nodeSpec.hostname(), nodeSpec);
+ }
+
+ public void updateNodeSpec(String hostname, Function<NodeSpec.Builder, NodeSpec.Builder> mapper) {
+ nodeSpecByHostname.compute(hostname, (__, nodeSpec) -> {
+ if (nodeSpec == null) throw new NoSuchNodeException(hostname);
+ return mapper.apply(new NodeSpec.Builder(nodeSpec)).build();
+ });
+ }
+
+ public void resetNodeSpecs() {
+ nodeSpecByHostname.clear();
+ }
+
+ public void setAcl(Map<String, Acl> aclByHostname) {
+ this.aclByHostname = Map.copyOf(aclByHostname);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java
new file mode 100644
index 00000000000..a1440ba8669
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java
@@ -0,0 +1,44 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.HOST_HOSTNAME;
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
+/**
+ * Tests rebooting of Docker host
+ *
+ * @author musum
+ */
+public class RebootTest {
+
+ private final String hostname = "host1.test.yahoo.com";
+ private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image");
+
+ @Test
+ void test() {
+ try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) {
+ tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build());
+
+ ContainerName host1 = new ContainerName("host1");
+ tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any());
+
+ tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED);
+
+ tester.inOrder(tester.orchestrator).suspend(eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value())));
+ tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1));
+ assertTrue(tester.nodeAdmin.setFrozen(true));
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java
new file mode 100644
index 00000000000..1445546097a
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java
@@ -0,0 +1,50 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.integration;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
+/**
+ * Tests that different wanted and current restart generation leads to execution of restart command
+ *
+ * @author musum
+ */
+public class RestartTest {
+
+ @Test
+ void test() {
+ DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3");
+ try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) {
+ String hostname = "host1.test.yahoo.com";
+ NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname)
+ .wantedDockerImage(dockerImage)
+ .wantedVespaVersion(dockerImage.tagAsVersion())
+ .build();
+ tester.addChildNodeRepositoryNode(nodeSpec);
+
+ ContainerName host1 = new ContainerName("host1");
+ tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any());
+ tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+ eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())));
+
+ // Increment wantedRestartGeneration to 2 in node-repo
+ tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname))
+ .wantedRestartGeneration(2).build());
+
+ tester.inOrder(tester.orchestrator).suspend(eq(hostname));
+ tester.inOrder(tester.containerOperations).restartVespa(containerMatcher(host1));
+ tester.inOrder(tester.nodeRepository).updateNodeAttributes(
+ eq(hostname), eq(new NodeAttributes().withRestartGeneration(2)));
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
new file mode 100644
index 00000000000..51b3bb5e6c4
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -0,0 +1,178 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance;
+
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
+import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+/**
+ * @author dybis
+ */
+public class StorageMaintainerTest {
+
+ private final TestTerminal terminal = new TestTerminal();
+ private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class);
+ private final DiskCleanup diskCleanup = mock(DiskCleanup.class);
+ private final SyncClient syncClient = mock(SyncClient.class);
+ private final TestTimer timer = new TestTimer(Instant.ofEpochSecond(1234567890));
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, timer,
+ fileSystem.getPath("/data/vespa/storage/container-archive"));
+
+ @Test
+ void testDiskUsed() {
+ NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build();
+
+ terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/");
+ assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context));
+
+ // Value should still be cached, no new execution against the terminal
+ assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context));
+ }
+
+ @Test
+ void testNonExistingDiskUsed() {
+ DiskSize size = storageMaintainer.getDiskUsed(null, Path.of("/fake/path"));
+ assertEquals(DiskSize.ZERO, size);
+ }
+
+ @Test
+ void archive_container_data_test() throws IOException {
+ // Create some files in containers
+ NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1");
+ createNodeAgentContextAndContainerStorage(fileSystem, "container-2");
+
+ Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive");
+ Files.createDirectories(pathToArchiveDir);
+
+ Path containerStorageRoot = context1.paths().of("/").pathOnHost().getParent();
+ Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot)
+ .maxDepth(1)
+ .stream()
+ .map(FileFinder.FileAttributes::filename)
+ .collect(Collectors.toSet());
+ assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive);
+
+
+ // Archive container-1
+ storageMaintainer.archiveNodeStorage(context1);
+
+ timer.advance(Duration.ofSeconds(3));
+ storageMaintainer.archiveNodeStorage(context1);
+
+ // container-1 should be gone from container-storage
+ Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot)
+ .maxDepth(1)
+ .stream()
+ .map(FileFinder.FileAttributes::filename)
+ .collect(Collectors.toSet());
+ assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive);
+
+ // container archive directory should contain exactly 1 directory - the one we just archived
+ List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list();
+ assertEquals(1, containerArchiveContentsAfterArchive.size());
+ Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path();
+ assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString());
+ Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath)
+ .stream()
+ .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString())
+ .collect(Collectors.toSet());
+ assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents);
+ }
+
+ private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException {
+ NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld")
+ .fileSystem(fileSystem).build();
+
+ ContainerPath containerVespaHome = context.paths().underVespaHome("");
+ Files.createDirectories(context.paths().of("/etc/something"));
+ Files.createFile(context.paths().of("/etc/something/conf"));
+
+ Files.createDirectories(containerVespaHome.resolve("logs/vespa"));
+ Files.createFile(containerVespaHome.resolve("logs/vespa/vespa.log"));
+ Files.createFile(containerVespaHome.resolve("logs/vespa/zookeeper.log"));
+
+ Files.createDirectories(containerVespaHome.resolve("var/db"));
+ Files.createFile(containerVespaHome.resolve("var/db/some-file"));
+
+ Files.createDirectories(containerVespaHome.resolve("var/tmp"));
+ Files.createFile(containerVespaHome.resolve("var/tmp/some-file"));
+
+ ContainerPath containerRoot = context.paths().of("/");
+ Set<String> actualContents = FileFinder.files(containerRoot)
+ .stream()
+ .map(fileAttributes -> containerRoot.relativize(fileAttributes.path()).toString())
+ .collect(Collectors.toSet());
+ Set<String> expectedContents = Set.of(
+ "etc/something/conf",
+ "opt/vespa/logs/vespa/vespa.log",
+ "opt/vespa/logs/vespa/zookeeper.log",
+ "opt/vespa/var/tmp/some-file",
+ "opt/vespa/var/db/some-file");
+ assertEquals(expectedContents, actualContents);
+ return context;
+ }
+
+ @Test
+ void not_run_if_not_enough_used() {
+ NodeAgentContext context = NodeAgentContextImpl.builder(
+ NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build())
+ .fileSystem(fileSystem).build();
+ mockDiskUsage(500L);
+
+ storageMaintainer.cleanDiskIfFull(context);
+ verifyNoInteractions(diskCleanup);
+ }
+
+ @Test
+ void deletes_correct_amount() {
+ NodeAgentContext context = NodeAgentContextImpl.builder(
+ NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build())
+ .fileSystem(fileSystem).build();
+
+ mockDiskUsage(950_000L);
+
+ storageMaintainer.cleanDiskIfFull(context);
+ // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB
+ verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L));
+ }
+
+ @AfterEach
+ public void after() {
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ private void mockDiskUsage(long kBytes) {
+ terminal.expectCommand("du -xsk /data/vespa/storage/h123a 2>&1", 0, kBytes + "\t/path");
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
new file mode 100644
index 00000000000..063e8cb3f77
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
@@ -0,0 +1,351 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.endsWith;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class AclMaintainerTest {
+
+ private static final String EMPTY_FILTER_TABLE = "-P INPUT ACCEPT\n-P FORWARD ACCEPT\n-P OUTPUT ACCEPT\n";
+ private static final String EMPTY_NAT_TABLE = "-P PREROUTING ACCEPT\n-P INPUT ACCEPT\n-P OUTPUT ACCEPT\n-P POSTROUTING ACCEPT\n";
+
+ private final ContainerOperations containerOperations = mock(ContainerOperations.class);
+ private final IPAddressesMock ipAddresses = new IPAddressesMock();
+ private final AclMaintainer aclMaintainer = new AclMaintainer(containerOperations, ipAddresses);
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Function<Acl, NodeAgentContext> contextGenerator =
+ acl -> NodeAgentContextImpl.builder("container1.host.com").fileSystem(fileSystem).acl(acl).build();
+ private final List<String> writtenFileContents = new ArrayList<>();
+
+ @Test
+ void configures_full_container_acl_from_empty() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22, 4443)
+ .withTrustedNode("hostname1", "3001::abcd")
+ .withTrustedNode("hostname2", "3001::1234")
+ .withTrustedNode("hostname1", "192.168.0.5")
+ .withTrustedNode("hostname4", "172.16.5.234").build();
+ NodeAgentContext context = contextGenerator.apply(acl);
+
+ ipAddresses.addAddress(context.hostname().value(), "2001::1");
+ ipAddresses.addAddress(context.hostname().value(), "10.0.0.1");
+
+ whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE);
+ whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE);
+
+ aclMaintainer.converge(context);
+
+ verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any());
+ verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any());
+ verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any());
+ verifyNoMoreInteractions(containerOperations);
+
+ List<String> expected = List.of(
+ // IPv4 filter table restore
+ "*filter\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" +
+ "-A INPUT -s 172.16.5.234/32 -j ACCEPT\n" +
+ "-A INPUT -s 192.168.0.5/32 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" +
+ "COMMIT\n",
+
+ // IPv6 filter table restore
+ "*filter\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p ipv6-icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" +
+ "-A INPUT -s 3001::1234/128 -j ACCEPT\n" +
+ "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" +
+ "COMMIT\n",
+
+ // IPv4 nat table restore
+ "*nat\n" +
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 10.0.0.1/32 -j REDIRECT\n" +
+ "COMMIT\n",
+
+ // IPv6 nat table restore
+ "*nat\n" +
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 2001::1/128 -j REDIRECT\n" +
+ "COMMIT\n");
+ assertEquals(expected, writtenFileContents);
+ }
+
+ @Test
+ void configures_minimal_container_acl_from_empty() {
+ // The ACL spec is empty and our this node's addresses do not resolve
+ Acl acl = new Acl.Builder().withTrustedPorts().build();
+ NodeAgentContext context = contextGenerator.apply(acl);
+
+ whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE);
+ whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE);
+
+ aclMaintainer.converge(context);
+
+ verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any());
+ verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any());
+ verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any());
+ verifyNoMoreInteractions(containerOperations);
+
+ List<String> expected = List.of(
+ // IPv4 filter table restore
+ "*filter\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p icmp -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" +
+ "COMMIT\n",
+
+ // IPv6 filter table restore
+ "*filter\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p ipv6-icmp -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" +
+ "COMMIT\n");
+ assertEquals(expected, writtenFileContents);
+ }
+
+ @Test
+ void only_configure_iptables_for_ipversion_that_differs() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build();
+ NodeAgentContext context = contextGenerator.apply(acl);
+
+ ipAddresses.addAddress(context.hostname().value(), "2001::1");
+
+ whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE);
+ whenListRules(context, "filter", IPVersion.IPv6,
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p ipv6-icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" +
+ "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n");
+ whenListRules(context, "nat", IPVersion.IPv6,
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 2001::1/128 -j REDIRECT\n");
+
+ aclMaintainer.converge(context);
+
+ verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any());
+ verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any());
+ verify(containerOperations, never()).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); //we don't have a ip4 address for the container so no redirect
+ verifyNoMoreInteractions(containerOperations);
+
+ List<String> expected = List.of(
+ "*filter\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" +
+ "COMMIT\n");
+ assertEquals(expected, writtenFileContents);
+ }
+
+ @Test
+ void rollback_is_attempted_when_applying_acl_fail() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build();
+ NodeAgentContext context = contextGenerator.apply(acl);
+
+ ipAddresses.addAddress(context.hostname().value(), "2001::1");
+
+ whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE);
+ whenListRules(context, "filter", IPVersion.IPv6,
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p ipv6-icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" +
+ "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n");
+ whenListRules(context, "nat", IPVersion.IPv6,
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 2001::1/128 -j REDIRECT\n");
+
+ when(containerOperations.executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()))
+ .thenThrow(new RuntimeException("iptables restore failed"));
+
+ aclMaintainer.converge(context);
+
+ verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any());
+ verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any());
+ verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables"), eq("-F"), eq("-t"), eq("filter"));
+ verifyNoMoreInteractions(containerOperations);
+
+ aclMaintainer.converge(context);
+ }
+
+ @Test
+ public void config_server_acl() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22, 4443)
+ .withTrustedNode("cfg1", "2001:db8::1")
+ .withTrustedNode("cfg2", "2001:db8::2")
+ .withTrustedNode("cfg3", "2001:db8::3")
+ .withTrustedNode("cfg1", "172.17.0.41")
+ .withTrustedNode("cfg2", "172.17.0.42")
+ .withTrustedNode("cfg3", "172.17.0.43")
+ .build();
+ NodeAgentContext context = NodeAgentContextImpl.builder("cfg3.example.com")
+ .fileSystem(fileSystem)
+ .acl(acl)
+ .nodeSpecBuilder(builder -> builder.type(NodeType.config))
+ .build();
+
+ ipAddresses.addAddress(context.hostname().value(), "2001:db8::3");
+ ipAddresses.addAddress(context.hostname().value(), "172.17.0.43");
+
+ whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE);
+ whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE);
+ whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE);
+
+ aclMaintainer.converge(context);
+
+ verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any());
+ verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any());
+ verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any());
+ verifyNoMoreInteractions(containerOperations);
+
+ List<String> expected = List.of(
+ // IPv4 filter table restore
+ """
+ *filter
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT
+ -A INPUT -s 172.17.0.41/32 -j ACCEPT
+ -A INPUT -s 172.17.0.42/32 -j ACCEPT
+ -A INPUT -s 172.17.0.43/32 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp-port-unreachable
+ COMMIT
+ """,
+ // IPv6 filter table restore
+ """
+ *filter
+ -P INPUT ACCEPT
+ -P FORWARD ACCEPT
+ -P OUTPUT ACCEPT
+ -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+ -A INPUT -i lo -j ACCEPT
+ -A INPUT -p ipv6-icmp -j ACCEPT
+ -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT
+ -A INPUT -s 2001:db8::1/128 -j ACCEPT
+ -A INPUT -s 2001:db8::2/128 -j ACCEPT
+ -A INPUT -s 2001:db8::3/128 -j ACCEPT
+ -A INPUT -j REJECT --reject-with icmp6-port-unreachable
+ COMMIT
+ """,
+ // IPv4 nat table restore
+ """
+ *nat
+ -P PREROUTING ACCEPT
+ -P INPUT ACCEPT
+ -P OUTPUT ACCEPT
+ -P POSTROUTING ACCEPT
+ -A OUTPUT -d 172.17.0.43/32 -j REDIRECT
+ COMMIT
+ """,
+ // IPv6 nat table restore
+ """
+ *nat
+ -P PREROUTING ACCEPT
+ -P INPUT ACCEPT
+ -P OUTPUT ACCEPT
+ -P POSTROUTING ACCEPT
+ -A OUTPUT -d 2001:db8::3/128 -j REDIRECT
+ COMMIT
+ """);
+ assertEquals(expected, writtenFileContents);
+ }
+
+ @BeforeEach
+ public void setup() {
+ doAnswer(invoc -> {
+ String path = invoc.getArgument(2);
+ writtenFileContents.add(new UnixPath(path).readUtf8File());
+ return new CommandResult(null, 0, "");
+ }).when(containerOperations).executeCommandInNetworkNamespace(any(), endsWith("-restore"), any());
+ }
+
+ private void whenListRules(NodeAgentContext context, String table, IPVersion ipVersion, String output) {
+ when(containerOperations.executeCommandInNetworkNamespace(
+ eq(context), any(CommandLine.Options.class), eq(ipVersion.iptablesCmd()), eq("-S"), eq("-t"), eq(table)))
+ .thenReturn(new CommandResult(null, 0, output));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java
new file mode 100644
index 00000000000..52eac44fbc3
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java
@@ -0,0 +1,88 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class FilterTableLineEditorTest {
+
+ @Test
+ void filter_set_wanted_rules() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1").build();
+
+ assertFilterTableLineEditorResult(
+ acl, IPVersion.IPv6,
+
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n",
+
+ "-P INPUT ACCEPT\n" +
+ "-P FORWARD ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" +
+ "-A INPUT -i lo -j ACCEPT\n" +
+ "-A INPUT -p ipv6-icmp -j ACCEPT\n" +
+ "-A INPUT -p tcp -m multiport --dports 22 -j ACCEPT\n" +
+ "-A INPUT -s 3001::1/128 -j ACCEPT\n" +
+ "-A INPUT -j REJECT --reject-with icmp6-port-unreachable");
+ }
+
+ @Test
+ void produces_minimal_diff_simple() {
+ assertFilterTableDiff(List.of(2, 5, 3, 6, 1, 4), List.of(2, 5, 6, 1, 4),
+ "Patching file table:\n" +
+ "--A INPUT -s 2001::3/128 -j ACCEPT\n");
+ }
+
+ @Test
+ void produces_minimal_diff_complex() {
+ assertFilterTableDiff(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), List.of(5, 11, 6, 3, 10, 4, 8, 12),
+ "Patching file table:\n" +
+ "--A INPUT -s 2001::1/128 -j ACCEPT\n" +
+ "--A INPUT -s 2001::2/128 -j ACCEPT\n" +
+ "+-A INPUT -s 2001::11/128 -j ACCEPT\n" +
+ "+-A INPUT -s 2001::12/128 -j ACCEPT\n" +
+ "--A INPUT -s 2001::7/128 -j ACCEPT\n" +
+ "--A INPUT -s 2001::9/128 -j ACCEPT\n");
+ }
+
+ private static void assertFilterTableLineEditorResult(
+ Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) {
+ FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion);
+ Editor editor = new Editor(
+ "nat-table",
+ () -> List.of(currentFilterTable.split("\n")),
+ result -> assertEquals(expectedRestoreFileContent, String.join("\n", result)),
+ filterLineEditor);
+ editor.edit(m -> {});
+ }
+
+ private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) {
+ Acl.Builder currentAcl = new Acl.Builder();
+ NodeType nodeType = NodeType.tenant;
+ currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i));
+ List<String> currentTable = new ArrayList<>();
+
+ Acl.Builder wantedAcl = new Acl.Builder();
+ wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i));
+
+ new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6))
+ .edit(log -> {});
+
+ new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6))
+ .edit(log -> assertEquals(diff, log));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java
new file mode 100644
index 00000000000..d8d526050d7
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java
@@ -0,0 +1,96 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.acl;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class NatTableLineEditorTest {
+
+ @Test
+ void nat_set_redirect_rule_without_touching_docker_rules() {
+ assertNatTableLineEditorResult(
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-N DOCKER_OUTPUT\n" +
+ "-N DOCKER_POSTROUTING\n" +
+ "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" +
+ "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" +
+ "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" +
+ "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" +
+ "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" +
+ "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-N DOCKER_OUTPUT\n" +
+ "-N DOCKER_POSTROUTING\n" +
+ "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" +
+ "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" +
+ "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" +
+ "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" +
+ "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" +
+ "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n" +
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT");
+ }
+
+ @Test
+ void nat_cleanup_wrong_redirect_rules() {
+ assertNatTableLineEditorResult(
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 3001::2/128 -j REDIRECT\n",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT");
+ }
+
+ @Test
+ void nat_delete_duplicate_rules() {
+ assertNatTableLineEditorResult(
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 3001::2/128 -j REDIRECT\n" +
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT\n" +
+ "-A OUTPUT -d 3001::4/128 -j REDIRECT\n",
+
+ "-P PREROUTING ACCEPT\n" +
+ "-P INPUT ACCEPT\n" +
+ "-P OUTPUT ACCEPT\n" +
+ "-P POSTROUTING ACCEPT\n" +
+ "-A OUTPUT -d 3001::1/128 -j REDIRECT");
+ }
+
+ private static void assertNatTableLineEditorResult(String redirectRule, String currentNatTable, String expectedNatTable) {
+ NatTableLineEditor natLineEditor = NatTableLineEditor.from(redirectRule);
+ Editor editor = new Editor(
+ "nat-table",
+ () -> List.of(currentNatTable.split("\n")),
+ result -> assertEquals(expectedNatTable, String.join("\n", result)),
+ natLineEditor);
+ editor.edit(m -> {});
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
new file mode 100644
index 00000000000..9487affd376
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -0,0 +1,234 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class CoreCollectorTest {
+ private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L);
+
+ private final ContainerOperations docker = mock(ContainerOperations.class);
+ private final CoreCollector coreCollector = new CoreCollector(docker);
+ private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld")
+ .fileSystem(TestFileSystem.create()).build();
+
+ private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234"))
+ .createParents()
+ .createNewFile()
+ .setLastModifiedTime(CORE_CREATED)
+ .toPath();
+ private final String TEST_BIN_PATH = "/usr/bin/program";
+ private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]",
+ "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.",
+ "#0 0x00000000004004d8 in main (argv=...) at main.c:4", "4\t printf(argv[3]);",
+ "#0 0x00000000004004d8 in main (argv=...) at main.c:4");
+
+ @Test
+ void extractsBinaryPathTest() {
+ final String[] cmd = {"file", TEST_CORE_PATH.pathInContainer()};
+
+ mockExec(cmd,
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
+ "'/usr/bin/program'");
+ assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
+
+ mockExec(cmd,
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
+ "'/usr/bin/program --foo --bar baz'");
+ assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
+
+ mockExec(cmd,
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " +
+ "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " +
+ "execfn: '/usr/bin/program', platform: 'x86_64");
+ assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH));
+
+ String fallbackResponse = "/response/from/fallback";
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"},
+ """
+ GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1
+ Type “apropos word” to search for commands related to “word”…
+ Reading symbols from abc…(no debugging symbols found)…done.
+ [New LWP 23678]
+ Core was generated by `/response/from/fallback'. \s
+ Program terminated with signal SIGSEGV, Segmentation fault. \s
+ #0 0x0000000000400541 in main ()
+ #0 0x0000000000400541 in main ()
+ (gdb) bt
+ #0 0x0000000000400541 in main ()
+ (gdb)
+ """);
+ mockExec(cmd,
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style");
+ assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH));
+
+ mockExec(cmd, "", "Error code 1234");
+ assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH));
+ }
+
+ @Test
+ void extractsBinaryPathUsingGdbTest() {
+ String[] cmd = new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"};
+
+ mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'.");
+ assertEquals("/usr/bin/program-from-gdb", coreCollector.readBinPathFallback(context, TEST_CORE_PATH));
+
+ mockExec(cmd, "", "Error 123");
+ try {
+ coreCollector.readBinPathFallback(context, TEST_CORE_PATH);
+ fail("Expected not to be able to get bin path");
+ } catch (RuntimeException e) {
+ assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " +
+ "[/opt/rh/gcc-toolset-12/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage());
+ }
+ }
+
+ @Test
+ void extractsBacktraceUsingGdb() {
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+ assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false));
+
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+ "", "Failure");
+ try {
+ coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false);
+ fail("Expected not to be able to read backtrace");
+ } catch (RuntimeException e) {
+ assertEquals("Failed to read backtrace exit status 1, output 'Failure', Command: " +
+ "[" + GDB_PATH_RHEL8 + ", -n, -ex, set print frame-arguments none, -ex, bt, -batch, " +
+ "/usr/bin/program, /tmp/core.1234]", e.getMessage());
+ }
+ }
+
+ @Test
+ void extractsBacktraceFromAllThreadsUsingGdb() {
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n",
+ "-ex", "set print frame-arguments none",
+ "-ex", "thread apply all bt", "-batch",
+ "/usr/bin/program", "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+ assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true));
+ }
+
+ @Test
+ void collectsDataTest() {
+ mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()},
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
+ "'/usr/bin/program'");
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "thread apply all bt", "-batch",
+ "/usr/bin/program", "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+
+ var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH)
+ .setCreated(CORE_CREATED)
+ .setType(CoreDumpMetadata.Type.CORE_DUMP)
+ .setBacktrace(GDB_BACKTRACE)
+ .setBacktraceAllThreads(GDB_BACKTRACE);
+ assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
+ }
+
+ @Test
+ void collectsDataRelativePath() {
+ mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()},
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'sbin/distributord-bin'");
+ String absolutePath = "/opt/vespa/sbin/distributord-bin";
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "bt", "-batch", absolutePath, "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none",
+ "-ex", "thread apply all bt", "-batch", absolutePath, "/tmp/core.1234"},
+ String.join("\n", GDB_BACKTRACE));
+
+ var expected = new CoreDumpMetadata()
+ .setBinPath(absolutePath)
+ .setCreated(CORE_CREATED)
+ .setType(CoreDumpMetadata.Type.CORE_DUMP)
+ .setBacktrace(GDB_BACKTRACE)
+ .setBacktraceAllThreads(GDB_BACKTRACE);
+ assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
+ }
+
+ @Test
+ void collectsPartialIfBacktraceFailsTest() {
+ mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()},
+ "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " +
+ "'/usr/bin/program'");
+ mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"},
+ "", "Failure");
+
+ var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP);
+ assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
+ }
+
+ @Test
+ void reportsJstackInsteadOfGdbForJdkCores() {
+ mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()},
+ "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)");
+
+ String jdkPath = "/path/to/jdk/java";
+ mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"},
+ "Core was generated by `" + jdkPath + " -Dconfig.id=default/container.11 -XX:+Pre'.");
+
+ String jstack = "jstack11";
+ mockExec(new String[]{"jhsdb", "jstack", "--exe", jdkPath, "--core", "/tmp/core.1234"},
+ jstack);
+
+ var expected = new CoreDumpMetadata().setBinPath(jdkPath)
+ .setCreated(CORE_CREATED)
+ .setType(CoreDumpMetadata.Type.CORE_DUMP)
+ .setBacktraceAllThreads(List.of(jstack));
+ assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
+ }
+
+ @Test
+ void metadata_for_java_heap_dump() {
+ var expected = new CoreDumpMetadata().setBinPath("java")
+ .setType(CoreDumpMetadata.Type.JVM_HEAP)
+ .setCreated(CORE_CREATED)
+ .setBacktrace(List.of("Heap dump, no backtrace available"));
+
+ assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof"))
+ .createNewFile()
+ .setLastModifiedTime(CORE_CREATED)
+ .toPath()));
+ }
+
+ private void mockExec(String[] cmd, String output) {
+ mockExec(cmd, output, "");
+ }
+
+ private void mockExec(String[] cmd, String output, String error) {
+ mockExec(context, cmd, output, error);
+ }
+
+ private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) {
+ when(docker.executeCommandInContainer(context, context.users().root(), cmd))
+ .thenReturn(new CommandResult(null, error.isEmpty() ? 0 : 1, error.isEmpty() ? output : error));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
new file mode 100644
index 00000000000..e65a226b789
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -0,0 +1,300 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.security.KeyId;
+import com.yahoo.security.SealedSharedKey;
+import com.yahoo.security.SecretSharedKey;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class CoredumpHandlerTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld")
+ .fileSystem(fileSystem).build();
+ private final ContainerPath containerCrashPath = context.paths().of("/var/crash");
+ private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps");
+
+ private final CoreCollector coreCollector = mock(CoreCollector.class);
+ private final Cores cores = mock(Cores.class);
+ private final Metrics metrics = new Metrics();
+ private final TestTimer timer = new TestTimer();
+ @SuppressWarnings("unchecked")
+ private final Supplier<String> coredumpIdSupplier = mock(Supplier.class);
+ private final SecretSharedKeySupplier secretSharedKeySupplier = mock(SecretSharedKeySupplier.class);
+ private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+ private final CoredumpHandler coredumpHandler =
+ new CoredumpHandler(coreCollector, cores, containerCrashPath.pathInContainer(),
+ doneCoredumpsPath, metrics, timer, coredumpIdSupplier, secretSharedKeySupplier,
+ flagSource);
+
+ @Test
+ void coredump_enqueue_test() throws IOException {
+ ContainerPath crashPath = context.paths().of("/some/crash/path");
+ ContainerPath processingDir = context.paths().of("/some/other/processing");
+
+ Files.createDirectories(crashPath);
+ createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO);
+
+ assertFolderContents(crashPath, "bash.core.431");
+ Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir);
+ assertEquals(Optional.empty(), enqueuedPath);
+
+ // bash.core.431 finished writing... and 2 more have since been written
+ timer.advance(Duration.ofMinutes(3));
+ createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10));
+ createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5));
+
+ when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321");
+ enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir);
+ assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+ assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673");
+ assertFolderContents(processingDir, "id-123");
+ assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119");
+ verify(coredumpIdSupplier, times(1)).get();
+
+ // Enqueue another
+ enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir);
+ assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath);
+ assertFolderContents(crashPath, "bash.core.431");
+ assertFolderContents(processingDir, "id-123", "id-321");
+ assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673");
+ verify(coredumpIdSupplier, times(2)).get();
+ }
+
+ @Test
+ void enqueue_with_hs_err_files() throws IOException {
+ ContainerPath crashPath = context.paths().of("/some/crash/path");
+ ContainerPath processingDir = context.paths().of("/some/other/processing");
+ Files.createDirectories(crashPath);
+
+ createFileAged(crashPath.resolve("java.core.69"), Duration.ofSeconds(515));
+ createFileAged(crashPath.resolve("hs_err_pid69.log"), Duration.ofSeconds(520));
+
+ createFileAged(crashPath.resolve("java.core.2420"), Duration.ofSeconds(540));
+ createFileAged(crashPath.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549));
+ createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550));
+
+ when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321");
+ Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir);
+ assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+ assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69");
+ assertFolderContents(processingDir, "id-123");
+ assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420");
+ }
+
+ @Test
+ void coredump_to_process_test() throws IOException {
+ ContainerPath processingDir = context.paths().of("/some/other/processing");
+
+ // Initially there are no core dumps
+ Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, containerCrashPath, processingDir);
+ assertEquals(Optional.empty(), enqueuedPath);
+
+ // 3 core dumps occur
+ Files.createDirectories(containerCrashPath);
+ createFileAged(containerCrashPath.resolve("bash.core.431"), Duration.ZERO);
+ createFileAged(containerCrashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10));
+ createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5));
+
+ when(coredumpIdSupplier.get()).thenReturn("id-123");
+ enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir);
+ assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+
+ // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously
+ enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir);
+ assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath);
+ verify(coredumpIdSupplier, times(1)).get();
+ }
+
+ @Test
+ void gather_metadata_test() throws IOException {
+ var metadata = new CoreDumpMetadata().setKernelVersion("3.10.0-862.9.1.el7.x86_64")
+ .setBacktrace(List.of("call 1", "function 2", "something something"))
+ .setVespaVersion("6.48.4")
+ .setBinPath("/bin/bash")
+ .setCoreDumpPath(context.paths().of("/home/docker/dumps/container-123/id-123/dump_core.456"))
+ .setDockerImage(DockerImage.fromString("example.com/vespa/ci:6.48.4"));
+
+ new UnixPath(fileSystem.getPath("/proc/cpuinfo")).createParents().writeUtf8File("microcode\t: 0xf0");
+
+ ContainerPath coredumpDirectory = context.paths().of("/var/crash/id-123");
+ Files.createDirectories(coredumpDirectory.pathOnHost());
+ Files.createFile(coredumpDirectory.resolve("dump_core.456"));
+ when(coreCollector.collect(eq(context), eq(coredumpDirectory.resolve("dump_core.456"))))
+ .thenReturn(metadata);
+
+ assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory));
+ verify(coreCollector, times(1)).collect(any(), any());
+
+ // On second invocation the test already exist, so times(1) is not incremented
+ assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory));
+ doThrow(new IllegalStateException("Should not be invoked"))
+ .when(coreCollector).collect(any(), any());
+ verify(coreCollector, times(1)).collect(any(), any());
+ }
+
+ @Test
+ void cant_get_metadata_if_no_core_file() {
+ assertThrows(IllegalStateException.class, () -> {
+ coredumpHandler.gatherMetadata(context, context.paths().of("/fake/path"));
+ });
+ }
+
+ @Test
+ void fails_to_get_core_file_if_only_compressed_or_encrypted() {
+ assertThrows(IllegalStateException.class, () -> {
+ ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123");
+ Files.createDirectories(coredumpDirectory);
+ Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zst"));
+ Files.createFile(coredumpDirectory.resolve("dump_bash.core.543.zst.enc"));
+ coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory);
+ });
+ }
+
+ void do_process_single_coredump_test(String expectedCoreFileName) throws IOException {
+ ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123");
+ Files.createDirectories(coredumpDirectory);
+ Files.write(coredumpDirectory.resolve("metadata2.json"), "{\"test-metadata\":{}}".getBytes());
+ Files.createFile(coredumpDirectory.resolve("dump_bash.core.431"));
+ assertFolderContents(coredumpDirectory, "metadata2.json", "dump_bash.core.431");
+ CoreDumpMetadata expectedMetadata = new CoreDumpMetadata();
+ expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE");
+
+ coredumpHandler.processAndReportSingleCoreDump(context, coredumpDirectory, Optional.empty());
+ verify(coreCollector, never()).collect(any(), any());
+ verify(cores, times(1)).report(eq(context.hostname()), eq("id-123"), eq(expectedMetadata));
+ assertFalse(Files.exists(coredumpDirectory));
+ assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123");
+ assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata2.json", expectedCoreFileName);
+ }
+
+ @Test
+ void processing_single_coredump_test_without_encryption_throws() throws IOException {
+ assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst"));
+ }
+
+ @Test
+ void process_single_coredump_test_with_encryption() throws IOException {
+ flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "bar-key");
+ when(secretSharedKeySupplier.create(KeyId.ofString("bar-key"))).thenReturn(Optional.of(makeFixedSecretSharedKey()));
+ do_process_single_coredump_test("dump_bash.core.431.zst.enc");
+ }
+
+ @Test
+ void processing_throws_when_no_public_key_set_in_feature_flag() throws IOException {
+ flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), ""); // empty -> not set
+ verify(secretSharedKeySupplier, never()).create(any());
+ assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst"));
+ }
+
+ @Test
+ void processing_throws_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException {
+ flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "baz-key");
+ when(secretSharedKeySupplier.create(KeyId.ofString("baz-key"))).thenReturn(Optional.empty());
+ assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst"));
+ }
+
+ @Test
+ void report_enqueued_and_processed_metrics() throws IOException {
+ Path processingPath = containerCrashPath.resolve("processing");
+ Files.createFile(containerCrashPath.resolve("dump-1"));
+ Files.createFile(containerCrashPath.resolve("dump-2"));
+ Files.createFile(containerCrashPath.resolve("hs_err_pid2.log"));
+ Files.createDirectory(processingPath);
+ Files.createFile(processingPath.resolve("metadata2.json"));
+ Files.createFile(processingPath.resolve("dump-3"));
+
+ new UnixPath(doneCoredumpsPath.resolve("container-123").resolve("dump-3-folder").resolve("dump-3"))
+ .createParents()
+ .createNewFile();
+
+ coredumpHandler.updateMetrics(context, containerCrashPath);
+ List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED);
+ assertEquals(1, updatedMetrics.size());
+ Map<String, Number> values = updatedMetrics.get(0).getMetrics();
+ assertEquals(3, values.get("coredumps.enqueued").intValue());
+ assertEquals(1, values.get("coredumps.processed").intValue());
+ }
+
+ @BeforeEach
+ public void setup() throws IOException {
+ Files.createDirectories(containerCrashPath.pathOnHost());
+ }
+
+ private static void assertFolderContents(Path pathToFolder, String... filenames) {
+ Set<String> expectedContentsOfFolder = Set.of(filenames);
+ Set<String> actualContentsOfFolder;
+ try (Stream<UnixPath> paths = new UnixPath(pathToFolder).listContentsOfDirectory()) {
+ actualContentsOfFolder = paths.map(unixPath -> unixPath.toPath().getFileName().toString())
+ .collect(Collectors.toSet());
+ }
+ assertEquals(expectedContentsOfFolder, actualContentsOfFolder);
+ }
+
+ private Path createFileAged(Path path, Duration age) {
+ return uncheck(() -> Files.setLastModifiedTime(
+ Files.createFile(path),
+ FileTime.from(timer.currentTime().minus(age))));
+ }
+
+ private static byte[] bytesOf(String str) {
+ return str.getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static SecretSharedKey makeFixedSecretSharedKey() {
+ byte[] keyBytes = bytesOf("very secret yes!"); // 128 bits
+ var secretKey = new SecretKeySpec(keyBytes, "AES");
+ var keyId = KeyId.ofString("the shiniest key");
+ // We don't parse any of these fields in the test, so just use dummy contents.
+ byte[] enc = bytesOf("hello world");
+ byte[] ciphertext = bytesOf("imaginary ciphertext");
+ return new SecretSharedKey(secretKey, new SealedSharedKey(SealedSharedKey.CURRENT_TOKEN_VERSION, keyId, enc, ciphertext));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java
new file mode 100644
index 00000000000..0e20d3965a0
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java
@@ -0,0 +1,103 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class CoredumpCleanupRuleTest {
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+
+ @Test
+ void for_container_test() throws IOException {
+ Path path = fileSystem.getPath("/test/path");
+ DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path);
+
+ assertPriorities(rule, Map.of());
+
+ createFile(path.resolve("core1"), Instant.ofEpochSecond(232));
+ assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM));
+
+ createFile(path.resolve("core2"), Instant.ofEpochSecond(123));
+ assertPriorities(rule, Map.of(
+ "/test/path/core2", Priority.MEDIUM,
+ "/test/path/core1", Priority.HIGHEST));
+
+ createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456));
+ createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256));
+ var expected = Map.of(
+ "/test/path/core2", Priority.HIGHEST,
+ "/test/path/core1", Priority.HIGHEST,
+ "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST,
+ "/test/path/vespa-distributor.core.764", Priority.MEDIUM);
+ assertPriorities(rule, expected);
+
+ // processing core has no effect on this
+ Files.createDirectories(path.resolve("processing/abcd-1234"));
+ createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67));
+ assertPriorities(rule, expected);
+ }
+
+ @Test
+ void for_host_test() throws IOException {
+ Path path = fileSystem.getPath("/test/path");
+ DiskCleanupRule rule = CoredumpCleanupRule.forHost(path);
+
+ assertPriorities(rule, Map.of());
+
+ createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z"));
+ createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z"));
+ assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM));
+
+ createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z"));
+ createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z"));
+ createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z"));
+ createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z"));
+
+ // Also create a core for a second container: h123b
+ createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z"));
+ createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z"));
+
+ assertPriorities(rule, Map.of(
+ "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH,
+ "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH,
+
+ // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa-
+ "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM,
+ "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM,
+ "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM,
+ "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM,
+ "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM
+ ));
+ }
+
+ private static void createFile(Path path, Instant instant) throws IOException {
+ Files.createDirectories(path.getParent());
+ Files.createFile(path);
+ Files.setLastModifiedTime(path, FileTime.from(instant));
+ }
+
+ private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) {
+ Map<String, Priority> actual = rule.prioritize().stream()
+ .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority));
+
+ assertEquals(new TreeMap<>(expected), new TreeMap<>(actual));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java
new file mode 100644
index 00000000000..390501a4530
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java
@@ -0,0 +1,129 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author freva
+ */
+public class DiskCleanupTest {
+
+ private final TestTaskContext context = new TestTaskContext();
+ private final DiskCleanupTester tester = new DiskCleanupTester();
+ private final DiskCleanup diskCleanup = new DiskCleanup();
+
+ @Test
+ void nothing_deleted() throws IOException {
+ assertFalse(diskCleanup.cleanup(context, List.of(), 0));
+ assertFalse(diskCleanup.cleanup(context, List.of(), 10));
+
+ DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock();
+ DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock();
+ assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0));
+ assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+
+ tester.createFile("/path/that-should-not-be-deleted", 5);
+ assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+ tester.assertAllFilesExistExcept();
+
+ // Create a file and let rule return it, but before cleanup is run, the file is deleted
+ rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST);
+ Files.delete(tester.path("/path/file-does-not-exist"));
+ assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+ }
+
+ @Test
+ void delete_test() throws IOException {
+ tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1);
+ tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1);
+ tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1);
+ tester.createFile("/opt/vespa/var/index/something-important", 1);
+
+ DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock()
+ .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM)
+ .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH)
+ .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST)
+ .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST);
+ DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock()
+ .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW)
+ .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH)
+ .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM)
+ .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH);
+
+ // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since
+ // it is >= 10 bytes, no more files are deleted
+ assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+ tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log");
+
+ // Called with the same arguments, but vespa-3.log is still missing...
+ assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10));
+ tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log");
+
+ assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500));
+ tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before
+ // 300 + 190 + 8 + 54
+ "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3");
+ }
+
+ private static class DiskCleanupRuleMock implements DiskCleanupRule {
+ private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>();
+
+ private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException {
+ PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes();
+ pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority));
+ return this;
+ }
+
+ @Override
+ public Collection<PrioritizedFileAttributes> prioritize() {
+ return Collections.unmodifiableList(pfa);
+ }
+ }
+
+ private static class DiskCleanupTester {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Set<String> files = new HashSet<>();
+
+ private Path path(String path) {
+ return fileSystem.getPath(path);
+ }
+
+ private Path createFile(String pathStr, int size) throws IOException {
+ Path path = path(pathStr);
+ Files.createDirectories(path.getParent());
+ Files.write(path, new byte[size]);
+ files.add(path.toString());
+ return path;
+ }
+
+ private void assertAllFilesExistExcept(String... deletedPaths) {
+ Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet());
+ Set<String> expected = new HashSet<>(files);
+ expected.removeAll(Set.of(deletedPaths));
+ assertEquals(expected, actual);
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java
new file mode 100644
index 00000000000..c85ddf41906
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.disk;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author freva
+ */
+public class LinearCleanupRuleTest {
+
+ @Test
+ void basic() {
+ assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST);
+
+ assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW);
+ assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM);
+
+ assertRule(Map.of(
+ -5.0, Priority.LOW,
+ 0.0, Priority.LOW,
+ 0.2, Priority.LOW,
+ 0.35, Priority.MEDIUM,
+ 0.65, Priority.MEDIUM,
+ 0.8, Priority.HIGH,
+ 1.0, Priority.HIGH,
+ 5.0, Priority.HIGH),
+ Priority.LOW, Priority.HIGH);
+ }
+
+ @Test
+ void fail_if_high_priority_lower_than_low() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST);
+ });
+ }
+
+ private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) {
+ Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream()
+ .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score));
+ LinearCleanupRule rule = new LinearCleanupRule(
+ () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high);
+
+ Map<Double, Priority> actualPriorities = rule.prioritize().stream()
+ .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority));
+ assertEquals(expectedPriorities, actualPriorities);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java
new file mode 100644
index 00000000000..607efa9771a
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java
@@ -0,0 +1,31 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.yolean.concurrent.Sleeper;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author bjorncs
+ */
+class ArtifactProducersTest {
+
+ @Test
+ void generates_exception_on_unknown_artifact() {
+ ArtifactProducers instance = ArtifactProducers.createDefault(Sleeper.NOOP);
+ IllegalArgumentException exception = assertThrows(
+ IllegalArgumentException.class, () -> instance.resolve(List.of("unknown-artifact")));
+ String expectedMsg =
+ "Invalid artifact type 'unknown-artifact'. Valid types are " +
+ "['config-dump', 'jvm-heap-dump', 'jvm-jfr', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'perf-report', 'pmap', " +
+ "'vespa-log', 'zookeeper-snapshot'] " +
+ "and valid aliases are " +
+ "['jvm-dump': ['jvm-heap-dump', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'vespa-log']]";
+ assertEquals(expectedMsg, exception.getMessage());
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java
new file mode 100644
index 00000000000..db19d6b0074
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java
@@ -0,0 +1,319 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient;
+import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import com.yahoo.yolean.concurrent.Sleeper;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.DumpOptions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bjorncs
+ */
+class VespaServiceDumperImplTest {
+
+ private static final String HOSTNAME = "host-1.domain.tld";
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Path tmpDirectory = fileSystem.getPath("/data/vespa/storage/host-1/opt/vespa/var/tmp");
+
+ @BeforeEach
+ void create_tmp_directory() throws IOException {
+ // Create temporary directory in container
+ Files.createDirectories(tmpDirectory);
+ }
+
+ @Test
+ void creates_valid_dump_id_from_dump_request() {
+ long nowMillis = Instant.now().toEpochMilli();
+ ServiceDumpReport request = new ServiceDumpReport(
+ nowMillis, null, null, null, null, "default/container.3", null, null, List.of("perf-report"), null);
+ String dumpId = VespaServiceDumperImpl.createDumpId(request);
+ assertEquals("default-container-3-" + nowMillis, dumpId);
+ }
+
+ @Test
+ void invokes_perf_commands_when_generating_perf_report() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class)))
+ .thenReturn(new CommandResult(null, 0, "12345"))
+ .thenReturn(new CommandResult(null, 0, ""))
+ .thenReturn(new CommandResult(null, 0, ""));
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report"), new ServiceDumpReport.DumpOptions(true, 45.0, null));
+
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+
+ verify(operations).executeCommandInContainer(
+ context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1");
+ verify(operations).executeCommandInContainer(
+ context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin",
+ "--pid=12345", "sleep", "45");
+ verify(operations).executeCommandInContainer(
+ context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin" +
+ " > /opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-report.txt");
+
+ String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," +
+ "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," +
+ "\"configId\":\"default/container.1\",\"artifacts\":[\"perf-report\"]," +
+ "\"dumpOptions\":{\"callGraphRecording\":true,\"duration\":45.0}}";
+ assertReportEquals(nodeRepository, expectedJson);
+
+ List<URI> expectedUris = List.of(
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"),
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt"));
+ assertSyncedFiles(context, syncClient, expectedUris);
+ }
+
+ @Test
+ void invokes_jcmd_commands_when_creating_jfr_recording() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class)))
+ .thenReturn(new CommandResult(null, 0, "12345"))
+ .thenReturn(new CommandResult(null, 0, "ok"))
+ .thenReturn(new CommandResult(null, 0, "name=host-admin success"));
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("jvm-jfr"));
+
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+
+ verify(operations).executeCommandInContainer(
+ context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1");
+ verify(operations).executeCommandInContainer(
+ context, context.users().vespa(), "jcmd", "12345", "JFR.start", "name=host-admin", "path-to-gc-roots=true", "settings=profile",
+ "filename=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/recording.jfr", "duration=30s");
+ verify(operations).executeCommandInContainer(context, context.users().vespa(), "jcmd", "12345", "JFR.check", "name=host-admin");
+
+ String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000," +
+ "\"completedAt\":1600001000000," +
+ "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," +
+ "\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-jfr\"],\"dumpOptions\":{}}";
+ assertReportEquals(nodeRepository, expectedJson);
+
+ List<URI> expectedUris = List.of(
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst"));
+ assertSyncedFiles(context, syncClient, expectedUris);
+ }
+
+ @Test
+ void invokes_zookeeper_backup_command_when_generating_snapshot() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class)))
+ .thenReturn(new CommandResult(null, 0, "12345"));
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("zookeeper-snapshot"));
+
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+
+ verify(operations).executeCommandInContainer(
+ context,
+ context.users().vespa(),
+ "bash",
+ "-c",
+ "/opt/vespa/bin/vespa-backup-zk-data.sh -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/zookeeper-snapshot.tgz -k -f");
+
+ String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," +
+ "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," +
+ "\"configId\":\"default/container.1\",\"artifacts\":[\"zookeeper-snapshot\"],\"dumpOptions\":{}}";
+ assertReportEquals(nodeRepository, expectedJson);
+
+ List<URI> expectedUris = List.of(
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/zookeeper-snapshot.tgz"));
+ assertSyncedFiles(context, syncClient, expectedUris);
+ }
+
+ @Test
+ void invokes_config_proxy_command_whn_invoking_config_dump() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class)))
+ .thenReturn(new CommandResult(null, 0, "12345"));
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("config-dump"));
+
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+
+ verify(operations).executeCommandInContainer(
+ context,
+ context.users().vespa(),
+ "bash",
+ "-c",
+ "mkdir -p /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" +
+ " /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" +
+ " tar cvf /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" +
+ " zstd /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config-dump.tar.zst");
+
+ String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," +
+ "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," +
+ "\"configId\":\"default/container.1\",\"artifacts\":[\"config-dump\"],\"dumpOptions\":{}}";
+ assertReportEquals(nodeRepository, expectedJson);
+
+ List<URI> expectedUris = List.of(
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/config-dump.tar.zst"));
+ assertSyncedFiles(context, syncClient, expectedUris);
+ }
+
+ @Test
+ void handles_multiple_artifact_types() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ when(operations.executeCommandInContainer(
+ any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class)))
+ // For perf report:
+ .thenReturn(new CommandResult(null, 0, "12345"))
+ .thenReturn(new CommandResult(null, 0, ""))
+ .thenReturn(new CommandResult(null, 0, ""))
+ // For jfr recording:
+ .thenReturn(new CommandResult(null, 0, "12345"))
+ .thenReturn(new CommandResult(null, 0, "ok"))
+ .thenReturn(new CommandResult(null, 0, "name=host-admin success"));
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report", "jvm-jfr"),
+ new ServiceDumpReport.DumpOptions(true, 20.0, null));
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+
+ List<URI> expectedUris = List.of(
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"),
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt"),
+ URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst"));
+ assertSyncedFiles(context, syncClient, expectedUris);
+ }
+
+ @Test
+ void fails_gracefully_on_invalid_request_json() {
+ // Setup mocks
+ ContainerOperations operations = mock(ContainerOperations.class);
+ SyncClient syncClient = createSyncClientMock();
+ NodeRepoMock nodeRepository = new NodeRepoMock();
+ TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+ JsonNodeFactory fac = new ObjectMapper().getNodeFactory();
+ ObjectNode invalidRequest = new ObjectNode(fac)
+ .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType"));
+ NodeSpec spec = NodeSpec.Builder
+ .testSpec(HOSTNAME, NodeState.active)
+ .report(ServiceDumpReport.REPORT_ID, invalidRequest)
+ .build();
+ nodeRepository.updateNodeSpec(spec);
+ VespaServiceDumper reporter = new VespaServiceDumperImpl(
+ ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+ NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec)
+ .fileSystem(fileSystem)
+ .build();
+ reporter.processServiceDumpRequest(context);
+ String expectedJson = "{\"createdMillis\":1600001000000,\"startedAt\":1600001000000,\"failedAt\":1600001000000," +
+ "\"configId\":\"unknown\",\"error\":\"Invalid JSON in service dump request\",\"artifacts\":[]}";
+ assertReportEquals(nodeRepository, expectedJson);
+ }
+
+ private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts) {
+ return createNodeSpecWithDumpRequest(repository, artifacts, new DumpOptions(null, null, null));
+ }
+
+ private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, DumpOptions options) {
+ ServiceDumpReport request = ServiceDumpReport.createRequestReport(
+ Instant.ofEpochMilli(1600000000000L), null, "default/container.1", artifacts, options);
+ NodeSpec spec = NodeSpec.Builder
+ .testSpec(HOSTNAME, NodeState.active)
+ .report(ServiceDumpReport.REPORT_ID, request.toJsonNode())
+ .archiveUri(URI.create("s3://uri-1/tenant1/"))
+ .build();
+ repository.updateNodeSpec(spec);
+ return spec;
+ }
+
+ private static void assertReportEquals(NodeRepoMock nodeRepository, String expectedJson) {
+ ServiceDumpReport report = nodeRepository.getNode(HOSTNAME).reports()
+ .getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class).get();
+ String actualJson = report.toJson();
+ assertEquals(expectedJson, actualJson);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void assertSyncedFiles(NodeAgentContextImpl context, SyncClient client, List<URI> expectedDestinations) {
+ ArgumentCaptor<List<SyncFileInfo>> filesCaptor = ArgumentCaptor.forClass(List.class);
+ verify(client).sync(eq(context), filesCaptor.capture(), eq(Integer.MAX_VALUE));
+ List<SyncFileInfo> actualFiles = filesCaptor.getValue();
+ List<URI> actualFilenames = actualFiles.stream()
+ .map(SyncFileInfo::destination)
+ .sorted()
+ .toList();
+ assertEquals(expectedDestinations, actualFilenames);
+ }
+
+ private SyncClient createSyncClientMock() {
+ SyncClient client = mock(SyncClient.class);
+ when(client.sync(any(TaskContext.class), anyList(), anyInt()))
+ .thenReturn(true);
+ return client;
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
new file mode 100644
index 00000000000..8e56741274e
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -0,0 +1,134 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.ZSTD;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class SyncFileInfoTest {
+
+ private static final FileSystem fileSystem = TestFileSystem.create();
+
+ private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/");
+ private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210211");
+ private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210212.zst");
+ private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/access-json.log.20210213.zst");
+ private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.20210214.zst");
+ private static final Path accessLogPath5 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.container.20210214.zst");
+ private static final Path accessLogPath6 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.metrics-proxy.20210214.zst");
+ private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210210");
+ private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210212.zst");
+ private static final Path connectionLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.metrics-proxy.20210210");
+ private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log");
+ private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12");
+ private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log");
+ private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log");
+ private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out");
+ private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143");
+ private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555");
+ private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log");
+ private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234");
+
+ @Test
+ void access_logs() {
+ assertForLogFile(accessLogPath1, null, null, true);
+ assertForLogFile(accessLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210211.zst", ZSTD, false);
+
+ assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, true);
+ assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, false);
+
+ assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, true);
+ assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, false);
+
+ assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, true);
+ assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, false);
+
+ assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, true);
+ assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, false);
+
+ assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, true, ApplicationId.defaultId()));
+ assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, false, ApplicationId.defaultId()));
+ }
+
+ @Test
+ void connection_logs() {
+ assertForLogFile(connectionLogPath1, null, null, true);
+ assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210210.zst", ZSTD, false);
+
+ assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, true);
+ assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, false);
+
+ assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, true, ApplicationId.defaultId()));
+ assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, false, ApplicationId.defaultId()));
+ }
+
+ @Test
+ void vespa_logs() {
+ new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+ assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true);
+ assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false);
+
+ assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true);
+ assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false);
+ }
+
+ @Test
+ void zookeeper_logs() {
+ new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z"));
+ assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true);
+ assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false);
+
+ new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+ assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true);
+ assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false);
+ }
+
+ @Test
+ void nginx_error_logs() {
+ new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+ assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true);
+ assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false);
+
+ new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+ assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true);
+ assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false);
+
+ // Does not sync access logs
+ new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+ Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId());
+ assertEquals(Optional.empty(), sfi);
+ }
+
+ @Test
+ void start_services() {
+ assertForLogFile(startServicesPath1, null, null, true);
+ assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true);
+ }
+
+ private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) {
+ assertForLogFile(srcPath, destination, compression, null, rotatedOnly);
+ }
+
+ private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) {
+ Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId());
+ assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null));
+ assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null));
+ assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java
new file mode 100644
index 00000000000..616100363e9
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java
@@ -0,0 +1,58 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.compress.ZstdCompressor;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class ZstdCompressingInputStreamTest {
+
+ @Test
+ void compression_test() {
+ Random rnd = new Random();
+ byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))];
+ rnd.nextBytes(data);
+ assertCompression(data, 1 << 14);
+ }
+
+ @Test
+ void compress_empty_file_test() {
+ byte[] compressedData = compress(new byte[0], 1 << 10);
+ assertEquals(13, compressedData.length, "zstd compressing an empty file results in a 13 bytes file");
+ }
+
+ private static void assertCompression(byte[] data, int bufferSize) {
+ byte[] compressedData = compress(data, bufferSize);
+ byte[] decompressedData = new byte[data.length];
+ var compressor = new ZstdCompressor();
+ compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length);
+
+ assertArrayEquals(data, decompressedData);
+ }
+
+ private static byte[] compress(byte[] data, int bufferSize) {
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) {
+ byte[] buffer = new byte[bufferSize];
+ for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; )
+ baos.write(buffer, 0, nRead);
+ baos.flush();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return baos.toByteArray();
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
new file mode 100644
index 00000000000..355c997a3e0
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -0,0 +1,166 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bakksjo
+ */
+public class NodeAdminImplTest {
+
+ private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class);
+ private final TestTimer timer = new TestTimer();
+ private final ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class);
+ private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory,
+ new Metrics(), timer, Duration.ZERO, Duration.ZERO, procMeminfoReader);
+
+ @Test
+ void nodeAgentsAreProperlyLifeCycleManaged() {
+ final NodeAgentContext context1 = createNodeAgentContext("host1.test.yahoo.com");
+ final NodeAgentContext context2 = createNodeAgentContext("host2.test.yahoo.com");
+ final NodeAgentWithScheduler nodeAgent1 = mockNodeAgentWithSchedulerFactory(context1);
+ final NodeAgentWithScheduler nodeAgent2 = mockNodeAgentWithSchedulerFactory(context2);
+
+ final InOrder inOrder = inOrder(nodeAgentWithSchedulerFactory, nodeAgent1, nodeAgent2);
+ nodeAdmin.refreshContainersToRun(Set.of());
+ verifyNoMoreInteractions(nodeAgentWithSchedulerFactory);
+
+ nodeAdmin.refreshContainersToRun(Set.of(context1));
+ inOrder.verify(nodeAgent1).start();
+ inOrder.verify(nodeAgent2, never()).start();
+ inOrder.verify(nodeAgent1, never()).stopForRemoval();
+
+ nodeAdmin.refreshContainersToRun(Set.of(context1));
+ inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any());
+ inOrder.verify(nodeAgent1, never()).start();
+ inOrder.verify(nodeAgent1, never()).stopForRemoval();
+
+ nodeAdmin.refreshContainersToRun(Set.of());
+ inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any());
+ verify(nodeAgent1).stopForRemoval();
+
+ nodeAdmin.refreshContainersToRun(Set.of(context2));
+ inOrder.verify(nodeAgent2).start();
+ inOrder.verify(nodeAgent2, never()).stopForRemoval();
+ inOrder.verify(nodeAgent1, never()).stopForRemoval();
+
+ nodeAdmin.refreshContainersToRun(Set.of());
+ inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any());
+ inOrder.verify(nodeAgent2, never()).start();
+ inOrder.verify(nodeAgent2).stopForRemoval();
+ inOrder.verify(nodeAgent1, never()).start();
+ inOrder.verify(nodeAgent1, never()).stopForRemoval();
+ }
+
+ @Test
+ void testSetFrozen() {
+ Set<NodeAgentContext> contexts = new HashSet<>();
+ List<NodeAgentWithScheduler> nodeAgents = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ NodeAgentContext context = createNodeAgentContext("host" + i + ".test.yahoo.com");
+ NodeAgentWithScheduler nodeAgent = mockNodeAgentWithSchedulerFactory(context);
+
+ contexts.add(context);
+ nodeAgents.add(nodeAgent);
+ }
+
+ nodeAdmin.refreshContainersToRun(contexts);
+
+ assertTrue(nodeAdmin.isFrozen()); // Initially everything is frozen to force convergence
+ mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true);
+ assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything
+
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false);
+ assertFalse(nodeAdmin.setFrozen(true)); // NodeAdmin freezes only when all the NodeAgents are frozen
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, false, true, true);
+ assertFalse(nodeAdmin.setFrozen(true));
+ assertFalse(nodeAdmin.isFrozen());
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true);
+ assertTrue(nodeAdmin.setFrozen(true));
+ assertTrue(nodeAdmin.isFrozen());
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true);
+ assertTrue(nodeAdmin.setFrozen(true));
+ assertTrue(nodeAdmin.isFrozen());
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false);
+ assertFalse(nodeAdmin.setFrozen(false));
+ assertFalse(nodeAdmin.isFrozen()); // NodeAdmin unfreezes instantly
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, false, false, true);
+ assertFalse(nodeAdmin.setFrozen(false));
+ assertFalse(nodeAdmin.isFrozen());
+
+ mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true);
+ assertTrue(nodeAdmin.setFrozen(false));
+ assertFalse(nodeAdmin.isFrozen());
+ }
+
+ @Test
+ void testSubsystemFreezeDuration() {
+ // Initially everything is frozen to force convergence
+ assertTrue(nodeAdmin.isFrozen());
+ assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
+ timer.advance(Duration.ofSeconds(1));
+ assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration());
+
+ // Unfreezing floors freeze duration
+ assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything
+ assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
+ timer.advance(Duration.ofSeconds(1));
+ assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
+
+ // Advancing time now will make freeze duration proceed according to clock
+ assertTrue(nodeAdmin.setFrozen(true));
+ assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
+ timer.advance(Duration.ofSeconds(1));
+ assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration());
+ }
+
+ private void mockNodeAgentSetFrozenResponse(List<NodeAgentWithScheduler> nodeAgents, boolean... responses) {
+ for (int i = 0; i < nodeAgents.size(); i++) {
+ NodeAgentWithScheduler nodeAgent = nodeAgents.get(i);
+ when(nodeAgent.setFrozen(anyBoolean(), any())).thenReturn(responses[i]);
+ }
+ }
+
+ private NodeAgentContext createNodeAgentContext(String hostname) {
+ return NodeAgentContextImpl.builder(hostname).fileSystem(TestFileSystem.create()).build();
+ }
+
+ private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) {
+ NodeAgentWithScheduler nodeAgentWithScheduler = mock(NodeAgentWithScheduler.class);
+ when(nodeAgentWithSchedulerFactory.create(eq(context))).thenReturn(nodeAgentWithScheduler);
+ return nodeAgentWithScheduler;
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
new file mode 100644
index 00000000000..420146b52f0
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
@@ -0,0 +1,277 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeadmin;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Basic test of NodeAdminStateUpdater
+ *
+ * @author freva
+ */
+public class NodeAdminStateUpdaterTest {
+ private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class);
+ private final NodeRepoMock nodeRepository = spy(new NodeRepoMock());
+ private final Orchestrator orchestrator = mock(Orchestrator.class);
+ private final NodeAdmin nodeAdmin = mock(NodeAdmin.class);
+ private final HostName hostHostname = HostName.of("basehost1.test.yahoo.com");
+
+ private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater(
+ nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname));
+
+
+ @Test
+ void state_convergence() {
+ mockNodeRepo(NodeState.active, 4);
+ List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream()
+ .map(NodeSpec::hostname)
+ .toList();
+ List<String> suspendHostnames = new ArrayList<>(activeHostnames);
+ suspendHostnames.add(hostHostname.value());
+ when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1));
+
+ {
+ // Initially everything is frozen to force convergence
+ assertConvergeError(RESUMED, "NodeAdmin is not yet unfrozen");
+ when(nodeAdmin.setFrozen(eq(false))).thenReturn(true);
+ updater.converge(RESUMED);
+ verify(orchestrator, times(1)).resume(hostHostname.value());
+
+ // We are already resumed, so this should return without resuming again
+ updater.converge(RESUMED);
+ verify(orchestrator, times(1)).resume(hostHostname.value());
+ verify(nodeAdmin, times(2)).setFrozen(eq(false));
+
+ // Host is externally suspended in orchestrator, should be resumed by node-admin
+ setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN);
+ updater.converge(RESUMED);
+ verify(orchestrator, times(2)).resume(hostHostname.value());
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+ setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS);
+
+ // Lets try to suspend node admin only
+ when(nodeAdmin.setFrozen(eq(true))).thenReturn(false);
+ assertConvergeError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen");
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+ }
+
+ {
+ // First orchestration failure happens within the freeze convergence timeout,
+ // and so should not call setFrozen(false)
+ final String exceptionMessage = "Cannot allow to suspend because some reason";
+ when(nodeAdmin.setFrozen(eq(true))).thenReturn(true);
+ doThrow(new RuntimeException(exceptionMessage)).doNothing()
+ .when(orchestrator).suspend(eq(hostHostname.value()));
+ assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMessage);
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+
+ updater.converge(SUSPENDED_NODE_ADMIN);
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+ verify(orchestrator, times(2)).suspend(hostHostname.value());
+ setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN);
+
+ // Already suspended, no changes
+ updater.converge(SUSPENDED_NODE_ADMIN);
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+ verify(orchestrator, times(2)).suspend(hostHostname.value());
+
+ // Host is externally resumed
+ setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS);
+ updater.converge(SUSPENDED_NODE_ADMIN);
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+ verify(orchestrator, times(3)).suspend(hostHostname.value());
+ setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ {
+ // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services
+ final String exceptionMessage = "Failed to stop services";
+ verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames));
+ doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices();
+ assertConvergeError(SUSPENDED, exceptionMessage);
+ verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames));
+ // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick
+ verify(nodeAdmin, times(3)).setFrozen(eq(false));
+
+ // Finally we are successful in transitioning to frozen
+ updater.converge(SUSPENDED);
+ }
+ }
+
+ @Test
+ void half_transition_revert() {
+ final String exceptionMsg = "Cannot allow to suspend because some reason";
+ mockNodeRepo(NodeState.active, 3);
+
+ // Initially everything is frozen to force convergence
+ when(nodeAdmin.setFrozen(eq(false))).thenReturn(true);
+ updater.converge(RESUMED);
+ verify(nodeAdmin, times(1)).setFrozen(eq(false));
+ verify(nodeAdmin, times(1)).refreshContainersToRun(any());
+
+ // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension
+ when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1));
+ when(nodeAdmin.setFrozen(eq(true))).thenReturn(true);
+ doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value()));
+
+ assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg);
+ verify(nodeAdmin, times(1)).setFrozen(eq(true));
+ verify(orchestrator, times(1)).suspend(eq(hostHostname.value()));
+ assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg);
+ verify(nodeAdmin, times(2)).setFrozen(eq(true));
+ verify(orchestrator, times(2)).suspend(eq(hostHostname.value()));
+ assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg);
+ verify(nodeAdmin, times(3)).setFrozen(eq(true));
+ verify(orchestrator, times(3)).suspend(eq(hostHostname.value()));
+
+ // No new unfreezes nor refresh while trying to freeze
+ verify(nodeAdmin, times(1)).setFrozen(eq(false));
+ verify(nodeAdmin, times(1)).refreshContainersToRun(any());
+
+ // Only resume and fetch containers when subsystem freeze duration expires
+ when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1));
+ assertConvergeError(SUSPENDED_NODE_ADMIN, "Timed out trying to freeze all nodes: will force an unfrozen tick");
+ verify(nodeAdmin, times(2)).setFrozen(eq(false));
+ verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); // no new suspend calls
+ verify(nodeAdmin, times(2)).refreshContainersToRun(any());
+
+ // We change our mind, want to remain resumed
+ updater.converge(RESUMED);
+ verify(nodeAdmin, times(3)).setFrozen(eq(false)); // Make sure that we unfreeze!
+ }
+
+ @Test
+ void do_not_orchestrate_host_when_not_active() {
+ when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1));
+ when(nodeAdmin.setFrozen(anyBoolean())).thenReturn(true);
+ mockNodeRepo(NodeState.ready, 3);
+
+ // Resume and suspend only require that node-agents are frozen and permission from
+ // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze.
+ updater.converge(RESUMED);
+ verify(orchestrator, never()).resume(eq(hostHostname.value()));
+
+ updater.converge(SUSPENDED_NODE_ADMIN);
+ verify(orchestrator, never()).suspend(eq(hostHostname.value()));
+
+ // When doing batch suspend, only suspend the containers if the host is not active
+ List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream()
+ .map(NodeSpec::hostname)
+ .toList();
+ updater.converge(SUSPENDED);
+ verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames));
+ }
+
+ @Test
+ void node_spec_and_acl_aligned() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22).build();
+ mockNodeRepo(NodeState.active, 3);
+ mockAcl(acl, 1, 2, 3);
+
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl));
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl));
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(acl));
+ verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value()));
+ verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value()));
+ }
+
+ @Test
+ void node_spec_and_acl_mismatch_missing_one_acl() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22).build();
+ mockNodeRepo(NodeState.active, 3);
+ mockAcl(acl, 1, 2); // Acl for 3 is missing
+
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ mockNodeRepo(NodeState.active, 2); // Next tick, the spec for 3 is no longer returned
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl));
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl));
+ verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(Acl.EMPTY));
+ verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value()));
+ verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value()));
+ }
+
+ @Test
+ void node_spec_and_acl_mismatch_additional_acl() {
+ Acl acl = new Acl.Builder().withTrustedPorts(22).build();
+ mockNodeRepo(NodeState.active, 2);
+ mockAcl(acl, 1, 2, 3); // Acl for 3 is extra
+
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+ updater.adjustNodeAgentsToRunFromNodeRepository();
+
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl));
+ verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl));
+ verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value()));
+ verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value()));
+ }
+
+ private void assertConvergeError(NodeAdminStateUpdater.State targetState, String reason) {
+ try {
+ updater.converge(targetState);
+ fail("Expected converging to " + targetState + " to fail with \"" + reason + "\", but it succeeded without error");
+ } catch (RuntimeException e) {
+ assertEquals(reason, e.getMessage());
+ }
+ }
+
+ private void mockNodeRepo(NodeState hostState, int numberOfNodes) {
+ nodeRepository.resetNodeSpecs();
+
+ IntStream.rangeClosed(1, numberOfNodes)
+ .mapToObj(i -> NodeSpec.Builder.testSpec("host" + i + ".yahoo.com").parentHostname(hostHostname.value()).build())
+ .forEach(nodeRepository::updateNodeSpec);
+
+ nodeRepository.updateNodeSpec(NodeSpec.Builder.testSpec(hostHostname.value(), hostState).type(NodeType.host).build());
+ }
+
+ private void mockAcl(Acl acl, int... nodeIds) {
+ nodeRepository.setAcl(Arrays.stream(nodeIds)
+ .mapToObj(i -> "host" + i + ".yahoo.com")
+ .collect(Collectors.toMap(Function.identity(), h -> acl)));
+ }
+
+ private void setHostOrchestratorStatus(HostName hostname, OrchestratorStatus orchestratorStatus) {
+ nodeRepository.updateNodeSpec(hostname.value(), node -> node.orchestratorStatus(orchestratorStatus));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
new file mode 100644
index 00000000000..589eceebb74
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
@@ -0,0 +1,103 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author freva
+ */
+public class NodeAgentContextImplTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final NodeAgentContext context = NodeAgentContextImpl.builder("container-1.domain.tld")
+ .fileSystem(fileSystem).build();
+
+ @Test
+ void path_on_host_from_path_in_node_test() {
+ assertEquals(
+ "/data/vespa/storage/container-1",
+ context.paths().of("/").pathOnHost().toString());
+
+ assertEquals(
+ "/data/vespa/storage/container-1/dev/null",
+ context.paths().of("/dev/null").pathOnHost().toString());
+ }
+
+ @Test
+ void path_in_container_must_be_absolute() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ context.paths().of("some/relative/path");
+ });
+ }
+
+ @Test
+ void path_in_node_from_path_on_host_test() {
+ assertEquals(
+ "/dev/null",
+ context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).pathInContainer());
+ }
+
+ @Test
+ void path_on_host_must_be_absolute() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ context.paths().fromPathOnHost(Path.of("some/relative/path"));
+ });
+ }
+
+ @Test
+ void path_on_host_must_be_inside_container_storage_of_context() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null"));
+ });
+ }
+
+ @Test
+ void path_on_host_must_be_inside_container_storage() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ context.paths().fromPathOnHost(fileSystem.getPath("/home"));
+ });
+ }
+
+ @Test
+ void path_under_vespa_host_in_container_test() {
+ assertEquals(
+ "/opt/vespa",
+ context.paths().underVespaHome("").pathInContainer());
+
+ assertEquals(
+ "/opt/vespa/logs/vespa/vespa.log",
+ context.paths().underVespaHome("logs/vespa/vespa.log").pathInContainer());
+ }
+
+ @Test
+ void path_under_vespa_home_must_be_relative() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ context.paths().underVespaHome("/home");
+ });
+ }
+
+ @Test
+ void disabledTasksTest() {
+ NodeAgentContext context1 = createContextWithDisabledTasks();
+ assertFalse(context1.isDisabled(NodeAgentTask.DiskCleanup));
+ assertFalse(context1.isDisabled(NodeAgentTask.CoreDumps));
+
+ NodeAgentContext context2 = createContextWithDisabledTasks("root>UpgradeTask", "DiskCleanup", "node>CoreDumps");
+ assertFalse(context2.isDisabled(NodeAgentTask.DiskCleanup));
+ assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps));
+ }
+
+ private NodeAgentContext createContextWithDisabledTasks(String... tasks) {
+ InMemoryFlagSource flagSource = new InMemoryFlagSource();
+ flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class);
+ return NodeAgentContextImpl.builder("node123").fileSystem(fileSystem).flagSource(flagSource).build();
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java
new file mode 100644
index 00000000000..5e09c45d217
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java
@@ -0,0 +1,152 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.jdisc.core.SystemTimer;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+
+import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author freva
+ */
+public class NodeAgentContextManagerTest {
+
+ private static final int TIMEOUT = 10_000;
+
+ private final SystemTimer timer = new SystemTimer();
+ private final NodeAgentContext initialContext = generateContext();
+ private final NodeAgentContextManager manager = new NodeAgentContextManager(timer, initialContext);
+
+ @Test
+ @Timeout(TIMEOUT)
+ void context_is_ignored_unless_scheduled_while_waiting() {
+ NodeAgentContext context1 = generateContext();
+ manager.scheduleTickWith(context1, timer.currentTime());
+ assertSame(initialContext, manager.currentContext());
+
+ AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
+ manager.waitUntilWaitingForNextContext();
+ assertFalse(async.isCompleted());
+
+ NodeAgentContext context2 = generateContext();
+ manager.scheduleTickWith(context2, timer.currentTime());
+
+ assertSame(context2, async.awaitResult().response.get());
+ assertSame(context2, manager.currentContext());
+ }
+
+ @Test
+ @Timeout(TIMEOUT)
+ void returns_no_earlier_than_at_given_time() {
+ AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
+ manager.waitUntilWaitingForNextContext();
+
+ NodeAgentContext context1 = generateContext();
+ Instant returnAt = timer.currentTime().plusMillis(500);
+ manager.scheduleTickWith(context1, returnAt);
+
+ assertSame(context1, async.awaitResult().response.get());
+ assertSame(context1, manager.currentContext());
+ // Is accurate to a millisecond
+ assertFalse(timer.currentTime().plusMillis(1).isBefore(returnAt));
+ }
+
+ @Test
+ @Timeout(TIMEOUT)
+ void blocks_in_nextContext_until_one_is_scheduled() {
+ AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
+ manager.waitUntilWaitingForNextContext();
+ assertFalse(async.isCompleted());
+
+ NodeAgentContext context1 = generateContext();
+ manager.scheduleTickWith(context1, timer.currentTime());
+
+ async.awaitResult();
+ assertEquals(Optional.of(context1), async.response);
+ assertFalse(async.exception.isPresent());
+ }
+
+ @Test
+ @Timeout(TIMEOUT)
+ void blocks_in_nextContext_until_interrupt() {
+ AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
+ manager.waitUntilWaitingForNextContext();
+ assertFalse(async.isCompleted());
+
+ manager.interrupt();
+
+ async.awaitResult();
+ assertEquals(Optional.of(ContextSupplierInterruptedException.class), async.exception.map(Exception::getClass));
+ assertFalse(async.response.isPresent());
+ }
+
+ @Test
+ @Timeout(TIMEOUT)
+ void setFrozen_does_not_block_with_no_timeout() {
+ assertFalse(manager.setFrozen(false, Duration.ZERO));
+
+ // Generate new context and get it from the supplier, this completes the unfreeze
+ NodeAgentContext context1 = generateContext();
+ AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
+ manager.waitUntilWaitingForNextContext();
+ manager.scheduleTickWith(context1, timer.currentTime());
+ assertSame(context1, async.awaitResult().response.get());
+
+ assertTrue(manager.setFrozen(false, Duration.ZERO));
+ }
+
+ @Test
+ @Timeout(TIMEOUT)
+ void setFrozen_blocks_at_least_for_duration_of_timeout() {
+ long wantedDurationMillis = 100;
+ long start = timer.currentTimeMillis();
+ assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis)));
+ long actualDurationMillis = timer.currentTimeMillis() - start;
+
+ assertTrue(actualDurationMillis >= wantedDurationMillis);
+ }
+
+ private static NodeAgentContext generateContext() {
+ return NodeAgentContextImpl.builder("container-123.domain.tld").fileSystem(TestFileSystem.create()).build();
+ }
+
+ private static class AsyncExecutor<T> {
+ private final CountDownLatch latch = new CountDownLatch(1);
+ private volatile Optional<T> response = Optional.empty();
+ private volatile Optional<Exception> exception = Optional.empty();
+
+ private AsyncExecutor(Callable<T> supplier) {
+ new Thread(() -> {
+ try {
+ response = Optional.of(supplier.call());
+ } catch (Exception e) {
+ exception = Optional.of(e);
+ }
+ latch.countDown();
+ }).start();
+ }
+
+ private AsyncExecutor<T> awaitResult() {
+ try {
+ latch.await();
+ } catch (InterruptedException ignored) { }
+ return this;
+ }
+
+ private boolean isCompleted() {
+ return latch.getCount() == 0;
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
new file mode 100644
index 00000000000..709326cc3b8
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -0,0 +1,889 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException;
+import com.yahoo.vespa.hosted.node.admin.container.Container;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerId;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
+import com.yahoo.vespa.hosted.node.admin.container.ContainerResources;
+import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
+import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper;
+import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+
+import java.nio.file.FileSystem;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiFunction;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author Øyvind Bakksjø
+ */
+public class NodeAgentImplTest {
+ private static final NodeResources resources = new NodeResources(2, 16, 250, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local);
+ private static final Version vespaVersion = Version.fromString("1.2.3");
+ private static final ContainerId containerId = new ContainerId("af23");
+ private static final String hostName = "host1.test.yahoo.com";
+
+ private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class);
+ private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image");
+ private final ContainerOperations containerOperations = mock(ContainerOperations.class);
+ private final NodeRepository nodeRepository = mock(NodeRepository.class);
+ private final Orchestrator orchestrator = mock(Orchestrator.class);
+ private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
+ private final AclMaintainer aclMaintainer = mock(AclMaintainer.class);
+ private final HealthChecker healthChecker = mock(HealthChecker.class);
+ private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
+ private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+ private final TestTimer timer = new TestTimer(Instant.now());
+ private final FileSystem fileSystem = TestFileSystem.create();
+
+ @BeforeEach
+ public void setUp() {
+ when(containerOperations.suspendNode(any())).thenReturn("");
+ when(containerOperations.resumeNode(any())).thenReturn("");
+ when(containerOperations.restartVespa(any())).thenReturn("");
+ when(containerOperations.startServices(any())).thenReturn("");
+ when(containerOperations.stopServices(any())).thenReturn("");
+ }
+
+ @Test
+ void upToDateContainerIsUntouched() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .orchestratorStatus(OrchestratorStatus.NO_REMARKS)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(orchestrator, never()).suspend(any(String.class));
+ verify(containerOperations, never()).pullImageAsyncIfNeeded(any(), any(), any());
+
+ final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository);
+ // TODO: Verify this isn't run unless 1st time
+ inOrder.verify(containerOperations, never()).startServices(eq(context));
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
+ inOrder.verify(orchestrator, never()).resume(hostName);
+ }
+
+ @Test
+ void verifyRemoveOldFilesIfDiskFull() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context));
+ }
+
+ @Test
+ void startsAfterStoppingServices() {
+ final InOrder inOrder = inOrder(containerOperations);
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+ inOrder.verify(containerOperations, never()).startServices(eq(context));
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
+
+ nodeAgent.stopForHostSuspension(context);
+ nodeAgent.doConverge(context);
+ inOrder.verify(containerOperations, never()).startServices(eq(context));
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services
+
+ // No new suspends/stops, so no need to resume/start
+ nodeAgent.doConverge(context);
+ inOrder.verify(containerOperations, never()).startServices(eq(context));
+ inOrder.verify(containerOperations, never()).resumeNode(eq(context));
+
+ nodeAgent.stopForHostSuspension(context);
+ nodeAgent.doConverge(context);
+ inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any());
+ inOrder.verify(containerOperations, times(1)).startContainer(eq(context));
+ inOrder.verify(containerOperations, times(0)).startServices(eq(context)); // done as part of startContainer
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
+ }
+
+ @Test
+ void absentContainerCausesStart() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+ when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false);
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(containerOperations, never()).startServices(any());
+ verify(orchestrator, never()).suspend(any(String.class));
+
+ final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker);
+ inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any());
+ inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any());
+ inOrder.verify(containerOperations, times(1)).startContainer(eq(context));
+ inOrder.verify(aclMaintainer, times(1)).converge(eq(context));
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
+ inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context));
+ inOrder.verify(nodeRepository).updateNodeAttributes(
+ hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0));
+ inOrder.verify(orchestrator, never()).resume(hostName);
+ }
+
+ @Test
+ void containerIsNotStoppedIfNewImageMustBePulled() {
+ final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/repo/new-image");
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+ when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true);
+
+ nodeAgent.doConverge(context);
+
+ verify(orchestrator, never()).suspend(any(String.class));
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+
+ final InOrder inOrder = inOrder(containerOperations);
+ inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(newDockerImage), any());
+ }
+
+ @Test
+ void containerIsUpdatedIfCpuChanged() {
+ NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .orchestratorStatus(OrchestratorStatus.NO_REMARKS);
+
+ NodeAgentContext firstContext = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true);
+
+ InOrder inOrder = inOrder(orchestrator, containerOperations);
+
+ nodeAgent.doConverge(firstContext);
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
+
+ NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build());
+ nodeAgent.doConverge(secondContext);
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
+
+ NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).fileSystem(fileSystem).cpuSpeedUp(1.25).build();
+ nodeAgent.doConverge(thirdContext);
+ ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16);
+ mockGetContainer(dockerImage, resourcesAfterThird, true);
+
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(resourcesAfterThird));
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(containerOperations, never()).startContainer(any());
+ inOrder.verify(orchestrator, never()).resume(any());
+
+ // No changes
+ nodeAgent.converge(thirdContext);
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations, never()).updateContainer(eq(thirdContext), eq(containerId), any());
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(orchestrator, never()).resume(any());
+
+ // Set the feature flag
+ flagSource.withDoubleFlag(PermanentFlags.CONTAINER_CPU_CAP.id(), 2.3);
+
+ nodeAgent.doConverge(thirdContext);
+ inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(ContainerResources.from(9.2, 4, 16)));
+ inOrder.verify(orchestrator, never()).resume(any());
+ }
+
+ @Test
+ void containerIsRecreatedIfMemoryChanged() {
+ NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(2).currentRestartGeneration(1);
+
+ NodeAgentContext firstContext = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true);
+
+ nodeAgent.doConverge(firstContext);
+ NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build());
+ nodeAgent.doConverge(secondContext);
+ ContainerResources resourcesAfterThird = ContainerResources.from(0, 2, 20);
+ mockGetContainer(dockerImage, resourcesAfterThird, true);
+
+ InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository);
+ inOrder.verify(orchestrator).resume(any(String.class));
+ inOrder.verify(containerOperations).removeContainer(eq(secondContext), any());
+ inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any());
+ inOrder.verify(containerOperations, never()).restartVespa(any());
+ inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0)));
+
+ nodeAgent.doConverge(secondContext);
+ inOrder.verify(orchestrator).resume(any(String.class));
+ inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any());
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ }
+
+ @Test
+ void noRestartIfOrchestratorSuspendFails() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(2).currentRestartGeneration(1)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ doThrow(new OrchestratorException("Denied")).when(orchestrator).suspend(eq(hostName));
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to throw an exception");
+ } catch (OrchestratorException ignored) {
+ }
+
+ verify(containerOperations, never()).createContainer(eq(context), any());
+ verify(containerOperations, never()).startContainer(eq(context));
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class));
+
+ // Verify aclMaintainer is called even if suspension fails
+ verify(aclMaintainer, times(1)).converge(eq(context));
+ }
+
+ @Test
+ void recreatesContainerIfRebootWanted() {
+ final long wantedRebootGeneration = 2;
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRebootGeneration(wantedRebootGeneration).currentRebootGeneration(1)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+ when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false);
+ doThrow(ConvergenceException.ofTransient("Connection refused")).doNothing()
+ .when(healthChecker).verifyHealth(eq(context));
+
+ try {
+ nodeAgent.doConverge(context);
+ } catch (ConvergenceException ignored) {
+ }
+
+ // First time we fail to resume because health verification fails
+ verify(orchestrator, times(1)).suspend(eq(hostName));
+ verify(containerOperations, times(1)).removeContainer(eq(context), any());
+ verify(containerOperations, times(1)).createContainer(eq(context), any());
+ verify(containerOperations, times(1)).startContainer(eq(context));
+ verify(orchestrator, never()).resume(eq(hostName));
+ verify(nodeRepository, never()).updateNodeAttributes(any(), any());
+
+ nodeAgent.doConverge(context);
+
+ // Do not reboot the container again
+ verify(containerOperations, times(1)).removeContainer(eq(context), any());
+ verify(containerOperations, times(1)).createContainer(eq(context), any());
+ verify(orchestrator, times(1)).resume(eq(hostName));
+ verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes()
+ .withRebootGeneration(wantedRebootGeneration)));
+ }
+
+ @Test
+ void failedNodeRunningContainerShouldStillBeRunning() {
+ final NodeSpec node = nodeBuilder(NodeState.failed)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
+ }
+
+ @Test
+ void readyNodeLeadsToNoAction() {
+ final NodeSpec node = nodeBuilder(NodeState.ready).build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+ nodeAgent.doConverge(context);
+ nodeAgent.doConverge(context);
+
+ // Should only be called once, when we initialize
+ verify(containerOperations, times(1)).getContainer(eq(context));
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(containerOperations, never()).createContainer(eq(context), any());
+ verify(containerOperations, never()).startContainer(eq(context));
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
+ }
+
+ @Test
+ void inactiveNodeRunningContainerShouldStillBeRunning() {
+ final NodeSpec node = nodeBuilder(NodeState.inactive)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ final InOrder inOrder = inOrder(storageMaintainer, containerOperations);
+ inOrder.verify(containerOperations, never()).removeContainer(eq(context), any());
+
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
+ }
+
+ @Test
+ void reservedNodeDoesNotUpdateNodeRepoWithVersion() {
+ final NodeSpec node = nodeBuilder(NodeState.reserved)
+ .wantedDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
+ }
+
+ private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) {
+ NodeSpec.Builder builder = nodeBuilder(nodeState)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage);
+ wantedRestartGeneration.ifPresent(restartGeneration -> builder
+ .wantedRestartGeneration(restartGeneration).currentRestartGeneration(restartGeneration));
+
+ NodeSpec node = builder.build();
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ final InOrder inOrder = inOrder(storageMaintainer, containerOperations, nodeRepository);
+ inOrder.verify(containerOperations, times(1)).stopServices(eq(context));
+ inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any(), eq(true));
+ inOrder.verify(containerOperations, times(1)).removeContainer(eq(context), any());
+ inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context));
+ inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready));
+
+ verify(containerOperations, never()).createContainer(eq(context), any());
+ verify(containerOperations, never()).startContainer(eq(context));
+ verify(containerOperations, never()).suspendNode(eq(context));
+ verify(containerOperations, times(1)).stopServices(eq(context));
+ verify(orchestrator, never()).resume(any(String.class));
+ verify(orchestrator, never()).suspend(any(String.class));
+ // current Docker image and vespa version should be cleared
+ verify(nodeRepository, times(1)).updateNodeAttributes(
+ eq(hostName), eq(new NodeAttributes().withDockerImage(DockerImage.EMPTY).withVespaVersion(Version.emptyVersion)));
+ }
+
+ @Test
+ void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() {
+ nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.of(1L));
+ }
+
+ @Test
+ void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycledNoRestartGeneration() {
+ nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.empty());
+ }
+
+ @Test
+ void testRestartDeadContainerAfterNodeAdminRestart() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .currentDockerImage(dockerImage).wantedDockerImage(dockerImage)
+ .currentVespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false);
+
+ when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, times(1)).removeContainer(eq(context), any());
+ verify(containerOperations, times(1)).createContainer(eq(context), any());
+ verify(containerOperations, times(1)).startContainer(eq(context));
+ }
+
+ @Test
+ void resumeProgramRunsUntilSuccess() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(1).currentRestartGeneration(1)
+ .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
+
+ final InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository);
+ doThrow(new RuntimeException("Failed 1st time"))
+ .doReturn("")
+ .when(containerOperations).resumeNode(eq(context));
+
+ // 1st try
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to throw an exception");
+ } catch (RuntimeException ignored) {
+ }
+
+ inOrder.verify(containerOperations, times(1)).resumeNode(any());
+ inOrder.verifyNoMoreInteractions();
+
+ // 2nd try
+ nodeAgent.doConverge(context);
+
+ inOrder.verify(containerOperations).resumeNode(any());
+ inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ void start_container_subtask_failure_leads_to_container_restart() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion)
+ .wantedRestartGeneration(1).currentRestartGeneration(1)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false));
+
+ when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false);
+ doThrow(new RuntimeException("Failed to set up network")).doNothing().when(containerOperations).startContainer(eq(context));
+
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to get RuntimeException");
+ } catch (RuntimeException ignored) {
+ }
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(containerOperations, times(1)).createContainer(eq(context), any());
+ verify(containerOperations, times(1)).startContainer(eq(context));
+ verify(nodeAgent, never()).resumeNodeIfNeeded(any());
+
+ // The docker container was actually started and is running, but subsequent exec calls to set up
+ // networking failed
+ mockGetContainer(dockerImage, true);
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, times(1)).removeContainer(eq(context), any());
+ verify(containerOperations, times(2)).createContainer(eq(context), any());
+ verify(containerOperations, times(2)).startContainer(eq(context));
+ verify(nodeAgent, times(1)).resumeNodeIfNeeded(any());
+ }
+
+ @Test
+ void testRunningConfigServer() {
+ final NodeSpec node = nodeBuilder(NodeState.active)
+ .type(NodeType.config)
+ .wantedDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion)
+ .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN)
+ .build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+ when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false);
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(orchestrator, never()).suspend(any(String.class));
+
+ final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer);
+ inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any());
+ inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any());
+ inOrder.verify(containerOperations, times(1)).startContainer(eq(context));
+ inOrder.verify(aclMaintainer, times(1)).converge(eq(context));
+ inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
+ inOrder.verify(nodeRepository).updateNodeAttributes(
+ hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0));
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+
+ // Tests that only containers without owners are stopped
+ @Test
+ void testThatStopContainerDependsOnOwnerPresent() {
+ verifyThatContainerIsStopped(NodeState.parked, Optional.empty());
+ verifyThatContainerIsStopped(NodeState.parked, Optional.of(ApplicationId.defaultId()));
+ verifyThatContainerIsStopped(NodeState.failed, Optional.empty());
+ verifyThatContainerIsStopped(NodeState.failed, Optional.of(ApplicationId.defaultId()));
+ verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId()));
+ }
+
+ @Test
+ void initial_cpu_cap_test() {
+ NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion);
+
+ NodeAgentContext context = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30));
+
+ InOrder inOrder = inOrder(orchestrator, containerOperations);
+
+ ConvergenceException healthCheckException = ConvergenceException.ofTransient("Not yet up");
+ doThrow(healthCheckException).when(healthChecker).verifyHealth(any());
+ for (int i = 0; i < 3; i++) {
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to fail with health check exception");
+ } catch (ConvergenceException e) {
+ assertEquals(healthCheckException, e);
+ }
+ timer.advance(Duration.ofSeconds(30));
+ }
+
+ doNothing().when(healthChecker).verifyHealth(any());
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to fail due to warm up period not yet done");
+ } catch (ConvergenceException e) {
+ assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage());
+ }
+ inOrder.verify(orchestrator, never()).resume(any());
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any());
+
+
+ timer.advance(Duration.ofSeconds(31));
+ nodeAgent.doConverge(context);
+
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 2, 16)));
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(containerOperations, never()).startContainer(any());
+ inOrder.verify(orchestrator, never()).resume(any());
+
+ // No changes
+ nodeAgent.converge(context);
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any());
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(orchestrator, never()).resume(any());
+ }
+
+ @Test
+ void resumes_normally_if_container_is_already_capped_on_start() {
+ NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(1).currentRestartGeneration(1);
+
+ NodeAgentContext context = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+ mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true);
+
+ InOrder inOrder = inOrder(orchestrator, containerOperations);
+
+ nodeAgent.doConverge(context);
+
+ nodeAgent.converge(context);
+ inOrder.verify(orchestrator, never()).suspend(any(String.class));
+ inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any());
+ inOrder.verify(containerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
+ }
+
+ @Test
+ void uncaps_and_caps_cpu_for_services_restart() {
+ NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(2).currentRestartGeneration(1);
+
+ NodeAgentContext context = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+ mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true);
+
+ InOrder inOrder = inOrder(orchestrator, containerOperations);
+
+ nodeAgent.converge(context);
+ inOrder.verify(orchestrator, times(1)).suspend(eq(hostName));
+ inOrder.verify(containerOperations, times(1)).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 0, 16)));
+ inOrder.verify(containerOperations, times(1)).restartVespa(eq(context));
+
+ mockGetContainer(dockerImage, ContainerResources.from(0, 0, 16), true);
+ doNothing().when(healthChecker).verifyHealth(any());
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to fail due to warm up period not yet done");
+ } catch (ConvergenceException e) {
+ assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage());
+ }
+ inOrder.verify(orchestrator, never()).resume(any());
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any());
+
+
+ timer.advance(Duration.ofSeconds(31));
+ nodeAgent.doConverge(context);
+ inOrder.verify(orchestrator, times(1)).resume(eq(hostName));
+ }
+
+ @Test
+ void resume_during_first_warmup() {
+ InOrder inOrder = inOrder(orchestrator, nodeRepository);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+ mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true);
+
+ // Warmup period prevents resume when node has a current docker image, i.e., already existed.
+ nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).currentDockerImage(dockerImage).build()));
+ inOrder.verifyNoMoreInteractions();
+
+ nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).build()));
+ inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage)
+ .withRebootGeneration(0)
+ .withVespaVersion(Version.fromString("7.1.1"))));
+ inOrder.verifyNoMoreInteractions();
+ }
+
+
+ @Test
+ void drop_all_documents() {
+ InOrder inOrder = inOrder(orchestrator, nodeRepository);
+ BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ?
+ nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode()))
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .build();
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+
+ NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null));
+ UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile();
+ mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true);
+ assertTrue(indexPath.exists());
+
+ // Initially no changes, index is not dropped
+ nodeAgent.converge(context);
+ assertTrue(indexPath.exists());
+ inOrder.verifyNoMoreInteractions();
+
+ context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null)));
+ nodeAgent.converge(context);
+ verify(containerOperations).removeContainer(eq(context), any());
+ assertFalse(indexPath.exists());
+ inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, timer.currentTimeMillis(), null, null).toJsonNode())));
+ inOrder.verifyNoMoreInteractions();
+
+ // After droppedAt and before readiedAt are set, we cannot proceed
+ mockGetContainer(null, false);
+ context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null)));
+ nodeAgent.converge(context);
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ verify(containerOperations, never()).startContainer(eq(context));
+ inOrder.verifyNoMoreInteractions();
+
+ context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null)));
+ nodeAgent.converge(context);
+ verify(containerOperations).startContainer(eq(context));
+ inOrder.verifyNoMoreInteractions();
+
+ mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true);
+ timer.advance(Duration.ofSeconds(31));
+ nodeAgent.converge(context);
+ verify(containerOperations, times(1)).startContainer(eq(context));
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes()
+ .withRebootGeneration(0)
+ .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, timer.currentTimeMillis()).toJsonNode())));
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) {
+ NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState)
+ .type(NodeType.tenant)
+ .flavor("docker")
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage);
+
+ owner.ifPresent(nodeBuilder::owner);
+ NodeSpec node = nodeBuilder.build();
+
+ NodeAgentContext context = createContext(node);
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+
+ when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
+
+ nodeAgent.doConverge(context);
+
+ verify(containerOperations, never()).removeContainer(eq(context), any());
+ if (owner.isPresent()) {
+ verify(containerOperations, never()).stopServices(eq(context));
+ } else {
+ verify(containerOperations, times(1)).stopServices(eq(context));
+ nodeAgent.doConverge(context);
+ // Should not be called more than once, have already been stopped
+ verify(containerOperations, times(1)).stopServices(eq(context));
+ }
+ }
+
+ private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
+ return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1));
+ }
+
+ private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) {
+ mockGetContainer(dockerImage, isRunning);
+ doAnswer(invoc -> {
+ NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+ ContainerResources resources = invoc.getArgument(1, ContainerResources.class);
+ mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+ return null;
+ }).when(containerOperations).createContainer(any(), any());
+
+ doAnswer(invoc -> {
+ NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+ ContainerResources resources = invoc.getArgument(2, ContainerResources.class);
+ mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+ return null;
+ }).when(containerOperations).updateContainer(any(), any(), any());
+
+ return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations,
+ () -> RegistryCredentials.none, storageMaintainer, flagSource,
+ List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker),
+ timer, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of());
+ }
+
+ private void mockGetContainer(DockerImage dockerImage, boolean isRunning) {
+ mockGetContainer(dockerImage, ContainerResources.from(0, resources.vcpu(), resources.memoryGb()), isRunning);
+ }
+
+ private void mockGetContainer(DockerImage dockerImage, ContainerResources containerResources, boolean isRunning) {
+ doAnswer(invoc -> {
+ NodeAgentContext context = invoc.getArgument(0);
+ if (!hostName.equals(context.hostname().value()))
+ throw new IllegalArgumentException();
+ return dockerImage != null ?
+ Optional.of(new Container(
+ containerId,
+ ContainerName.fromHostname(hostName),
+ timer.currentTime(),
+ isRunning ? Container.State.running : Container.State.exited,
+ "image-id-1",
+ dockerImage,
+ Map.of(),
+ 42,
+ 43,
+ hostName,
+ containerResources,
+ List.of(),
+ true)) :
+ Optional.empty();
+ }).when(containerOperations).getContainer(any());
+ }
+
+ private NodeAgentContext createContext(NodeSpec nodeSpec) {
+ return NodeAgentContextImpl.builder(nodeSpec).fileSystem(fileSystem).build();
+ }
+
+ private NodeSpec.Builder nodeBuilder(NodeState state) {
+ return NodeSpec.Builder.testSpec(hostName, state).realResources(resources);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java
new file mode 100644
index 00000000000..c45d9ab4b61
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java
@@ -0,0 +1,29 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.nodeagent;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author freva
+ */
+class UserNamespaceTest {
+
+ private final UserNamespace userNamespace = new UserNamespace(1000, 2000, 10000);
+
+ @Test
+ public void translates_between_ids() {
+ assertEquals(1001, userNamespace.userIdOnHost(1));
+ assertEquals(2001, userNamespace.groupIdOnHost(1));
+ assertEquals(1, userNamespace.userIdInContainer(1001));
+ assertEquals(1, userNamespace.groupIdInContainer(2001));
+
+ assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(1));
+ assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(999999));
+
+ assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(-1));
+ assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(70_000));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java
new file mode 100644
index 00000000000..eddc7edd597
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java
@@ -0,0 +1,28 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.provider;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class DebugHandlerHelperTest {
+ @Test
+ void trivial() {
+ DebugHandlerHelper helper = new DebugHandlerHelper();
+ helper.addConstant("constant-key", "constant-value");
+
+ NodeAdminDebugHandler handler = () -> Map.of("handler-value-key", "handler-value-value");
+ helper.addHandler("handler-key", handler);
+
+ helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value");
+
+ assertEquals("{" +
+ "supplier-key=supplier-value, " +
+ "handler-key={handler-value-key=handler-value-value}, " +
+ "constant-key=constant-value" +
+ "}",
+ helper.getDebugPage().toString());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java
new file mode 100644
index 00000000000..115969c5ded
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java
@@ -0,0 +1,68 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author bjorncs
+ */
+public class DefaultEnvWriterTest {
+
+ @TempDir
+ public File temporaryFolder;
+
+ private static final Path EXAMPLE_FILE = Path.of("src/test/resources/default-env-example.txt");
+ private static final Path EXPECTED_RESULT_FILE = Path.of("src/test/resources/default-env-rewritten.txt");
+
+ private final TaskContext context = mock(TaskContext.class);
+
+ @Test
+ void default_env_is_correctly_rewritten() throws IOException {
+ Path tempFile = File.createTempFile("junit", null, temporaryFolder).toPath();
+ Files.copy(EXAMPLE_FILE, tempFile, REPLACE_EXISTING);
+
+ DefaultEnvWriter writer = new DefaultEnvWriter();
+ writer.addOverride("VESPA_HOSTNAME", "my-new-hostname");
+ writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver");
+ writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file");
+
+ boolean modified = writer.updateFile(context, tempFile);
+
+ assertTrue(modified);
+ assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile));
+ verify(context, times(1)).log(any(Logger.class), any(String.class));
+
+ modified = writer.updateFile(context, tempFile);
+ assertFalse(modified);
+ assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile));
+ verify(context, times(1)).log(any(Logger.class), any(String.class));
+ }
+
+ @Test
+ void generates_default_env_content() throws IOException {
+ DefaultEnvWriter writer = new DefaultEnvWriter();
+ writer.addOverride("VESPA_HOSTNAME", "my-new-hostname");
+ writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver");
+ writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file");
+ writer.addUnset("VESPA_LEGACY_OPTION");
+ String generatedContent = writer.generateContent();
+ assertEquals(Files.readString(EXPECTED_RESULT_FILE), generatedContent);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java
new file mode 100644
index 00000000000..76676739613
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java
@@ -0,0 +1,148 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class StringEditorTest {
+ private final StringEditor editor = new StringEditor();
+ private final Cursor cursor = editor.cursor();
+
+ @Test
+ void testBasics() {
+ assertCursor(0, 0, "");
+
+ cursor.write("hello");
+ assertCursor(0, 5, "hello");
+
+ cursor.write("one\ntwo");
+ assertCursor(1, 3, "helloone\ntwo");
+
+ cursor.deleteAll();
+ assertCursor(0, 0, "");
+
+ cursor.moveForward();
+ assertCursor(0, 0, "");
+
+ cursor.writeLine("foo");
+ assertCursor(1, 0, "foo\n");
+
+ cursor.writeLines("one", "two");
+ assertCursor(3, 0, "foo\none\ntwo\n");
+
+ cursor.deleteBackward();
+ assertCursor(2, 3, "foo\none\ntwo");
+
+ cursor.deleteBackward(2);
+ assertCursor(2, 1, "foo\none\nt");
+
+ Mark mark = cursor.createMark();
+
+ cursor.moveToStartOfPreviousLine().moveBackward(2);
+ assertCursor(0, 2, "foo\none\nt");
+
+ assertEquals("o\none\nt", cursor.getTextTo(mark));
+
+ cursor.deleteTo(mark);
+ assertCursor(0, 2, "fo");
+
+ cursor.deleteBackward(2);
+ assertCursor(0, 0, "");
+
+ cursor.writeLines("one", "two", "three").moveToStartOfBuffer();
+ assertCursor(0, 0, "one\ntwo\nthree\n");
+
+ Pattern pattern = Pattern.compile("t(.)");
+ Optional<Match> match = cursor.moveForwardToEndOfMatch(pattern);
+ assertCursor(1, 2, "one\ntwo\nthree\n");
+ assertTrue(match.isPresent());
+ assertEquals("tw", match.get().match());
+ assertEquals("", match.get().prefix());
+ assertEquals("o", match.get().suffix());
+ assertEquals(new Position(1, 0), match.get().startOfMatch());
+ assertEquals(new Position(1, 2), match.get().endOfMatch());
+ assertEquals(1, match.get().groupCount());
+ assertEquals("w", match.get().group(1));
+
+ match = cursor.moveForwardToEndOfMatch(pattern);
+ assertCursor(2, 2, "one\ntwo\nthree\n");
+ assertTrue(match.isPresent());
+ assertEquals("th", match.get().match());
+ assertEquals(1, match.get().groupCount());
+ assertEquals("h", match.get().group(1));
+
+ match = cursor.moveForwardToEndOfMatch(pattern);
+ assertCursor(2, 2, "one\ntwo\nthree\n");
+ assertFalse(match.isPresent());
+
+ assertTrue(cursor.skipBackward("h"));
+ assertCursor(2, 1, "one\ntwo\nthree\n");
+ assertFalse(cursor.skipBackward("x"));
+
+ assertTrue(cursor.skipForward("hre"));
+ assertCursor(2, 4, "one\ntwo\nthree\n");
+ assertFalse(cursor.skipForward("x"));
+
+ try {
+ cursor.moveTo(mark);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ mark = cursor.createMark();
+ cursor.moveToStartOfBuffer();
+ assertEquals(new Position(0, 0), cursor.getPosition());
+ cursor.moveTo(mark);
+ assertEquals(new Position(2, 4), cursor.getPosition());
+
+ cursor.moveTo(1, 2);
+ assertCursor(1, 2, "one\ntwo\nthree\n");
+
+ cursor.deleteSuffix();
+ assertCursor(1, 2, "one\ntw\nthree\n");
+
+ cursor.deletePrefix();
+ assertCursor(1, 0, "one\n\nthree\n");
+
+ cursor.deleteLine();
+ assertCursor(1, 0, "one\nthree\n");
+
+ cursor.deleteLine();
+ assertCursor(1, 0, "one\n");
+
+ cursor.deleteLine();
+ assertCursor(1, 0, "one\n");
+
+ cursor.moveToStartOfBuffer().moveForward().writeNewlineAfter();
+ assertCursor(0, 1, "o\nne\n");
+
+ cursor.deleteAll().writeLines("one", "two", "three", "four");
+ cursor.moveToStartOfBuffer().moveToStartOfNextLine();
+ assertCursor(1, 0, "one\ntwo\nthree\nfour\n");
+ Pattern pattern2 = Pattern.compile("(o)(.)?");
+ int count = cursor.replaceMatches(pattern2, m -> {
+ String prefix = m.group(2) == null ? "" : m.group(2);
+ return prefix + m.match() + m.group(1);
+ });
+ assertCursor(3, 5, "one\ntwoo\nthree\nfuouor\n");
+ assertEquals(2, count);
+
+ cursor.moveToStartOfBuffer().moveToEndOfLine();
+ Pattern pattern3 = Pattern.compile("o");
+ count = cursor.replaceMatches(pattern3, m -> "a");
+ assertEquals(4, count);
+ assertCursor(3, 5, "one\ntwaa\nthree\nfuauar\n");
+ }
+
+ private void assertCursor(int lineIndex, int columnIndex, String text) {
+ assertEquals(text, cursor.getBufferText());
+ assertEquals(new Position(lineIndex, columnIndex), cursor.getPosition());
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java
new file mode 100644
index 00000000000..15fb36dc3d5
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java
@@ -0,0 +1,59 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.editor;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TextBufferImplTest {
+ private final TextBufferImpl textBuffer = new TextBufferImpl();
+
+ @Test
+ void testWrite() {
+ assertEquals("", textBuffer.getString());
+ assertWrite(2, 0, "foo\nbar\n",
+ 0, 0, "foo\nbar\n");
+
+ assertWrite(1, 6, "fofirst\nsecondo\nbar\n",
+ 0, 2, "first\nsecond");
+
+ assertWrite(3, 1, "fofirst\nsecondo\nbar\na",
+ 3, 0, "a");
+ assertWrite(4, 0, "fofirst\nsecondo\nbar\na\n",
+ 3, 1, "\n");
+ }
+
+ @Test
+ void testDelete() {
+ write(0, 0, "foo\nbar\nzoo\n");
+ delete(0, 2, 2, 1);
+ assertEquals("fooo\n", textBuffer.getString());
+
+ delete(0, 4, 1, 0);
+ assertEquals("fooo", textBuffer.getString());
+
+ delete(0, 0, 0, 4);
+ assertEquals("", textBuffer.getString());
+
+ delete(0, 0, 0, 0);
+ assertEquals("", textBuffer.getString());
+ }
+
+ private void assertWrite(int expectedLineIndex, int expectedColumnIndex, String expectedString,
+ int lineIndex, int columnIndex, String text) {
+ Position position = write(lineIndex, columnIndex, text);
+ assertEquals(new Position(expectedLineIndex, expectedColumnIndex), position);
+ assertEquals(expectedString, textBuffer.getString());
+ }
+
+ private Position write(int lineIndex, int columnIndex, String text) {
+ return textBuffer.write(new Position(lineIndex, columnIndex), text);
+ }
+
+ private void delete(int startLineIndex, int startColumnIndex,
+ int endLineIndex, int endColumnIndex) {
+ textBuffer.delete(new Position(startLineIndex, startColumnIndex),
+ new Position(endLineIndex, endColumnIndex));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java
new file mode 100644
index 00000000000..507bf706484
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java
@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class DiskSizeTest {
+
+ @Test
+ void bytes_to_display_count_test() {
+ assertEquals("-1 bytes", DiskSize.of(-1).asString());
+ assertEquals("123 bytes", DiskSize.of(123).asString());
+ assertEquals("1 kB", DiskSize.of(1_000).asString());
+ assertEquals("15 MB", DiskSize.of(15_000_000).asString());
+ assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString());
+ assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString());
+ assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1));
+ assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2));
+ assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString());
+ assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java
new file mode 100644
index 00000000000..9a651494854
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java
@@ -0,0 +1,122 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.file.FileSystem;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class EditorTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final UnixPath path = new UnixPath(fileSystem.getPath("/file"));
+
+ @Test
+ void testEdit() {
+ path.writeUtf8File(joinLines("first", "second", "third"));
+
+ LineEditor lineEditor = mock(LineEditor.class);
+ when(lineEditor.edit(any())).thenReturn(
+ LineEdit.none(), // don't edit the first line
+ LineEdit.remove(), // remove the second
+ LineEdit.replaceWith("replacement")); // replace the third
+
+ Editor editor = new Editor(path.toPath(), lineEditor);
+ TaskContext context = mock(TaskContext.class);
+
+ assertTrue(editor.converge(context));
+
+ verify(lineEditor, times(3)).edit(any());
+
+ // Verify the system modification message
+ ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class);
+ verify(context).recordSystemModification(any(), modificationMessage.capture());
+ assertEquals(
+ "Patching file /file:\n-second\n-third\n+replacement\n",
+ modificationMessage.getValue());
+
+ // Verify the new contents of the file:
+ assertEquals(joinLines("first", "replacement"), path.readUtf8File());
+ }
+
+ @Test
+ void testInsert() {
+ path.writeUtf8File(joinLines("second", "eight", "fifth", "seventh"));
+
+ LineEditor lineEditor = mock(LineEditor.class);
+ when(lineEditor.edit(any())).thenReturn(
+ LineEdit.insertBefore("first"), // insert first, and keep the second line
+ LineEdit.replaceWith("third", "fourth"), // remove eight, and replace with third and fourth instead
+ LineEdit.none(), // Keep fifth
+ LineEdit.insert(List.of("sixth"), // insert sixth before seventh
+ List.of("eight"))); // add eight after seventh
+
+ Editor editor = new Editor(path.toPath(), lineEditor);
+ TaskContext context = mock(TaskContext.class);
+
+ assertTrue(editor.converge(context));
+
+ // Verify the system modification message
+ ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class);
+ verify(context).recordSystemModification(any(), modificationMessage.capture());
+ assertEquals(
+ "Patching file /file:\n" +
+ "+first\n" +
+ "-eight\n" +
+ "+third\n" +
+ "+fourth\n" +
+ "+sixth\n" +
+ "+eight\n",
+ modificationMessage.getValue());
+
+ // Verify the new contents of the file:
+ assertEquals(joinLines("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eight"),
+ path.readUtf8File());
+ }
+
+ @Test
+ void noop() {
+ path.writeUtf8File("line\n");
+
+ LineEditor lineEditor = mock(LineEditor.class);
+ when(lineEditor.edit(any())).thenReturn(LineEdit.none());
+
+ Editor editor = new Editor(path.toPath(), lineEditor);
+ TaskContext context = mock(TaskContext.class);
+
+ assertFalse(editor.converge(context));
+
+ verify(lineEditor, times(1)).edit(any());
+
+ // Verify the system modification message
+ verify(context, times(0)).recordSystemModification(any(), any());
+
+ // Verify same contents
+ assertEquals("line\n", path.readUtf8File());
+ }
+
+ @Test
+ void testMissingFile() {
+ LineEditor lineEditor = mock(LineEditor.class);
+ when(lineEditor.onComplete()).thenReturn(List.of("line"));
+
+ TaskContext context = mock(TaskContext.class);
+ var editor = new Editor(path.toPath(), lineEditor);
+ editor.converge(context);
+
+ assertEquals("line\n", path.readUtf8File());
+ }
+
+ private static String joinLines(String... lines) {
+ return String.join("\n", lines) + "\n";
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java
new file mode 100644
index 00000000000..8559e36fe8b
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java
@@ -0,0 +1,39 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class FileAttributesCacheTest {
+ @Test
+ void exists() {
+ UnixPath unixPath = mock(UnixPath.class);
+ FileAttributesCache cache = new FileAttributesCache(unixPath);
+
+ when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty());
+ assertFalse(cache.get().isPresent());
+ verify(unixPath, times(1)).getAttributesIfExists();
+ verifyNoMoreInteractions(unixPath);
+
+ FileAttributes attributes = new FileAttributes(Instant.EPOCH, 0, 0, "", false, false, 0, 0, 0);
+ when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes));
+ when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes));
+ assertTrue(cache.get().isPresent());
+ verify(unixPath, times(1 + 1)).getAttributesIfExists();
+ verifyNoMoreInteractions(unixPath);
+
+ assertEquals(attributes, cache.getOrThrow());
+ verifyNoMoreInteractions(unixPath);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java
new file mode 100644
index 00000000000..ed183738ef0
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java
@@ -0,0 +1,20 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import org.junit.jupiter.api.Test;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMajor;
+import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMinor;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+class FileAttributesTest {
+
+ @Test
+ void parse_dev_t() {
+ assertEquals(0x12345BCD, deviceMajor(0x1234567890ABCDEFL));
+ assertEquals(0x67890AEF, deviceMinor(0x1234567890ABCDEFL));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java
new file mode 100644
index 00000000000..e1cea37ccbc
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java
@@ -0,0 +1,62 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class FileContentCacheTest {
+ private final UnixPath unixPath = mock(UnixPath.class);
+ private final FileContentCache cache = new FileContentCache(unixPath);
+
+ private final byte[] content = "content".getBytes(StandardCharsets.UTF_8);
+ private final byte[] newContent = "new-content".getBytes(StandardCharsets.UTF_8);
+
+ @Test
+ void get() {
+ when(unixPath.readBytes()).thenReturn(content);
+ assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0)));
+ verify(unixPath, times(1)).readBytes();
+ verifyNoMoreInteractions(unixPath);
+
+ // cache hit
+ assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0)));
+ verify(unixPath, times(1)).readBytes();
+ verifyNoMoreInteractions(unixPath);
+
+ // cache miss
+ when(unixPath.readBytes()).thenReturn(newContent);
+ assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1)));
+ verify(unixPath, times(1 + 1)).readBytes();
+ verifyNoMoreInteractions(unixPath);
+
+ // cache hit both at times 0 and 1
+ assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(0)));
+ verify(unixPath, times(1 + 1)).readBytes();
+ verifyNoMoreInteractions(unixPath);
+ assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1)));
+ verify(unixPath, times(1 + 1)).readBytes();
+ verifyNoMoreInteractions(unixPath);
+ }
+
+ @Test
+ void updateWith() {
+ cache.updateWith(content, Instant.ofEpochMilli(2));
+ assertArrayEquals(content, cache.get(Instant.ofEpochMilli(2)));
+ verifyNoMoreInteractions(unixPath);
+
+ cache.updateWith(newContent, Instant.ofEpochMilli(4));
+ assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(4)));
+ verifyNoMoreInteractions(unixPath);
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java
new file mode 100644
index 00000000000..f7fb66fca94
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java
@@ -0,0 +1,27 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class FileDeleterTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final UnixPath path = new UnixPath(fileSystem.getPath("/tmp/foo"));
+ private final FileDeleter deleter = new FileDeleter(path.toPath());
+ private final TaskContext context = mock(TaskContext.class);
+
+ @Test
+ void deleteExisting() {
+ assertFalse(deleter.converge(context));
+ path.createParents().writeUtf8File("bar");
+ assertTrue(deleter.converge(context));
+ assertFalse(deleter.converge(context));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java
new file mode 100644
index 00000000000..76941d3333b
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java
@@ -0,0 +1,238 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Set.of;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * @author freva
+ */
+
+public class FileFinderTest {
+
+ @Nested
+ public class GeneralLogicTests {
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+
+ @Test
+ void all_files_non_recursive() {
+ assertFileHelper(FileFinder.files(testRoot())
+ .maxDepth(1),
+
+ of("file-1.json", "test.json", "test.txt"),
+ of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2"));
+ }
+
+ @Test
+ void all_files_recursive() {
+ assertFileHelper(FileFinder.files(testRoot()),
+
+ of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json", "test/subdir-1/test"),
+ of("test", "test/subdir-1", "test/subdir-2"));
+ }
+
+ @Test
+ void all_files_recursive_with_prune_relative() {
+ assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")),
+
+ of("file-1.json", "test.json", "test.txt"),
+ of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2"));
+ }
+
+ @Test
+ void all_files_recursive_with_prune_absolute() {
+ assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")),
+
+ of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"),
+ of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2"));
+ }
+
+ @Test
+ void throws_if_prune_path_not_under_base_path() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ FileFinder.files(Path.of("/some/path")).prune(Path.of("/other/path"));
+ });
+ }
+
+ @Test
+ void with_file_filter_recursive() {
+ assertFileHelper(FileFinder.files(testRoot())
+ .match(FileFinder.nameEndsWith(".json")),
+
+ of("file-1.json", "test.json", "test/data.json"),
+ of("test.txt", "test", "test/file.txt", "test/subdir-1", "test/subdir-1/test", "test/subdir-2"));
+ }
+
+ @Test
+ void all_files_limited_depth() {
+ assertFileHelper(FileFinder.files(testRoot())
+ .maxDepth(2),
+
+ of("test.txt", "file-1.json", "test.json", "test/file.txt", "test/data.json"),
+ of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2"));
+ }
+
+ @Test
+ void directory_with_filter() {
+ assertFileHelper(FileFinder.directories(testRoot())
+ .match(FileFinder.nameStartsWith("subdir"))
+ .maxDepth(2),
+
+ of("test/subdir-1", "test/subdir-2"),
+ of("file-1.json", "test.json", "test.txt", "test", "test/file.txt", "test/data.json"));
+ }
+
+ @Test
+ void match_file_and_directory_with_same_name() {
+ assertFileHelper(FileFinder.from(testRoot())
+ .match(FileFinder.nameEndsWith("test")),
+
+ of("test", "test/subdir-1/test"),
+ of("file-1.json", "test.json", "test.txt"));
+ }
+
+ @Test
+ void all_contents() {
+ assertFileHelper(FileFinder.from(testRoot())
+ .maxDepth(1),
+
+ of("file-1.json", "test.json", "test.txt", "test"),
+ of());
+
+ assertTrue(Files.exists(testRoot()));
+ }
+
+ @BeforeEach
+ public void setup() throws IOException {
+ Path root = testRoot();
+ Files.createDirectories(root);
+
+ Files.createFile(root.resolve("file-1.json"));
+ Files.createFile(root.resolve("test.json"));
+ Files.createFile(root.resolve("test.txt"));
+
+ Files.createDirectories(root.resolve("test"));
+ Files.createFile(root.resolve("test/file.txt"));
+ Files.createFile(root.resolve("test/data.json"));
+
+ Files.createDirectories(root.resolve("test/subdir-1"));
+ Files.createFile(root.resolve("test/subdir-1/test"));
+
+ Files.createDirectories(root.resolve("test/subdir-2"));
+ }
+
+ private Path testRoot() {
+ return fileSystem.getPath("/file-finder");
+ }
+
+ private void assertFileHelper(FileFinder fileFinder, Set<String> expectedList, Set<String> expectedContentsAfterDelete) {
+ Set<String> actualList = fileFinder.stream()
+ .map(FileFinder.FileAttributes::path)
+ .map(testRoot()::relativize)
+ .map(Path::toString)
+ .collect(Collectors.toSet());
+ assertEquals(expectedList, actualList);
+
+ fileFinder.deleteRecursively(mock(TaskContext.class));
+ Set<String> actualContentsAfterDelete = recursivelyListContents(testRoot()).stream()
+ .map(testRoot()::relativize)
+ .map(Path::toString)
+ .collect(Collectors.toSet());
+ assertEquals(expectedContentsAfterDelete, actualContentsAfterDelete);
+ }
+
+ private List<Path> recursivelyListContents(Path basePath) {
+ try (Stream<Path> pathStream = Files.list(basePath)) {
+ List<Path> paths = new LinkedList<>();
+ pathStream.forEach(path -> {
+ paths.add(path);
+ if (Files.isDirectory(path))
+ paths.addAll(recursivelyListContents(path));
+ });
+ return paths;
+ } catch (NoSuchFileException e) {
+ return List.of();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ @Nested
+ public class FilterUnitTests {
+
+ private final BasicFileAttributes attributes = mock(BasicFileAttributes.class);
+
+ @Test
+ void age_filter_test() {
+ Path path = Path.of("/my/fake/path");
+ when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1))));
+ FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes);
+
+ assertFalse(FileFinder.olderThan(Duration.ofMinutes(61)).test(fileAttributes));
+ assertTrue(FileFinder.olderThan(Duration.ofMinutes(59)).test(fileAttributes));
+
+ assertTrue(FileFinder.youngerThan(Duration.ofMinutes(61)).test(fileAttributes));
+ assertFalse(FileFinder.youngerThan(Duration.ofMinutes(59)).test(fileAttributes));
+ }
+
+ @Test
+ void size_filters() {
+ Path path = Path.of("/my/fake/path");
+ when(attributes.size()).thenReturn(100L);
+ FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes);
+
+ assertFalse(FileFinder.largerThan(101).test(fileAttributes));
+ assertTrue(FileFinder.largerThan(99).test(fileAttributes));
+
+ assertTrue(FileFinder.smallerThan(101).test(fileAttributes));
+ assertFalse(FileFinder.smallerThan(99).test(fileAttributes));
+ }
+
+ @Test
+ void filename_filters() {
+ Path path = Path.of("/my/fake/path/some-12352-file.json");
+ FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes);
+
+ assertTrue(FileFinder.nameStartsWith("some-").test(fileAttributes));
+ assertFalse(FileFinder.nameStartsWith("som-").test(fileAttributes));
+
+ assertTrue(FileFinder.nameEndsWith(".json").test(fileAttributes));
+ assertFalse(FileFinder.nameEndsWith("file").test(fileAttributes));
+
+ assertTrue(FileFinder.nameMatches(Pattern.compile("some-[0-9]+-file.json")).test(fileAttributes));
+ assertTrue(FileFinder.nameMatches(Pattern.compile("^some-[0-9]+-file.json$")).test(fileAttributes));
+ assertFalse(FileFinder.nameMatches(Pattern.compile("some-[0-9]-file.json")).test(fileAttributes));
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java
new file mode 100644
index 00000000000..e418833ab50
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java
@@ -0,0 +1,73 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystem;
+import java.nio.file.NoSuchFileException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author hakonhall
+ */
+class FileMoverTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final TaskContext context = mock(TaskContext.class);
+ private final UnixPath source = new UnixPath(fileSystem.getPath("/from/source"));
+ private final UnixPath destination = new UnixPath(fileSystem.getPath("/to/destination"));
+ private final FileMover mover = new FileMover(source.toPath(), destination.toPath());
+
+ @Test
+ void movingRegularFile() {
+ assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/from/source");
+
+ source.createParents().writeUtf8File("content");
+ assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/to/destination");
+
+ destination.createParents();
+ assertTrue(mover.converge(context));
+ assertFalse(source.exists());
+ assertTrue(destination.exists());
+ assertEquals("content", destination.readUtf8File());
+
+ assertFalse(mover.converge(context));
+
+ source.writeUtf8File("content 2");
+ assertConvergeThrows(() -> mover.converge(context), FileAlreadyExistsException.class, "/to/destination");
+
+ mover.replaceExisting();
+ assertTrue(mover.converge(context));
+
+ source.writeUtf8File("content 3");
+ destination.deleteIfExists();
+ destination.createDirectory();
+ assertTrue(mover.converge(context));
+ }
+
+ private void assertConvergeThrows(Runnable runnable, Class<?> expectedRootExceptionClass, String expectedMessage) {
+ try {
+ runnable.run();
+ fail();
+ } catch (Throwable t) {
+ Throwable rootCause = t;
+ do {
+ Throwable cause = rootCause.getCause();
+ if (cause == null) break;
+ rootCause = cause;
+ } while (true);
+
+ assertTrue(expectedRootExceptionClass.isInstance(rootCause), "Unexpected root cause: " + rootCause);
+ assertEquals(expectedMessage, rootCause.getMessage());
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java
new file mode 100644
index 00000000000..b0992e9826a
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java
@@ -0,0 +1,60 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author hakonhall
+ */
+public class FileSnapshotTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt"));
+
+ private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath());
+
+ @Test
+ void fileDoesNotExist() {
+ assertFalse(fileSnapshot.exists());
+ assertFalse(fileSnapshot.attributes().isPresent());
+ assertFalse(fileSnapshot.content().isPresent());
+ assertEquals(path.toPath(), fileSnapshot.path());
+ }
+
+ @Test
+ void directory() {
+ path.createParents().createDirectory();
+ fileSnapshot = fileSnapshot.snapshot();
+ assertTrue(fileSnapshot.exists());
+ assertTrue(fileSnapshot.attributes().isPresent());
+ assertTrue(fileSnapshot.attributes().get().isDirectory());
+ }
+
+ @Test
+ void regularFile() {
+ path.createParents().writeUtf8File("file content");
+ fileSnapshot = fileSnapshot.snapshot();
+ assertTrue(fileSnapshot.exists());
+ assertTrue(fileSnapshot.attributes().isPresent());
+ assertTrue(fileSnapshot.attributes().get().isRegularFile());
+ assertTrue(fileSnapshot.utf8Content().isPresent());
+ assertEquals("file content", fileSnapshot.utf8Content().get());
+
+ FileSnapshot newFileSnapshot = fileSnapshot.snapshot();
+ assertSame(fileSnapshot, newFileSnapshot);
+ }
+
+ @Test
+ void fileRemoval() {
+ path.createParents().writeUtf8File("file content");
+ fileSnapshot = fileSnapshot.snapshot();
+ assertTrue(fileSnapshot.exists());
+ path.deleteIfExists();
+ fileSnapshot = fileSnapshot.snapshot();
+ assertFalse(fileSnapshot.exists());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java
new file mode 100644
index 00000000000..c60de78bf8c
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java
@@ -0,0 +1,79 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class FileSyncTest {
+ private final TestTaskContext taskContext = new TestTaskContext();
+ private final FileSystem fileSystem = TestFileSystem.create();
+
+ private final Path path = fileSystem.getPath("/dir/file.txt");
+ private final UnixPath unixPath = new UnixPath(path);
+ private final FileSync fileSync = new FileSync(path);
+
+ private String content = "content";
+ private int ownerId = 123; // default is 1
+ private int groupId = 456; // default is 2
+ private String permissions = "rw-r-xr--";
+
+ @Test
+ void trivial() {
+ assertConvergence("Creating file /dir/file.txt with permissions rw-r-xr--",
+ "Changing user ID of /dir/file.txt from 1 to 123",
+ "Changing group ID of /dir/file.txt from 2 to 456");
+
+ content = "new-content";
+ assertConvergence("Patching file /dir/file.txt");
+
+ ownerId = 124;
+ assertConvergence("Changing user ID of /dir/file.txt from 123 to 124");
+
+ groupId = 457;
+ assertConvergence("Changing group ID of /dir/file.txt from 456 to 457");
+
+ permissions = "rwxr--rwx";
+ assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " +
+ permissions);
+ }
+
+ private void assertConvergence(String... systemModificationMessages) {
+ PartialFileData fileData = PartialFileData.builder()
+ .withContent(content)
+ .withOwnerId(ownerId)
+ .withGroupId(groupId)
+ .withPermissions(permissions)
+ .create();
+ taskContext.clearSystemModificationLog();
+ assertTrue(fileSync.convergeTo(taskContext, fileData));
+
+ assertTrue(Files.isRegularFile(path));
+ fileData.getContent().ifPresent(content -> assertArrayEquals(content, unixPath.readBytes()));
+ fileData.getOwnerId().ifPresent(owner -> assertEquals((int) owner, unixPath.getOwnerId()));
+ fileData.getGroupId().ifPresent(group -> assertEquals((int) group, unixPath.getGroupId()));
+ fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions()));
+
+ List<String> actualMods = taskContext.getSystemModificationLog();
+ List<String> expectedMods = List.of(systemModificationMessages);
+ assertEquals(expectedMods, actualMods);
+
+ UnixPath unixPath = new UnixPath(path);
+ Instant lastModifiedTime = unixPath.getLastModifiedTime();
+ taskContext.clearSystemModificationLog();
+ assertFalse(fileSync.convergeTo(taskContext, fileData));
+ assertEquals(lastModifiedTime, unixPath.getLastModifiedTime());
+
+ actualMods = taskContext.getSystemModificationLog();
+ assertEquals(new ArrayList<>(), actualMods);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java
new file mode 100644
index 00000000000..1264206bef3
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java
@@ -0,0 +1,62 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.time.Instant;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class FileWriterTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final TaskContext context = mock(TaskContext.class);
+
+ @Test
+ void testWrite() {
+ final String content = "content";
+ final String permissions = "rwxr-xr-x";
+ final int owner = 123;
+ final int group = 456;
+
+ Path path = fileSystem.getPath("/opt/vespa/tmp/file.txt");
+ FileWriter writer = new FileWriter(path, () -> content)
+ .withPermissions(permissions)
+ .withOwnerId(owner)
+ .withGroupId(group)
+ .onlyIfFileDoesNotAlreadyExist();
+ assertTrue(writer.converge(context));
+ verify(context, times(1)).recordSystemModification(any(), eq("Creating file " + path + " with permissions rwxr-xr-x"));
+
+ UnixPath unixPath = new UnixPath(path);
+ assertEquals(content, unixPath.readUtf8File());
+ assertEquals(permissions, unixPath.getPermissions());
+ assertEquals(owner, unixPath.getOwnerId());
+ assertEquals(group, unixPath.getGroupId());
+ Instant fileTime = unixPath.getLastModifiedTime();
+
+ // Second time is a no-op.
+ assertFalse(writer.converge(context));
+ assertEquals(fileTime, unixPath.getLastModifiedTime());
+ }
+
+ @Test
+ void testAtomicWrite() {
+ FileWriter writer = new FileWriter(fileSystem.getPath("/foo/bar"))
+ .atomicWrite(true);
+
+ assertTrue(writer.converge(context, "content"));
+
+ verify(context).recordSystemModification(any(), eq("Creating file /foo/bar"));
+ assertEquals("content", new UnixPath(writer.path()).readUtf8File());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
new file mode 100644
index 00000000000..11675bbe46f
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
@@ -0,0 +1,87 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author hakonhall
+ */
+public class MakeDirectoryTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final TestTaskContext context = new TestTaskContext();
+
+ private final String path = "/parent/dir";
+ private String permissions = "rwxr----x";
+ private int ownerId = 123;
+ private int groupId = 456;
+
+ @Test
+ void newDirectory() {
+ verifySystemModifications(
+ "Creating directory " + path,
+ "Changing user ID of /parent/dir from 1 to 123",
+ "Changing group ID of /parent/dir from 2 to 456");
+
+ ownerId = 124;
+ verifySystemModifications("Changing user ID of /parent/dir from 123 to 124");
+
+ groupId = 457;
+ verifySystemModifications("Changing group ID of /parent/dir from 456 to 457");
+
+ permissions = "--x---r--";
+ verifySystemModifications("Changing permissions of /parent/dir from rwxr----x to --x---r--");
+ }
+
+ private void verifySystemModifications(String... modifications) {
+ context.clearSystemModificationLog();
+ MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path))
+ .createParents()
+ .withPermissions(permissions)
+ .withOwnerId(ownerId)
+ .withGroupId(groupId);
+ assertTrue(makeDirectory.converge(context));
+
+ assertEquals(List.of(modifications), context.getSystemModificationLog());
+
+ context.clearSystemModificationLog();
+ assertFalse(makeDirectory.converge(context));
+ assertEquals(List.of(), context.getSystemModificationLog());
+ }
+
+ @Test
+ void exceptionIfMissingParent() {
+ String path = "/parent/dir";
+ MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path));
+
+ try {
+ makeDirectory.converge(context);
+ } catch (UncheckedIOException e) {
+ if (e.getCause() instanceof NoSuchFileException) {
+ return;
+ }
+ throw e;
+ }
+ fail();
+ }
+
+ @Test
+ void okIfParentExists() {
+ String path = "/dir";
+ MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path));
+ assertTrue(makeDirectory.converge(context));
+ assertTrue(Files.isDirectory(fileSystem.getPath(path)));
+
+ MakeDirectory makeDirectory2 = new MakeDirectory(fileSystem.getPath(path));
+ assertFalse(makeDirectory2.converge(context));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java
new file mode 100644
index 00000000000..79fa1cf6ea2
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author hakonhall
+ */
+public class StoredBooleanTest {
+ private final TaskContext context = mock(TaskContext.class);
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Path path = fileSystem.getPath("/foo");
+ private final StoredBoolean storedBoolean = new StoredBoolean(path);
+
+ @Test
+ void storedBoolean() {
+ assertFalse(storedBoolean.value());
+ storedBoolean.set(context);
+ assertTrue(storedBoolean.value());
+ storedBoolean.clear(context);
+ assertFalse(storedBoolean.value());
+ }
+
+ @Test
+ void testCompatibility() throws IOException {
+ StoredInteger storedInteger = new StoredInteger(path);
+ assertFalse(storedBoolean.value());
+
+ storedInteger.write(context, 1);
+ assertTrue(storedBoolean.value());
+
+ storedInteger.write(context, 2);
+ assertTrue(storedBoolean.value());
+
+ storedInteger.write(context, 0);
+ assertFalse(storedBoolean.value());
+
+ Files.delete(path);
+ assertFalse(storedBoolean.value());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java
new file mode 100644
index 00000000000..d9dfcefc7e3
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java
@@ -0,0 +1,39 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class TemplateTest {
+
+ @Test
+ void basic() {
+ FileSystem fileSystem = TestFileSystem.create();
+ Path templatePath = fileSystem.getPath("/example.vm");
+ String templateContent = "a $x, $y b";
+ new UnixPath(templatePath).writeUtf8File(templateContent);
+
+ Path toPath = fileSystem.getPath("/example");
+ TaskContext taskContext = mock(TaskContext.class);
+ boolean converged = Template.at(templatePath)
+ .set("x", "foo")
+ .set("y", "bar")
+ .getFileWriterTo(toPath)
+ .converge(taskContext);
+
+ assertTrue(converged);
+
+ String actualContent = new UnixPath(toPath).readUtf8File();
+ assertEquals("a foo, bar b", actualContent);
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java
new file mode 100644
index 00000000000..5892a9b9f53
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java
@@ -0,0 +1,199 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author hakonhall
+ */
+public class UnixPathTest {
+
+ private final FileSystem fs = TestFileSystem.create();
+
+ @Test
+ void createParents() {
+ Path parentDirectory = fs.getPath("/a/b/c");
+ Path filePath = parentDirectory.resolve("bar");
+ UnixPath path = new UnixPath(filePath);
+
+ assertFalse(Files.exists(fs.getPath("/a")));
+ path.createParents();
+ assertTrue(Files.exists(parentDirectory));
+ }
+
+ @Test
+ void utf8File() {
+ String original = "foo\nbar\n";
+ UnixPath path = new UnixPath(fs.getPath("example.txt"));
+ path.writeUtf8File(original);
+ String fromFile = path.readUtf8File();
+ assertEquals(original, fromFile);
+ assertEquals(List.of("foo", "bar"), path.readLines());
+ }
+
+ @Test
+ void touch() {
+ UnixPath path = new UnixPath(fs.getPath("example.txt"));
+ assertTrue(path.create());
+ assertEquals("", path.readUtf8File());
+ assertFalse(path.create());
+ }
+
+ @Test
+ void permissions() {
+ String expectedPermissions = "rwxr-x---";
+ UnixPath path = new UnixPath(fs.getPath("file.txt"));
+ path.writeUtf8File("foo");
+ path.setPermissions(expectedPermissions);
+ assertEquals(expectedPermissions, path.getPermissions());
+ }
+
+ @Test
+ void badPermissionsString() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new UnixPath(fs.getPath("file.txt")).setPermissions("abcdefghi");
+ });
+ }
+
+ @Test
+ void owner() {
+ Path path = fs.getPath("file.txt");
+ UnixPath unixPath = new UnixPath(path);
+ unixPath.writeUtf8File("foo");
+
+ unixPath.setOwnerId(123);
+ assertEquals(123, unixPath.getOwnerId());
+
+ unixPath.setGroupId(456);
+ assertEquals(456, unixPath.getGroupId());
+ }
+
+ @Test
+ void createDirectoryWithPermissions() {
+ Path path = fs.getPath("dir");
+ UnixPath unixPath = new UnixPath(path);
+ String permissions = "rwxr-xr--";
+ assertTrue(unixPath.createDirectory(permissions));
+ assertTrue(unixPath.isDirectory());
+ assertEquals(permissions, unixPath.getPermissions());
+ assertFalse(unixPath.createDirectory(permissions));
+ }
+
+ @Test
+ void createSymbolicLink() {
+ String original = "foo\nbar\n";
+ UnixPath path = new UnixPath(fs.getPath("example.txt"));
+ path.writeUtf8File(original);
+ String fromFile = path.readUtf8File();
+ assertEquals(original, fromFile);
+
+ UnixPath link = path.createSymbolicLink(fs.getPath("link-to-example.txt"));
+ assertEquals(original, link.readUtf8File());
+ }
+
+ @Test
+ void readBytesIfExists() {
+ UnixPath path = new UnixPath(fs.getPath("example.txt"));
+ assertFalse(path.readBytesIfExists().isPresent());
+ path.writeBytes(new byte[]{42});
+ assertArrayEquals(new byte[]{42}, path.readBytesIfExists().get());
+ }
+
+ @Test
+ void deleteRecursively() throws Exception {
+ // Create the following file tree:
+ //
+ // /dir1
+ // |--- dir2
+ // |--- file1
+ // /link1 -> /dir1/dir2
+ //
+ var dir1 = fs.getPath("/dir1");
+ var dir2 = dir1.resolve("dir2");
+ var file1 = dir2.resolve("file1");
+ Files.createDirectories(dir2);
+ Files.writeString(file1, "file1");
+ var link1 = Files.createSymbolicLink(fs.getPath("/link1"), dir2);
+
+ new UnixPath(link1).deleteRecursively();
+ assertTrue(Files.exists(dir2), "Deleting " + link1 + " recursively does not remove " + dir2);
+ assertTrue(Files.exists(file1), "Deleting " + link1 + " recursively does not remove " + file1);
+
+ new UnixPath(dir1).deleteRecursively();
+ assertFalse(Files.exists(file1), dir1 + " deleted recursively");
+ assertFalse(Files.exists(dir2), dir1 + " deleted recursively");
+ assertFalse(Files.exists(dir1), dir1 + " deleted recursively");
+ }
+
+ @Test
+ void isEmptyDirectory() {
+ var path = new UnixPath((fs.getPath("/foo")));
+ assertFalse(path.isEmptyDirectory());
+
+ path.writeUtf8File("");
+ assertFalse(path.isEmptyDirectory());
+
+ path.deleteIfExists();
+ path.createDirectory();
+ assertTrue(path.isEmptyDirectory());
+
+ path.resolve("bar").writeUtf8File("");
+ assertFalse(path.isEmptyDirectory());
+ }
+
+ @Test
+ void atomicWrite() {
+ var path = new UnixPath(fs.getPath("/dir/foo"));
+ path.createParents();
+ path.writeUtf8File("bar");
+ path.atomicWriteBytes("bar v2".getBytes(StandardCharsets.UTF_8));
+ assertEquals("bar v2", path.readUtf8File());
+ }
+
+ @Test
+ void testParentAndFilename() {
+ var absolutePath = new UnixPath("/foo/bar");
+ assertEquals("/foo", absolutePath.getParent().toString());
+ assertEquals("bar", absolutePath.getFilename());
+
+ var pathWithoutSlash = new UnixPath("foo");
+ assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", pathWithoutSlash::getParent);
+ assertEquals("foo", pathWithoutSlash.getFilename());
+
+ var pathWithSlash = new UnixPath("/foo");
+ assertEquals("/", pathWithSlash.getParent().toString());
+ assertEquals("foo", pathWithSlash.getFilename());
+
+ assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent());
+ assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename());
+ }
+
+ private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) {
+ try {
+ runnable.run();
+ fail("No exception was thrown");
+ } catch (RuntimeException e) {
+ if (!baseClass.isInstance(e)) {
+ fail("Exception class mismatch " + baseClass.getName() + " != " + e.getClass().getName());
+ }
+
+ assertEquals(message, e.getMessage());
+ }
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java
new file mode 100644
index 00000000000..37fe90209ea
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java
@@ -0,0 +1,211 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author freva
+ */
+class ContainerFileSystemTest {
+
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1"));
+ private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000));
+ private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userScope);
+
+ @Test
+ public void creates_files_and_directories_with_container_root_as_owner() throws IOException {
+ ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file"), userScope.root());
+ UnixPath unixPath = new UnixPath(containerPath).createParents().writeUtf8File("hello world");
+
+ for (ContainerPath p = containerPath; p.getParent() != null; p = p.getParent())
+ assertOwnership(p, 0, 0, 10000, 11000);
+
+ unixPath.setOwnerId(500).setGroupId(1000);
+ assertOwnership(containerPath, 500, 1000, 10500, 12000);
+
+ UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile();
+ ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/copy1"), userScope.root());
+ Files.copy(hostFile.toPath(), destination);
+ assertOwnership(destination, 0, 0, 10000, 11000);
+ }
+
+ @Test
+ public void file_write_and_read() throws IOException {
+ ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/file"), userScope.root());
+ UnixPath unixPath = new UnixPath(containerPath);
+ unixPath.writeUtf8File("hello");
+ assertOwnership(containerPath, 0, 0, 10000, 11000);
+
+ unixPath.setOwnerId(500).setGroupId(200);
+ assertOwnership(containerPath, 500, 200, 10500, 11200);
+ Files.writeString(containerPath, " world", StandardOpenOption.APPEND);
+ assertOwnership(containerPath, 500, 200, 10500, 11200); // Owner should not have been updated as the file already existed
+
+ assertEquals("hello world", unixPath.readUtf8File());
+
+ unixPath.deleteIfExists();
+ new UnixPath(containerPath.withUser(userScope.vespa())).writeUtf8File("test123");
+ assertOwnership(containerPath, 1000, 1000, 11000, 12000);
+ }
+
+ @Test
+ public void copy() throws IOException {
+ UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile();
+ ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root());
+
+ // If file is copied to JimFS path, the UID/GIDs are not fixed
+ Files.copy(hostFile.toPath(), destination.pathOnHost());
+ assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName());
+ Files.delete(destination);
+
+ Files.copy(hostFile.toPath(), destination);
+ assertOwnership(destination, 0, 0, 10000, 11000);
+
+ // Set owner + group on both source host file and destination container file
+ hostFile.setOwnerId(5).setGroupId(10);
+ new UnixPath(destination).setOwnerId(500).setGroupId(200);
+ assertOwnership(destination, 500, 200, 10500, 11200);
+ // Copy the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING
+ Files.copy(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
+ // The destination is recreated, so the owner should be root
+ assertOwnership(destination, 0, 0, 10000, 11000);
+
+ // Set owner + group and copy within ContainerFS
+ new UnixPath(destination).setOwnerId(500).setGroupId(200);
+ ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root());
+ Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
+ assertOwnership(destination2, 500, 200, 10500, 11200);
+ }
+
+ @Test
+ public void move() throws IOException {
+ UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile();
+ ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root());
+
+ // If file is moved to JimFS path, the UID/GIDs are not fixed
+ Files.move(hostFile.toPath(), destination.pathOnHost());
+ assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName());
+ Files.delete(destination);
+
+ hostFile.createNewFile();
+ Files.move(hostFile.toPath(), destination);
+ assertOwnership(destination, 0, 0, 10000, 11000);
+
+ // Set owner + group on both source host file and destination container file
+ hostFile.createNewFile();
+ hostFile.setOwnerId(5).setGroupId(10);
+ new UnixPath(destination).setOwnerId(500).setGroupId(200);
+ assertOwnership(destination, 500, 200, 10500, 11200);
+ // Move the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING
+ Files.move(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
+ // The destination is recreated, so the owner should be root
+ assertOwnership(destination, 0, 0, 10000, 11000);
+
+ // Set owner + group and move within ContainerFS
+ new UnixPath(destination).setOwnerId(500).setGroupId(200);
+ ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root());
+ Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
+ assertOwnership(destination2, 500, 200, 10500, 11200);
+ }
+
+ @Test
+ public void symlink() throws IOException {
+ ContainerPath source = ContainerPath.fromPathInContainer(containerFs, Path.of("/src"), userScope.root());
+ // Symlink from ContainerPath to some relative path (different FS provider)
+ Files.createSymbolicLink(source, fileSystem.getPath("../relative/target"));
+ assertEquals(fileSystem.getPath("../relative/target"), Files.readSymbolicLink(source));
+ Files.delete(source);
+
+ // Symlinks from ContainerPath to a ContainerPath: Target is resolved within container with base FS provider
+ Files.createSymbolicLink(source, ContainerPath.fromPathInContainer(containerFs, Path.of("/path/in/container"), userScope.root()));
+ assertEquals(fileSystem.getPath("/path/in/container"), Files.readSymbolicLink(source));
+ assertOwnership(source, 0, 0, 10000, 11000);
+ }
+
+ @Test
+ public void disallow_operations_on_symlink() throws IOException {
+ Path destination = fileSystem.getPath("/dir/file");
+ Files.createDirectories(destination.getParent());
+
+ ContainerPath link = containerFs.getPath("/link");
+ Files.createSymbolicLink(link, destination);
+
+ // Cannot write file via symlink
+ assertThrows(IOException.class, () -> Files.writeString(link, "hello"));
+
+ assertOwnership(link, 0, 0, 10_000, 11_000);
+ Files.setAttribute(link, "unix:uid", 10); // This succeeds because attribute is set on the link (destination does not exist)
+ assertFalse(Files.exists(destination));
+ assertOwnership(link, 10, 0, 10_010, 11_000);
+ }
+
+ @Test
+ public void disallow_operations_on_parent_symlink() throws IOException {
+ Path destination = fileSystem.getPath("/dir/sub/folder");
+ Files.createDirectories(destination.getParent());
+
+ // Create symlink /some/dir/link -> /dir/sub
+ ContainerPath link = containerFs.getPath("/some/dir/link");
+ Files.createDirectories(link.getParent());
+ Files.createSymbolicLink(link, destination.getParent());
+
+ ContainerPath file = link.resolve("file");
+ assertThrows(IOException.class, () -> Files.writeString(file, "hello"));
+ Files.writeString(file.pathOnHost(), "hello"); // Writing through host FS works
+ }
+
+ @Test
+ public void permissions() throws IOException {
+ assertPermissions(Files.createDirectory(containerFs.getPath("/dir1")), "rwxr-x---");
+ assertPermissions(Files.createDirectory(containerFs.getPath("/dir2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-");
+
+ assertPermissions(Files.createDirectories(containerFs.getPath("/sub/dir/leaf"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-");
+ assertPermissions(containerFs.getPath("/sub/dir"), "r-x-w-rw-"); // Non-leafs get the same permission as the leaf
+
+ // TODO: Uncomment when JimFS forwards attributes for SecureDirectoryStream::newByteChannel
+// assertPermissions(Files.createFile(containerFs.getPath("/file1")), "rw-r-----");
+// assertPermissions(Files.createFile(containerFs.getPath("/file2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-");
+ }
+
+ private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException {
+ assertOwnership(path, contUid, contGid);
+ assertOwnership(path.pathOnHost(), hostUid, hostGid);
+ }
+
+ private static void assertOwnership(Path path, int uid, int gid) throws IOException {
+ Map<String, Object> attrs = Files.readAttributes(path, "unix:*", LinkOption.NOFOLLOW_LINKS);
+ assertEquals(uid, attrs.get("uid"));
+ assertEquals(gid, attrs.get("gid"));
+ }
+
+ private static void assertPermissions(Path path, String expected) throws IOException {
+ String actual = PosixFilePermissions.toString(Files.getPosixFilePermissions(path));
+ assertEquals(expected, actual);
+ }
+
+ private static FileAttribute<?> permissionsFromString(String permissions) {
+ return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java
new file mode 100644
index 00000000000..eb7a8e13925
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java
@@ -0,0 +1,120 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author freva
+ */
+class ContainerPathTest {
+
+ private final FileSystem baseFs = TestFileSystem.create();
+ private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserScope.class));
+
+ @Test
+ public void create_new_container_path() {
+ ContainerPath path = fromPathInContainer(Path.of("/opt/vespa//logs/./file"));
+ assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file");
+
+ path = fromPathOnHost(baseFs.getPath("/data/storage/ctr1/opt/vespa/logs/file"));
+ assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file");
+
+ path = fromPathOnHost(baseFs.getPath("/data/storage/ctr2/..////./ctr1/./opt"));
+ assertPaths(path, "/data/storage/ctr1/opt", "/opt");
+
+ assertThrows(() -> fromPathInContainer(Path.of("relative/path")), "Path in container must be absolute: relative/path");
+ assertThrows(() -> fromPathOnHost(baseFs.getPath("relative/path")), "Paths have different roots: /data/storage/ctr1, relative/path");
+ assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1");
+ assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr1/../ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1");
+ }
+
+ @Test
+ public void container_path_operations() {
+ ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs/file"));
+ ContainerPath parent = path.getParent();
+ assertPaths(path.getRoot(), "/data/storage/ctr1", "/");
+ assertPaths(parent, "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs");
+ assertNull(path.getRoot().getParent());
+
+ assertEquals(Path.of("file"), path.getFileName());
+ assertEquals(Path.of("logs"), path.getName(2));
+ assertEquals(4, path.getNameCount());
+ assertEquals(Path.of("vespa/logs"), path.subpath(1, 3));
+
+ assertTrue(path.startsWith(path));
+ assertTrue(path.startsWith(parent));
+ assertFalse(parent.startsWith(path));
+ assertFalse(path.startsWith(Path.of(path.toString())));
+
+ assertTrue(path.endsWith(Path.of(path.pathInContainer())));
+ assertTrue(path.endsWith(Path.of("logs/file")));
+ assertFalse(path.endsWith(Path.of("/logs/file")));
+ }
+
+ @Test
+ public void resolution() {
+ ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs"));
+ assertPaths(path.resolve(Path.of("/root")), "/data/storage/ctr1/root", "/root");
+ assertPaths(path.resolve(Path.of("relative")), "/data/storage/ctr1/opt/vespa/logs/relative", "/opt/vespa/logs/relative");
+ assertPaths(path.resolve(Path.of("/../../../dir2/../../../dir2")), "/data/storage/ctr1/dir2", "/dir2");
+ assertPaths(path.resolve(Path.of("/some/././///path")), "/data/storage/ctr1/some/path", "/some/path");
+
+ assertPaths(path.resolve(Path.of("../dir")), "/data/storage/ctr1/opt/vespa/dir", "/opt/vespa/dir");
+ assertEquals(path.resolve(Path.of("../dir")), path.resolveSibling("dir"));
+ }
+
+ @Test
+ public void resolves_real_paths() throws IOException {
+ ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs"));
+ Files.createDirectories(path.pathOnHost().getParent());
+
+ Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/vespa/target1"));
+ Files.createSymbolicLink(path.pathOnHost(), path.pathOnHost().resolveSibling("target1"));
+ assertPaths(path.toRealPath(LinkOption.NOFOLLOW_LINKS), "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs");
+ assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/vespa/target1", "/opt/vespa/target1");
+
+ Files.delete(path.pathOnHost());
+ Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/target2"));
+ Files.createSymbolicLink(path.pathOnHost(), baseFs.getPath("../target2"));
+ assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/target2", "/opt/target2");
+
+ Files.delete(path.pathOnHost());
+ Files.createFile(baseFs.getPath("/data/storage/ctr2"));
+ Files.createSymbolicLink(path.pathOnHost(), path.getRoot().pathOnHost().resolveSibling("ctr2"));
+ assertThrows(path::toRealPath, "Path /data/storage/ctr2 is not under container root /data/storage/ctr1");
+ }
+
+ private ContainerPath fromPathInContainer(Path pathInContainer) {
+ return ContainerPath.fromPathInContainer(containerFs, pathInContainer, UnixUser.ROOT);
+ }
+ private ContainerPath fromPathOnHost(Path pathOnHost) {
+ return ContainerPath.fromPathOnHost(containerFs, pathOnHost, UnixUser.ROOT);
+ }
+
+ private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) {
+ assertEquals(expectedPathOnHost, actual.pathOnHost().toString());
+ assertEquals(expectedPathInContainer, actual.pathInContainer());
+ }
+
+ private static void assertThrows(Executable executable, String expectedMsg) {
+ String actualMsg = Assertions.assertThrows(IllegalArgumentException.class, executable).getMessage();
+ assertEquals(expectedMsg, actualMsg);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java
new file mode 100644
index 00000000000..525c6d9162c
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java
@@ -0,0 +1,41 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.fs;
+
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.attribute.UserPrincipalNotFoundException;
+
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal;
+import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author freva
+ */
+class ContainerUserPrincipalLookupServiceTest {
+
+ private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000));
+ private final ContainerUserPrincipalLookupService userPrincipalLookupService =
+ new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userScope);
+
+ @Test
+ public void correctly_resolves_ids() throws IOException {
+ ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000");
+ assertEquals("vespa", user.getName());
+ assertEquals("11000", user.baseFsPrincipal().getName());
+ assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa"));
+
+ ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000");
+ assertEquals("vespa", group.getName());
+ assertEquals("12000", group.baseFsPrincipal().getName());
+ assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa"));
+
+ assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test"));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java
new file mode 100644
index 00000000000..299d3e4b441
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java
@@ -0,0 +1,32 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.google.common.net.InetAddresses;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesMock implements IPAddresses {
+
+ private final Map<String, List<InetAddress>> otherAddresses = new HashMap<>();
+
+ public IPAddressesMock addAddress(String hostname, String ip) {
+ List<InetAddress> addresses = otherAddresses.getOrDefault(hostname, new ArrayList<>());
+ addresses.add(InetAddresses.forString(ip));
+ otherAddresses.put(hostname, addresses);
+ return this;
+ }
+
+ @Override
+ public InetAddress[] getAddresses(String hostname) {
+ List<InetAddress> addresses = otherAddresses.get(hostname);
+ if (addresses == null) return new InetAddress[0];
+ return addresses.toArray(new InetAddress[addresses.size()]);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java
new file mode 100644
index 00000000000..59ddc1f6c8d
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java
@@ -0,0 +1,78 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.google.common.net.InetAddresses;
+import org.junit.jupiter.api.Test;
+
+import java.net.Inet6Address;
+import java.net.InetAddress;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesTest {
+
+ private final IPAddressesMock mock = new IPAddressesMock();
+
+ @Test
+ void choose_sitelocal_ipv4_over_public() {
+ mock.addAddress("localhost", "38.3.4.2")
+ .addAddress("localhost", "10.0.2.2")
+ .addAddress("localhost", "fe80::1")
+ .addAddress("localhost", "2001::1");
+
+ assertEquals(InetAddresses.forString("10.0.2.2"), mock.getIPv4Address("localhost").get());
+ }
+
+ @Test
+ void choose_ipv6_public_over_local() {
+ mock.addAddress("localhost", "38.3.4.2")
+ .addAddress("localhost", "10.0.2.2")
+ .addAddress("localhost", "fe80::1")
+ .addAddress("localhost", "2001::1");
+
+ assertEquals(InetAddresses.forString("2001::1"), mock.getIPv6Address("localhost").get());
+ }
+
+ @Test
+ void throws_when_multiple_ipv6_addresses() {
+ assertThrows(RuntimeException.class, () -> {
+ mock.addAddress("localhost", "2001::1")
+ .addAddress("localhost", "2001::2");
+ mock.getIPv6Address("localhost");
+ });
+ }
+
+ @Test
+ void throws_when_multiple_private_ipv4_addresses() {
+ assertThrows(RuntimeException.class, () -> {
+ mock.addAddress("localhost", "38.3.4.2")
+ .addAddress("localhost", "10.0.2.2")
+ .addAddress("localhost", "10.0.2.3");
+ mock.getIPv4Address("localhost");
+ });
+ }
+
+ @Test
+ void translator_with_valid_parameters() {
+
+ // Test simplest possible address
+ Inet6Address original = (Inet6Address) InetAddresses.forString("2001:db8::1");
+ Inet6Address prefix = (Inet6Address) InetAddresses.forString("fd00::");
+ InetAddress translated = IPAddresses.prefixTranslate(original, prefix, 8);
+ assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress());
+
+
+ // Test an actual aws address we use
+ original = (Inet6Address) InetAddresses.forString("2600:1f16:f34:5300:ccc6:1703:b7c2:369d");
+ translated = IPAddresses.prefixTranslate(original, prefix, 8);
+ assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress());
+
+ // Test different subnet size
+ translated = IPAddresses.prefixTranslate(original, prefix, 6);
+ assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress());
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java
new file mode 100644
index 00000000000..69d5c6f2c31
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java
@@ -0,0 +1,69 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+/**
+ * @author gjoranv
+ */
+public class VersionedIpAddressTest {
+
+ @Test
+ void ip4_address_can_be_generated_from_string() {
+ var ip4 = VersionedIpAddress.from("10.0.0.1");
+ assertEquals(IPVersion.IPv4, ip4.version());
+ assertEquals("10.0.0.1", ip4.asString());
+ }
+
+ @Test
+ void ip6_address_can_be_generated_from_string() {
+ var ip6 = VersionedIpAddress.from("::1");
+ assertEquals(IPVersion.IPv6, ip6.version());
+ assertEquals("::1", ip6.asString());
+ }
+
+ @Test
+ void they_are_sorted_by_version_then_by_address() {
+ var ip4 = VersionedIpAddress.from("10.0.0.1");
+ var ip4_2 = VersionedIpAddress.from("127.0.0.1");
+ var ip6 = VersionedIpAddress.from("::1");
+ var ip6_2 = VersionedIpAddress.from("::2");
+
+ var sorted = Stream.of(ip4_2, ip6, ip4, ip6_2)
+ .sorted()
+ .toList();
+ assertEquals(List.of(ip6, ip6_2, ip4, ip4_2), sorted);
+ }
+
+ @Test
+ void endpoint_with_port_is_generated_correctly_for_both_versions() {
+ var ip4 = VersionedIpAddress.from("10.0.0.1");
+ var ip6 = VersionedIpAddress.from("::1");
+
+ assertEquals("10.0.0.1:8080", ip4.asEndpoint(8080));
+ assertEquals("[::1]:8080", ip6.asEndpoint(8080));
+ }
+
+ @Test
+ void equals_and_hashCode_are_implemented() {
+ var one = VersionedIpAddress.from("::1");
+ var two = VersionedIpAddress.from("::2");
+ var local = VersionedIpAddress.from("127.0.0.1");
+ var ten = VersionedIpAddress.from("10.0.0.1");
+ assertEquals(one, VersionedIpAddress.from("::1"));
+ assertNotEquals(one, two);
+ assertNotEquals(one, local);
+ assertNotEquals(one, ten);
+
+ assertEquals(local, VersionedIpAddress.from("127.0.0.1"));
+ assertNotEquals(local, two);
+ assertNotEquals(local, 10);
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
new file mode 100644
index 00000000000..19bc2d59bb2
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java
@@ -0,0 +1,147 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.jdisc.Timer;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class ChildProcess2ImplTest {
+ private final FileSystem fileSystem = TestFileSystem.create();
+ private final Timer timer = mock(Timer.class);
+ private final CommandLine commandLine = mock(CommandLine.class);
+ private final ProcessApi2 processApi = mock(ProcessApi2.class);
+ private Path temporaryFile;
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", "");
+ }
+
+ @Test
+ void testSuccess() throws Exception {
+ when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1));
+ when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+ when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+ when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+ when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+ when(commandLine.toString()).thenReturn("program arg");
+
+ when(timer.currentTime()).thenReturn(
+ Instant.ofEpochMilli(1),
+ Instant.ofEpochMilli(2));
+
+ when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+ try (ChildProcess2Impl child =
+ new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+ child.waitForTermination();
+ }
+ }
+
+ @Test
+ void testTimeout() throws Exception {
+ when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+ when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+ when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+ when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+ when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+ when(commandLine.toString()).thenReturn("program arg");
+
+ when(timer.currentTime()).thenReturn(
+ Instant.ofEpochSecond(0),
+ Instant.ofEpochSecond(2));
+
+ when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+ try (ChildProcess2Impl child =
+ new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+ try {
+ child.waitForTermination();
+ fail();
+ } catch (TimeoutChildProcessException e) {
+ assertEquals(
+ "Command 'program arg' timed out after PT1S: stdout/stderr: ''",
+ e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ void testMaxOutputBytes() throws Exception {
+ when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+ when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+ when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+ when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+ when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+ when(commandLine.toString()).thenReturn("program arg");
+
+ when(timer.currentTime()).thenReturn(
+ Instant.ofEpochMilli(0),
+ Instant.ofEpochMilli(1));
+
+ when(processApi.waitFor(anyLong(), any())).thenReturn(true);
+
+ Files.writeString(temporaryFile, "1234567890123");
+
+ try (ChildProcess2Impl child =
+ new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+ try {
+ child.waitForTermination();
+ fail();
+ } catch (LargeOutputChildProcessException e) {
+ assertEquals(
+ "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'",
+ e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ void testUnkillable() throws Exception {
+ when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1));
+ when(commandLine.getMaxOutputBytes()).thenReturn(10L);
+ when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8);
+ when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2));
+ when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3));
+ when(commandLine.toString()).thenReturn("program arg");
+
+ when(timer.currentTime()).thenReturn(
+ Instant.ofEpochMilli(0),
+ Instant.ofEpochMilli(1));
+
+ when(processApi.waitFor(anyLong(), any())).thenReturn(false);
+
+ Files.writeString(temporaryFile, "1234567890123");
+
+ try (ChildProcess2Impl child =
+ new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) {
+ try {
+ child.waitForTermination();
+ fail();
+ } catch (UnkillableChildProcessException e) {
+ assertEquals(
+ "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'",
+ e.getMessage());
+ }
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
new file mode 100644
index 00000000000..fead96404a5
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
@@ -0,0 +1,190 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class CommandLineTest {
+ private final TestTerminal terminal = new TestTerminal();
+ private final TestTaskContext context = new TestTaskContext();
+ private final CommandLine commandLine = terminal.newCommandLine(context);
+
+ @AfterEach
+ public void tearDown() {
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ @Test
+ void testStrings() {
+ terminal.expectCommand(
+ "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1",
+ 0,
+ "");
+ commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute();
+ assertEquals("bash", commandLine.programName());
+ }
+
+ @Test
+ void testBasicExecute() {
+ terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n");
+ CommandResult result = commandLine.add("foo", "bar").execute();
+ assertEquals(0, result.getExitCode());
+ assertEquals("line1\nline2", result.getOutput());
+ assertEquals("line1\nline2\n\n", result.getUntrimmedOutput());
+ assertEquals(List.of("line1", "line2"), result.getOutputLines());
+ assertEquals(1, context.getSystemModificationLog().size());
+ assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0));
+
+ List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines();
+ assertEquals(1, commandLines.size());
+ assertEquals(commandLine, commandLines.get(0));
+
+ int lines = result.map(r -> r.getOutputLines().size());
+ assertEquals(2, lines);
+ }
+
+ @Test
+ void verifyDefaults() {
+ assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout());
+ assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes());
+ assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod());
+ assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod());
+ assertEquals(0, commandLine.getArguments().size());
+ assertEquals(Optional.empty(), commandLine.getOutputFile());
+ assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding());
+ assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard());
+ Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate();
+ assertTrue(defaultExitCodePredicate.test(0));
+ assertFalse(defaultExitCodePredicate.test(1));
+ }
+
+ @Test
+ void executeSilently() {
+ terminal.ignoreCommand("");
+ commandLine.add("foo", "bar").executeSilently();
+ assertEquals(0, context.getSystemModificationLog().size());
+ commandLine.recordSilentExecutionAsSystemModification();
+ assertEquals(1, context.getSystemModificationLog().size());
+ assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0));
+ }
+
+ @Test
+ void processFactorySpawnFails() {
+ assertThrows(NegativeArraySizeException.class, () -> {
+ terminal.interceptCommand(
+ commandLine.toString(),
+ command -> {
+ throw new NegativeArraySizeException();
+ });
+ commandLine.add("foo").execute();
+ });
+ }
+
+ @Test
+ void waitingForTerminationExceptionStillClosesChild() {
+ TestChildProcess2 child = new TestChildProcess2(0, "");
+ child.throwInWaitForTermination(new NegativeArraySizeException());
+ terminal.interceptCommand(commandLine.toString(), command -> child);
+ assertFalse(child.closeCalled());
+ try {
+ commandLine.add("foo").execute();
+ fail();
+ } catch (NegativeArraySizeException e) {
+ // OK
+ }
+
+ assertTrue(child.closeCalled());
+ }
+
+ @Test
+ void programFails() {
+ terminal.expectCommand("foo 2>&1", 1, "");
+ try {
+ commandLine.add("foo").execute();
+ fail();
+ } catch (ChildProcessFailureException e) {
+ assertEquals(
+ "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ void mapException() {
+ terminal.ignoreCommand("output");
+ CommandResult result = terminal.newCommandLine(context).add("program").execute();
+ IllegalArgumentException exception = new IllegalArgumentException("foo");
+ try {
+ result.mapOutput(output -> {
+ throw exception;
+ });
+ fail();
+ } catch (UnexpectedOutputException e) {
+ assertEquals("Command 'program 2>&1' output was not of the expected format: " +
+ "Failed to map output: stdout/stderr: 'output'", e.getMessage());
+ assertEquals(e.getCause(), exception);
+ }
+ }
+
+ @Test
+ void testMapEachLine() {
+ assertEquals(
+ 1 + 2 + 3,
+ terminal.ignoreCommand("1\n2\n3\n")
+ .newCommandLine(context)
+ .add("foo")
+ .execute()
+ .mapEachLine(Integer::valueOf)
+ .stream()
+ .mapToInt(i -> i)
+ .sum());
+ }
+
+ @Test
+ void addTokensWithMultipleWhiteSpaces() {
+ terminal.expectCommand("iptables -L 2>&1");
+ commandLine.addTokens("iptables -L").execute();
+
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ @Test
+ void addTokensWithSpecialCharacters() {
+ terminal.expectCommand("find . ! -name hei 2>&1");
+ commandLine.addTokens("find . ! -name hei").execute();
+
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ @Test
+ void testEnvironment() {
+ terminal.expectCommand("env k1=v1 -u k2 \"key 3=value 3\" programname 2>&1");
+ commandLine.add("programname")
+ .setEnvironmentVariable("key 3", "value 3")
+ .removeEnvironmentVariable("k2")
+ .setEnvironmentVariable("k1", "v1")
+ .execute();
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ @Test
+ public void testToString() {
+ commandLine.add("bash", "-c", "echo", "$MY_SECRET");
+ assertEquals("bash -c echo \"$MY_SECRET\" 2>&1", commandLine.toString());
+ commandLine.censorArgument();
+ assertEquals("bash -c echo <censored> 2>&1", commandLine.toString());
+
+ terminal.expectCommand("bash -c echo \"$MY_SECRET\" 2>&1");
+ commandLine.execute();
+ terminal.verifyAllCommandsExecuted();
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
new file mode 100644
index 00000000000..58429f9f084
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
@@ -0,0 +1,88 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.process;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.jdisc.test.TestTimer;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ProcessFactoryImplTest {
+ private final ProcessStarter starter = mock(ProcessStarter.class);
+ private final TestTimer timer = new TestTimer();
+ private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer);
+
+ @Test
+ void testSpawn() {
+ CommandLine commandLine = mock(CommandLine.class);
+ when(commandLine.getArguments()).thenReturn(List.of("program"));
+ when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true);
+ when(commandLine.programName()).thenReturn("program");
+ Path outputPath;
+ try (ChildProcess2Impl child = processFactory.spawn(commandLine)) {
+ outputPath = child.getOutputPath();
+ assertTrue(Files.exists(outputPath));
+ assertEquals("rw-------", new UnixPath(outputPath).getPermissions());
+ ArgumentCaptor<ProcessBuilder> processBuilderCaptor =
+ ArgumentCaptor.forClass(ProcessBuilder.class);
+ verify(starter).start(processBuilderCaptor.capture());
+ ProcessBuilder processBuilder = processBuilderCaptor.getValue();
+ assertTrue(processBuilder.redirectErrorStream());
+ ProcessBuilder.Redirect redirect = processBuilder.redirectOutput();
+ assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type());
+ assertEquals(outputPath.toFile(), redirect.file());
+ }
+
+ assertFalse(Files.exists(outputPath));
+ }
+
+ @Test
+ void testSpawnWithPersistentOutputFile() {
+
+ class TemporaryFile implements AutoCloseable {
+ private final Path path;
+
+ private TemporaryFile() {
+ String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out";
+ FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute(
+ PosixFilePermissions.fromString("rw-------"));
+ path = uncheck(() -> Files.createTempFile(outputFileName, ".out", fileAttribute));
+ }
+
+ @Override
+ public void close() {
+ uncheck(() -> Files.deleteIfExists(path));
+ }
+ }
+
+ try (TemporaryFile outputPath = new TemporaryFile()) {
+ CommandLine commandLine = mock(CommandLine.class);
+ when(commandLine.getArguments()).thenReturn(List.of("program"));
+ when(commandLine.programName()).thenReturn("program");
+ when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path));
+ try (ChildProcess2Impl child = processFactory.spawn(commandLine)) {
+ assertEquals(outputPath.path, child.getOutputPath());
+ assertTrue(Files.exists(outputPath.path));
+ assertEquals("rw-------", new UnixPath(outputPath.path).getPermissions());
+ }
+
+ assertTrue(Files.exists(outputPath.path));
+ }
+
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java
new file mode 100644
index 00000000000..f6a695ea003
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java
@@ -0,0 +1,149 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author hakonhall
+ */
+public class SystemCtlTest {
+
+ private final TaskContext taskContext = mock(TaskContext.class);
+ private final TestTerminal terminal = new TestTerminal();
+
+ @Test
+ void enable() {
+ terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "")
+ .expectCommand("systemctl enable docker 2>&1")
+ .expectCommand("systemctl --quiet is-enabled docker 2>&1");
+
+ SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker");
+ assertTrue(enableDockerService.converge(taskContext));
+ assertFalse(enableDockerService.converge(taskContext), "Already converged");
+ }
+
+ @Test
+ void enableCommandFailure() {
+ terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "")
+ .expectCommand("systemctl enable docker 2>&1", 1, "error enabling service");
+ SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker");
+ try {
+ enableDockerService.converge(taskContext);
+ fail();
+ } catch (ChildProcessFailureException e) {
+ // success
+ }
+ }
+
+
+ @Test
+ void start() {
+ terminal.expectCommand(
+ "systemctl show docker 2>&1",
+ 0,
+ "a=b\n" +
+ "ActiveState=failed\n" +
+ "bar=zoo\n")
+ .expectCommand("systemctl start docker 2>&1", 0, "");
+
+ SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker");
+ assertTrue(startDockerService.converge(taskContext));
+ }
+
+ @Test
+ void startIsNoop() {
+ terminal.expectCommand(
+ "systemctl show docker 2>&1",
+ 0,
+ "a=b\n" +
+ "ActiveState=active\n" +
+ "bar=zoo\n")
+ .expectCommand("systemctl start docker 2>&1", 0, "");
+
+ SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker");
+ assertFalse(startDockerService.converge(taskContext));
+ }
+
+
+ @Test
+ void startCommandFailre() {
+ terminal.expectCommand("systemctl show docker 2>&1", 1, "error");
+ SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker");
+ try {
+ startDockerService.converge(taskContext);
+ fail();
+ } catch (ChildProcessFailureException e) {
+ // success
+ }
+ }
+
+
+ @Test
+ void disable() {
+ terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1")
+ .expectCommand("systemctl disable docker 2>&1")
+ .expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "");
+
+ assertTrue(new SystemCtl(terminal).disable("docker").converge(taskContext));
+ assertFalse(new SystemCtl(terminal).disable("docker").converge(taskContext), "Already converged");
+ }
+
+ @Test
+ void stop() {
+ terminal.expectCommand(
+ "systemctl show docker 2>&1",
+ 0,
+ "a=b\n" +
+ "ActiveState=active\n" +
+ "bar=zoo\n")
+ .expectCommand("systemctl stop docker 2>&1", 0, "");
+
+ assertTrue(new SystemCtl(terminal).stop("docker").converge(taskContext));
+ }
+
+ @Test
+ void restart() {
+ terminal.expectCommand("systemctl restart docker 2>&1", 0, "");
+ assertTrue(new SystemCtl(terminal).restart("docker").converge(taskContext));
+ }
+
+ @Test
+ void testUnitExists() {
+ SystemCtl systemCtl = new SystemCtl(terminal);
+
+ terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0,
+ "UNIT FILE STATE\n" +
+ "\n" +
+ "0 unit files listed.\n");
+ assertFalse(systemCtl.serviceExists(taskContext, "foo"));
+
+ terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0,
+ "UNIT FILE STATE \n" +
+ "foo.service enabled\n" +
+ "\n" +
+ "1 unit files listed.\n");
+ assertTrue(systemCtl.serviceExists(taskContext, "foo"));
+
+ terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, "garbage");
+ try {
+ systemCtl.serviceExists(taskContext, "foo");
+ fail();
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("garbage"));
+ }
+ }
+
+ @Test
+ void withSudo() {
+ SystemCtl systemCtl = new SystemCtl(terminal).withSudo();
+ terminal.expectCommand("sudo systemctl restart docker 2>&1", 0, "");
+ assertTrue(systemCtl.restart("docker").converge(taskContext));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java
new file mode 100644
index 00000000000..3fc10a38a99
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java
@@ -0,0 +1,52 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class SystemCtlTesterTest {
+
+ private static final String unit = "my-unit";
+ private final TestTerminal terminal = new TestTerminal();
+ private final SystemCtlTester systemCtl = new SystemCtlTester(terminal);
+ private final TestTaskContext context = new TestTaskContext();
+
+ @Test
+ void return_expectations() {
+ assertSystemCtlMethod(sct -> sct.expectEnable(unit), sc -> sc.enable(unit).converge(context));
+ assertSystemCtlMethod(sct -> sct.expectDisable(unit), sc -> sc.disable(unit).converge(context));
+ assertSystemCtlMethod(sct -> sct.expectStart(unit), sc -> sc.start(unit).converge(context));
+ assertSystemCtlMethod(sct -> sct.expectStop(unit), sc -> sc.stop(unit).converge(context));
+ assertSystemCtlMethod(sct -> sct.expectServiceExists(unit), sc -> sc.serviceExists(context, unit));
+ assertSystemCtlMethod(sct -> sct.expectIsActive(unit), sc -> sc.isActive(context, unit));
+ }
+
+ @Test
+ void void_tests() {
+ systemCtl.expectRestart(unit);
+ systemCtl.restart(unit).converge(context);
+ terminal.verifyAllCommandsExecuted();
+
+ systemCtl.expectDaemonReload();
+ systemCtl.daemonReload(context);
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ private void assertSystemCtlMethod(Function<SystemCtlTester, SystemCtlTester.Expectation> systemCtlTesterExpectationFunction,
+ Function<SystemCtl, Boolean> systemCtlFunction) {
+ List.of(true, false).forEach(wantedReturnValue -> {
+ systemCtlTesterExpectationFunction.apply(systemCtl).andReturn(wantedReturnValue);
+ assertEquals(wantedReturnValue, systemCtlFunction.apply(systemCtl));
+ terminal.verifyAllCommandsExecuted();
+ });
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java
new file mode 100644
index 00000000000..1e2f69d7bc8
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java
@@ -0,0 +1,218 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.template;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author hakonhall
+ */
+public class TemplateTest {
+ @Test
+ void verifyNewlineRemoval() {
+ Template template = Template.from("a%{list a}\n" +
+ "b%{end}\n" +
+ "c%{list c-}\n" +
+ "d%{end-}\n" +
+ "e\n",
+ new TemplateDescriptor().setRemoveNewline(false));
+ template.add("a");
+ template.add("c");
+
+ assertEquals("a\n" +
+ "b\n" +
+ "cde\n",
+ template.render());
+ }
+
+ @Test
+ void verifyIfSection() {
+ Template template = Template.from("Hello%{if cond} world%{end}!");
+ assertEquals("Hello world!", template.snapshot().set("cond", true).render());
+ assertEquals("Hello!", template.snapshot().set("cond", false).render());
+ }
+
+ @Test
+ void verifyComplexIfSection() {
+ Template template = Template.from("%{if cond}\n" +
+ "var: %{=varname}\n" +
+ "if: %{if !inner}inner is false%{end-}\n" +
+ "list: %{list formname}element%{end-}\n" +
+ "%{end}\n");
+
+ assertEquals("", template.snapshot().set("cond", false).render());
+
+ assertEquals("var: varvalue\n" +
+ "if: \n" +
+ "list: \n",
+ template.snapshot()
+ .set("cond", true)
+ .set("varname", "varvalue")
+ .set("inner", true)
+ .render());
+
+ Template template2 = template.snapshot()
+ .set("cond", true)
+ .set("varname", "varvalue")
+ .set("inner", false);
+ template2.add("formname");
+
+ assertEquals("var: varvalue\n" +
+ "if: inner is false\n" +
+ "list: element\n", template2.render());
+ }
+
+ @Test
+ void verifyElse() {
+ var template = Template.from("%{if cond}\n" +
+ "if body\n" +
+ "%{else}\n" +
+ "else body\n" +
+ "%{end}\n");
+ assertEquals("if body\n", template.snapshot().set("cond", true).render());
+ assertEquals("else body\n", template.snapshot().set("cond", false).render());
+ }
+
+ @Test
+ void verifySnapshotPreservesList() {
+ var template = Template.from("%{list foo}hello %{=area}%{end}");
+ template.add("foo")
+ .set("area", "world");
+
+ assertEquals("hello world", template.render());
+ assertEquals("hello world", template.snapshot().render());
+
+ Template snapshot = template.snapshot();
+ snapshot.add("foo")
+ .set("area", "Norway");
+ assertEquals("hello worldhello Norway", snapshot.render());
+ }
+
+ @Test
+ void verifyVariableSection() {
+ Template template = getTemplate("template1.tmp");
+ template.set("varname", "varvalue");
+ assertEquals("variable section 'varvalue'\n" +
+ "end of text\n", template.render());
+ }
+
+ @Test
+ void verifySimpleListSection() {
+ Template template = getTemplate("template1.tmp");
+ template.set("varname", "varvalue")
+ .add("listname")
+ .set("varname", "different varvalue")
+ .set("varname2", "varvalue2");
+ assertEquals("variable section 'varvalue'\n" +
+ "same variable section 'different varvalue'\n" +
+ "different variable section 'varvalue2'\n" +
+ "between ends\n" +
+ "end of text\n", template.render());
+ }
+
+ @Test
+ void verifyNestedListSection() {
+ Template template = getTemplate("template2.tmp");
+ ListElement A0 = template.add("listA");
+ ListElement A0B0 = A0.add("listB");
+ ListElement A0B1 = A0.add("listB");
+
+ ListElement A1 = template.add("listA");
+ ListElement A1B0 = A1.add("listB");
+ assertEquals("body A\n" +
+ "body B\n" +
+ "body B\n" +
+ "body A\n" +
+ "body B\n",
+ template.render());
+ }
+
+ @Test
+ void verifyVariableReferences() {
+ Template template = getTemplate("template3.tmp");
+ template.set("varname", "varvalue")
+ .set("innerVarSetAtTop", "val2");
+ template.add("l");
+ template.add("l")
+ .set("varname", "varvalue2");
+ assertEquals("varvalue\n" +
+ "varvalue\n" +
+ "inner varvalue\n" +
+ "val2\n" +
+ "inner varvalue2\n" +
+ "val2\n",
+ template.render());
+ }
+
+ @Test
+ void badTemplates() {
+ assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6",
+ () -> Template.from("foo\nbar%{zoo}"));
+
+ assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4",
+ () -> Template.from("%{="));
+
+ assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4",
+ () -> Template.from("%{=&notatoken}"));
+
+ assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8",
+ () -> Template.from("%{list &notatoken}"));
+
+ assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12",
+ () -> Template.from("%{list foo}missing end"));
+
+ assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3",
+ () -> Template.from("%{end}stray end"));
+
+ assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset",
+ () -> Template.from("%{=notset}").render());
+
+ assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond",
+ () -> Template.from("%{if cond}%{end}").render());
+
+ assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false",
+ () -> Template.from("%{if cond}%{end}").set("cond", 1).render());
+
+ assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " +
+ "line 1 and column 1, and ending at line 1 and column 4",
+ () -> Template.from("foo").add("listname"));
+
+ assertException(NameAlreadyExistsTemplateException.class,
+ "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " +
+ "named list section at line 1 and column 1",
+ () -> Template.from("%{list a}%{end}%{list a}%{end}"));
+
+ assertException(NameAlreadyExistsTemplateException.class,
+ "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " +
+ "named variable section at line 1 and column 1",
+ () -> Template.from("%{=a}%{list a}%{end}"));
+
+ assertException(NameAlreadyExistsTemplateException.class,
+ "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " +
+ "named list section at line 1 and column 1",
+ () -> Template.from("%{list a}%{end}%{=a}"));
+
+ assertException(NameAlreadyExistsTemplateException.class,
+ "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " +
+ "named if section at line 1 and column 1",
+ () -> Template.from("%{if a}%{end}%{list a}%{end}"));
+
+ assertException(NameAlreadyExistsTemplateException.class,
+ "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " +
+ "named list section at line 1 and column 1",
+ () -> Template.from("%{list a}%{end}%{if a}%{end}"));
+ }
+
+ private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) {
+ T exception = assertThrows(class_, runnable::run);
+ assertEquals(message, exception.getMessage());
+ }
+
+ private Template getTemplate(String filename) {
+ return Template.at(Path.of("src/test/resources/" + filename));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
new file mode 100644
index 00000000000..505cf807116
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
@@ -0,0 +1,194 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author hakonhall
+ */
+public class YumPackageNameTest {
+
+ @Test
+ void testBuilder() {
+ YumPackageName yumPackage = new YumPackageName.Builder("docker")
+ .setEpoch("2")
+ .setVersion("1.12.6")
+ .setRelease("71.git3e8e77d.el7.centos.1")
+ .setArchitecture("x86_64")
+ .build();
+ assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName());
+ }
+
+ @Test
+ void testAllValidFormats() {
+ // name
+ verifyPackageName(
+ "docker-engine-selinux",
+ null,
+ "docker-engine-selinux",
+ null,
+ null,
+ null,
+ "docker-engine-selinux",
+ null);
+
+ // name with parenthesis
+ verifyPackageName(
+ "dnf-command(versionlock)",
+ null,
+ "dnf-command(versionlock)",
+ null,
+ null,
+ null,
+ "dnf-command(versionlock)",
+ null);
+
+ // name.arch
+ verifyPackageName(
+ "docker-engine-selinux.x86_64",
+ null,
+ "docker-engine-selinux",
+ null,
+ null,
+ "x86_64",
+ "docker-engine-selinux.x86_64",
+ null);
+
+ // name-ver
+ verifyPackageName("docker-engine-selinux-1.12.6",
+ null,
+ "docker-engine-selinux",
+ "1.12.6",
+ null,
+ null,
+ "docker-engine-selinux-0:1.12.6",
+ null);
+
+ // name-ver-rel
+ verifyPackageName("docker-engine-selinux-1.12.6-1.el7",
+ null,
+ "docker-engine-selinux",
+ "1.12.6",
+ "1.el7",
+ null,
+ "docker-engine-selinux-0:1.12.6-1.el7",
+ "docker-engine-selinux-0:1.12.6-1.el7.*");
+
+ // name-ver-rel.arch
+ verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64",
+ null,
+ "docker-engine-selinux",
+ "1.12.6",
+ "1.el7",
+ "x86_64",
+ "docker-engine-selinux-0:1.12.6-1.el7.x86_64",
+ "docker-engine-selinux-0:1.12.6-1.el7.*");
+
+ // name-epoch:ver-rel.arch
+ verifyPackageName(
+ "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
+ "2",
+ "docker",
+ "1.12.6",
+ "71.git3e8e77d.el7.centos.1",
+ "x86_64",
+ "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
+ "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*");
+
+ // epoch:name-ver-rel.arch
+ verifyPackageName(
+ "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
+ "2",
+ "docker",
+ "1.12.6",
+ "71.git3e8e77d.el7.centos.1",
+ "x86_64",
+ "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
+ "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*");
+ }
+
+ private void verifyPackageName(String input,
+ String epoch,
+ String name,
+ String version,
+ String release,
+ String architecture,
+ String toName,
+ String toVersionName) {
+ YumPackageName yumPackageName = YumPackageName.fromString(input);
+ assertPackageField("epoch", epoch, yumPackageName.getEpoch());
+ assertPackageField("name", name, Optional.of(yumPackageName.getName()));
+ assertPackageField("version", version, yumPackageName.getVersion());
+ assertPackageField("release", release, yumPackageName.getRelease());
+ assertPackageField("architecture", architecture, yumPackageName.getArchitecture());
+ assertPackageField("toName()", toName, Optional.of(yumPackageName.toName()));
+
+ if (toVersionName == null) {
+ try {
+ yumPackageName.toVersionLockName();
+ fail();
+ } catch (IllegalStateException e) {
+ assertTrue(e.getMessage().contains("Version is missing ") ||
+ e.getMessage().contains("Release is missing "),
+ "Exception message contains expected substring: " + e.getMessage());
+ }
+ } else {
+ assertEquals(toVersionName, yumPackageName.toVersionLockName());
+ }
+ }
+
+ private void assertPackageField(String field, String expected, Optional<String> actual) {
+ if (expected == null) {
+ assertFalse(actual.isPresent(), field + " is not present");
+ } else {
+ assertEquals(expected, actual.get(), field + " has expected value");
+ }
+ }
+
+ @Test
+ void testArchitectures() {
+ assertEquals("x86_64", YumPackageName.fromString("docker.x86_64").getArchitecture().get());
+ assertEquals("i686", YumPackageName.fromString("docker.i686").getArchitecture().get());
+ assertEquals("noarch", YumPackageName.fromString("docker.noarch").getArchitecture().get());
+ }
+
+ @Test
+ void unrecognizedArchitectureGetsGobbledUp() {
+ YumPackageName packageName = YumPackageName.fromString("docker-engine-selinux-1.12.6-1.el7.i486");
+ // This is not a great feature - please use YumPackageName.Builder instead.
+ assertEquals("1.el7.i486", packageName.getRelease().get());
+ }
+
+ @Test
+ void failParsingOfPackageNameWithEpochAndArchitecture() {
+ try {
+ YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64");
+ fail();
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().toLowerCase().contains("epoch"));
+ }
+ }
+
+ @Test
+ void testSubset() {
+ YumPackageName yumPackage = new YumPackageName.Builder("docker")
+ .setVersion("1.12.6")
+ .build();
+
+ assertTrue(yumPackage.isSubsetOf(yumPackage));
+ assertTrue(yumPackage.isSubsetOf(new YumPackageName.Builder("docker")
+ .setVersion("1.12.6")
+ .setEpoch("2")
+ .setRelease("71.git3e8e77d.el7.centos.1")
+ .setArchitecture("x86_64")
+ .build()));
+ assertFalse(yumPackage.isSubsetOf(new YumPackageName.Builder("docker")
+ .setVersion("1.13.1")
+ .build()));
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
new file mode 100644
index 00000000000..27b23d26b24
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
@@ -0,0 +1,335 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author hakonhall
+ */
+public class YumTest {
+
+ private final TaskContext taskContext = mock(TaskContext.class);
+ private final TestTerminal terminal = new TestTerminal();
+ private final Yum yum = new Yum(terminal);
+
+ @AfterEach
+ public void after() {
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ @Test
+ void testQueryInstalled() {
+ terminal.expectCommand(
+ "rpm -q docker --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1",
+ 0,
+ "docker\n2\n1.13.1\n74.git6e3bb8e.el7.centos\nx86_64");
+
+ Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "docker");
+
+ assertTrue(installed.isPresent());
+ assertEquals("docker", installed.get().getName());
+ assertEquals("2", installed.get().getEpoch().get());
+ assertEquals("1.13.1", installed.get().getVersion().get());
+ assertEquals("74.git6e3bb8e.el7.centos", installed.get().getRelease().get());
+ assertEquals("x86_64", installed.get().getArchitecture().get());
+ }
+
+ @Test
+ void testQueryInstalledPartial() {
+ terminal.expectCommand(
+ "rpm -q vespa-node-admin --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1",
+ 0,
+ "vespa-node-admin\n(none)\n6.283.62\n1.el7\nnoarch");
+
+ Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "vespa-node-admin");
+
+ assertTrue(installed.isPresent());
+ assertEquals("vespa-node-admin", installed.get().getName());
+ assertEquals("0", installed.get().getEpoch().get());
+ assertEquals("6.283.62", installed.get().getVersion().get());
+ assertEquals("1.el7", installed.get().getRelease().get());
+ assertEquals("noarch", installed.get().getArchitecture().get());
+ }
+
+ @Test
+ void testQueryNotInstalled() {
+ terminal.expectCommand(
+ "rpm -q fake-package --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1",
+ 1,
+ "package fake-package is not installed");
+
+ Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "fake-package");
+
+ assertFalse(installed.isPresent());
+ }
+
+ @Test
+ void testQueryInstalledMultiplePackages() {
+ terminal.expectCommand(
+ "rpm -q kernel-devel --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1",
+ 0,
+ "kernel-devel\n" +
+ "(none)\n" +
+ "4.18.0\n" +
+ "305.7.1.el8_4\n" +
+ "x86_64\n" +
+ "kernel-devel\n" +
+ "(none)\n" +
+ "4.18.0\n" +
+ "240.15.1.el8_3\n" +
+ "x86_64\n");
+ try {
+ yum.queryInstalled(taskContext, "kernel-devel");
+ fail("Expected exception");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Found multiple installed packages for 'kernel-devel'. Version is required to match package exactly", e.getMessage());
+ }
+ }
+
+ @Test
+ void testAlreadyInstalled() {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
+ 0,
+ "foobar\nNothing to do.\n"); // Note trailing dot
+ assertFalse(yum.install("package-1", "package-2")
+ .enableRepo("repo1", "repo2")
+ .converge(taskContext));
+ }
+
+ @Test
+ void testAlreadyUpgraded() {
+ terminal.expectCommand(
+ "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1",
+ 0,
+ "foobar\nNothing to do.\n"); // Same message as yum install no-op
+
+ assertFalse(yum.upgrade("package-1", "package-2")
+ .converge(taskContext));
+ }
+
+ @Test
+ void testAlreadyRemoved() {
+ mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1"));
+ terminal.expectCommand(
+ "yum remove --assumeyes package-1 package-2 2>&1",
+ 0,
+ "foobar\nNo packages marked for removal.\n"); // Different output
+
+ assertFalse(yum.remove("package-1", "package-2")
+ .converge(taskContext));
+ }
+
+ @Test
+ void skipsYumRemoveNotInRpm() {
+ mockRpmQuery("package-1", null);
+ mockRpmQuery("package-2", null);
+ assertFalse(yum.remove("package-1", "package-2").converge(taskContext));
+ }
+
+ @Test
+ void testInstall() {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
+ 0,
+ "installing, installing");
+
+ assertTrue(yum
+ .install("package-1", "package-2")
+ .converge(taskContext));
+ }
+
+ @Test
+ void skipsYumInstallIfInRpm() {
+ mockRpmQuery("package-1-0:1.2.3-1", YumPackageName.fromString("package-1-1.2.3-1"));
+ mockRpmQuery("package-2", YumPackageName.fromString("1:package-2-1.2.3-1.el7.x86_64"));
+ assertFalse(yum.install("package-1-1.2.3-1", "package-2").converge(taskContext));
+ }
+
+ @Test
+ void testInstallWithEnablerepo() {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
+ 0,
+ "installing, installing");
+
+ assertTrue(yum
+ .install("package-1", "package-2")
+ .enableRepo("repo-name")
+ .converge(taskContext));
+ }
+
+ @Test
+ void testInstallWithEnablerepoDisablerepo() {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes \"--disablerepo=*\" --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
+ 0,
+ "installing, installing");
+
+ assertTrue(yum
+ .install("package-1", "package-2")
+ .enableRepo("repo-name")
+ .disableRepo("*")
+ .converge(taskContext));
+ }
+
+ @Test
+ void testWithVersionLock() {
+ terminal.expectCommand("yum versionlock list 2>&1",
+ 0,
+ "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n");
+ terminal.expectCommand("yum versionlock add --assumeyes \"openssh-0:8.0p1-4.el8_1.*\" 2>&1");
+ terminal.expectCommand(
+ "yum install --assumeyes openssh-0:8.0p1-4.el8_1.x86_64 2>&1",
+ 0,
+ "installing");
+
+ YumPackageName pkg = new YumPackageName
+ .Builder("openssh")
+ .setVersion("8.0p1")
+ .setRelease("4.el8_1")
+ .setArchitecture("x86_64")
+ .build();
+ assertTrue(yum.installFixedVersion(pkg).converge(taskContext));
+ }
+
+ @Test
+ void testWithDifferentVersionLock() {
+ terminal.expectCommand("yum versionlock list 2>&1",
+ 0,
+ "Repository chef_rpms-release is listed more than once in the configuration\n" +
+ "chef-0:12.21.1-1.el7.*\n" +
+ "package-0:0.1-8.el7.*\n");
+
+ terminal.expectCommand("yum versionlock delete \"package-0:0.1-8.el7.*\" 2>&1");
+
+ terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"package-0:0.10-654.el7.*\" 2>&1");
+
+ terminal.expectCommand(
+ "yum install --assumeyes --enablerepo=somerepo package-0:0.10-654.el7 2>&1",
+ 0,
+ "Nothing to do\n");
+
+
+ assertTrue(yum
+ .installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7"))
+ .enableRepo("somerepo")
+ .converge(taskContext));
+ }
+
+ @Test
+ void testWithExistingVersionLock() {
+ terminal.expectCommand("yum versionlock list 2>&1",
+ 0,
+ "Repository chef_rpms-release is listed more than once in the configuration\n" +
+ "chef-0:12.21.1-1.el7.*\n" +
+ "package-0:0.10-654.el7.*\n");
+ terminal.expectCommand(
+ "yum install --assumeyes package-0:0.10-654.el7 2>&1",
+ 0,
+ "Nothing to do\n");
+
+ assertFalse(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext));
+ }
+
+ @Test
+ void testWithDowngrade() {
+ terminal.expectCommand("yum versionlock list 2>&1",
+ 0,
+ "Repository chef_rpms-release is listed more than once in the configuration\n" +
+ "chef-0:12.21.1-1.el7.*\n" +
+ "package-0:0.10-654.el7.*\n");
+
+ terminal.expectCommand(
+ "yum install --assumeyes package-0:0.10-654.el7 2>&1",
+ 0,
+ "Package matching package-=.0.10-654.el7 already installed. Checking for update.\n" +
+ "Nothing to do\n");
+
+ terminal.expectCommand("yum downgrade --assumeyes package-0:0.10-654.el7 2>&1");
+
+ assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext));
+ }
+
+ @Test
+ void testFailedInstall() {
+ assertThrows(ChildProcessFailureException.class, () -> {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
+ 1,
+ "error");
+
+ yum
+ .install("package-1", "package-2")
+ .enableRepo("repo-name")
+ .converge(taskContext);
+ fail();
+ });
+ }
+
+ @Test
+ void testUnknownPackages() {
+ mockRpmQuery("package-1", null);
+ terminal.expectCommand(
+ "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1",
+ 0,
+ "Loaded plugins: fastestmirror, langpacks\n" +
+ "Loading mirror speeds from cached hostfile\n" +
+ "No package package-1 available.\n" +
+ "No package package-2 available.\n" +
+ "Nothing to do\n");
+
+ var command = yum.install("package-1", "package-2", "package-3");
+ try {
+ command.converge(taskContext);
+ fail();
+ } catch (Exception e) {
+ assertNotNull(e.getCause());
+ assertEquals("Unknown package: package-1", e.getCause().getMessage());
+ }
+ }
+
+ @Test
+ void throwIfNoPackagesSpecified() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ yum.install();
+ });
+ }
+
+ @Test
+ void allowToCallUpgradeWithNoPackages() {
+ terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK");
+ yum.upgrade().converge(taskContext);
+ }
+
+ @Test
+ void testDeleteVersionLock() {
+ terminal.expectCommand("yum versionlock delete openssh-0:8.0p1-4.el8_1.x86_64 2>&1");
+
+ YumPackageName pkg = new YumPackageName
+ .Builder("openssh")
+ .setVersion("8.0p1")
+ .setRelease("4.el8_1")
+ .setArchitecture("x86_64")
+ .build();
+ assertTrue(yum.deleteVersionLock(pkg).converge(taskContext));
+ }
+
+ private void mockRpmQuery(String packageName, YumPackageName installedOrNull) {
+ new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
new file mode 100644
index 00000000000..aafa0fcfd72
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
@@ -0,0 +1,80 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author freva
+ */
+public class YumTesterTest {
+
+ private static final String[] packages = {"pkg1", "pkg2"};
+ private static final String[] repos = {"repo1", "repo2"};
+ private static final String[] disablerepos = {"disablerepo1", "disablerepo2"};
+ private static final YumPackageName minimalPackage = YumPackageName.fromString("pkg-1.13.1-0.el7");
+ private static final YumPackageName fullPackage = YumPackageName.fromString("2:pkg-1.13.1-0.el7.x86_64");
+
+ private final TestTerminal terminal = new TestTerminal();
+ private final YumTester yum = new YumTester(terminal);
+ private final TestTaskContext context = new TestTaskContext();
+
+ @Test
+ void generic_yum_methods() {
+ assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo(disablerepos).withEnableRepo(repos),
+ yum -> yum.install(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context));
+
+ assertYumMethod(yum -> yum.expectUpdate(packages).withDisableRepo(disablerepos).withEnableRepo(repos),
+ yum -> yum.upgrade(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context));
+
+ assertYumMethod(yum -> yum.expectRemove(packages).withDisableRepo(disablerepos).withEnableRepo(repos),
+ yum -> yum.remove(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context));
+
+ assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withDisableRepo(disablerepos).withEnableRepo(repos),
+ yum -> yum.installFixedVersion(minimalPackage).disableRepo(disablerepos).enableRepo(repos).converge(context));
+
+ // versionlock always returns success
+ assertYumMethodAlwaysSuccess(yum -> yum.expectDeleteVersionLock(minimalPackage.toName()),
+ yum -> yum.deleteVersionLock(minimalPackage).converge(context));
+
+ }
+
+ @Test
+ void disable_other_repos() {
+ assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo("*").withEnableRepo(repos),
+ yum -> yum.install(List.of(packages)).disableRepo("*").enableRepo(repos).converge(context));
+ }
+
+ @Test
+ void expect_query_installed() {
+ yum.expectQueryInstalled(packages[0]).andReturn(fullPackage);
+ assertEquals(Optional.of(fullPackage), yum.queryInstalled(context, packages[0]));
+ terminal.verifyAllCommandsExecuted();
+ }
+
+ private void assertYumMethod(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction,
+ Function<Yum, Boolean> yumFunction) {
+ List.of(true, false).forEach(wantedReturnValue -> {
+ yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue);
+ assertEquals(wantedReturnValue, yumFunction.apply(yum));
+ terminal.verifyAllCommandsExecuted();
+ });
+ }
+
+ private void assertYumMethodAlwaysSuccess(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction,
+ Function<Yum, Boolean> yumFunction) {
+ List.of(true, false).forEach(wantedReturnValue -> {
+ yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue);
+ assertEquals(true, yumFunction.apply(yum));
+ terminal.verifyAllCommandsExecuted();
+ });
+ }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java
new file mode 100644
index 00000000000..7ac47aad1fa
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java
@@ -0,0 +1,39 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.wireguard;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.config.provision.WireguardKeyWithTimestamp;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author gjoranv
+ */
+public class WireguardPeerTest {
+
+ @Test
+ void peers_are_sorted_by_hostname_ascending() {
+ List<WireguardPeer> peers = Stream.of(
+ peer("b"),
+ peer("a"),
+ peer("c")
+ ).sorted().toList();
+
+ assertEquals("a", peers.get(0).hostname().value());
+ assertEquals("b", peers.get(1).hostname().value());
+ assertEquals("c", peers.get(2).hostname().value());
+ }
+
+ private static WireguardPeer peer(String hostname) {
+ return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")),
+ new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH));
+ }
+
+}
diff --git a/node-admin/src/test/resources/default-env-example.txt b/node-admin/src/test/resources/default-env-example.txt
new file mode 100644
index 00000000000..debae073271
--- /dev/null
+++ b/node-admin/src/test/resources/default-env-example.txt
@@ -0,0 +1,5 @@
+override VESPA_HOSTNAME myhostname
+fallback VESPA_CONFIGSERVER fallback-configserver
+fallback VESPA_TLS_CONFIG_FILE /fallback/path/to/config.file
+unset VESPA_LEGACY_OPTION
+fallback VESPA_LEGACY_OPTION duplicated-variable \ No newline at end of file
diff --git a/node-admin/src/test/resources/default-env-rewritten.txt b/node-admin/src/test/resources/default-env-rewritten.txt
new file mode 100644
index 00000000000..94a91f4e793
--- /dev/null
+++ b/node-admin/src/test/resources/default-env-rewritten.txt
@@ -0,0 +1,4 @@
+override VESPA_HOSTNAME my-new-hostname
+fallback VESPA_CONFIGSERVER new-fallback-configserver
+override VESPA_TLS_CONFIG_FILE /override/path/to/config.file
+unset VESPA_LEGACY_OPTION
diff --git a/node-admin/src/test/resources/template1.tmp b/node-admin/src/test/resources/template1.tmp
new file mode 100644
index 00000000000..d53c875a0f3
--- /dev/null
+++ b/node-admin/src/test/resources/template1.tmp
@@ -0,0 +1,10 @@
+variable section '%{=varname}'
+%{list listname}
+same variable section '%{=varname}'
+different variable section '%{=varname2}'
+%{list innerlistname}
+inner form text
+%{end}
+between ends
+%{end}
+end of text
diff --git a/node-admin/src/test/resources/template2.tmp b/node-admin/src/test/resources/template2.tmp
new file mode 100644
index 00000000000..d36cb4a4a48
--- /dev/null
+++ b/node-admin/src/test/resources/template2.tmp
@@ -0,0 +1,4 @@
+%{list listA}body A
+%{list listB}body B
+%{end}
+%{end}
diff --git a/node-admin/src/test/resources/template3.tmp b/node-admin/src/test/resources/template3.tmp
new file mode 100644
index 00000000000..27566e72a9d
--- /dev/null
+++ b/node-admin/src/test/resources/template3.tmp
@@ -0,0 +1,6 @@
+%{=varname}
+%{=varname}
+%{list l}
+inner %{=varname}
+%{=innerVarSetAtTop}
+%{end}