From 4a68667c16cd2bd5ef44fd8916457ae9ef85e339 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 22 Dec 2017 22:37:13 +0100 Subject: Revert "Support node admin modes" --- .../com/yahoo/vespa/hosted/dockerapi/Docker.java | 5 - .../yahoo/vespa/hosted/dockerapi/DockerImpl.java | 81 +++++---------- .../vespa/hosted/dockerapi/DockerTestUtils.java | 6 +- .../node/admin/nodeadmin/NodeAdminConfig.java | 37 ------- .../hosted/node/admin/nodeadmin/NodeAdminMain.java | 115 --------------------- .../node/admin/provider/NodeAdminProvider.java | 67 ++++++++++-- .../node/admin/integrationTests/DockerMock.java | 3 - 7 files changed, 86 insertions(+), 228 deletions(-) delete mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java delete mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index bc94c39d135..2bf3f0f8d84 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -12,11 +12,6 @@ import java.util.Optional; * and to avoid OSGi exporting those classes. */ public interface Docker { - /** - * Must be called before any other method. May be called more than once. - */ - void start(); - interface CreateContainerCommand { CreateContainerCommand withLabel(String name, String value); CreateContainerCommand withEnvironment(String name, String value); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index fa093e0b4dc..9de2cae604f 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -59,77 +59,48 @@ public class DockerImpl implements Docker { static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; private final int SECONDS_TO_WAIT_BEFORE_KILLING; - private final boolean fallbackTo123OnErrors; private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; - private final DockerConfig config; - private final boolean inProduction; private Optional dockerImageGC = Optional.empty(); private CounterWrapper numberOfDockerDaemonFails; - private boolean started = false; private final Object monitor = new Object(); @GuardedBy("monitor") private final Set scheduledPulls = new HashSet<>(); // Exposed for testing. - DockerClient dockerClient; - - @Inject - public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { - this(config, - true, /* fallback to 1.23 on errors */ - metricReceiver, - !config.isRunningLocally()); - } - - private DockerImpl(final DockerConfig config, - boolean fallbackTo123OnErrors, - MetricReceiverWrapper metricReceiverWrapper, - boolean inProduction) { - this.config = config; - this.fallbackTo123OnErrors = fallbackTo123OnErrors; - this.inProduction = inProduction; - if (config == null) { - this.SECONDS_TO_WAIT_BEFORE_KILLING = 10; - } else { - SECONDS_TO_WAIT_BEFORE_KILLING = config.secondsToWaitBeforeKillingContainer(); - } - if (metricReceiverWrapper != null) { - setMetrics(metricReceiverWrapper); - } - } + final DockerClient dockerClient; // For testing DockerImpl(final DockerClient dockerClient) { - this(null, false, null, false); this.dockerClient = dockerClient; + this.SECONDS_TO_WAIT_BEFORE_KILLING = 10; } - // For testing - DockerImpl(final DockerConfig config, - boolean fallbackTo123OnErrors, - MetricReceiverWrapper metricReceiverWrapper) { - this(config, fallbackTo123OnErrors, metricReceiverWrapper, false); + DockerImpl( + final DockerConfig config, + boolean fallbackTo123OnErrors, + MetricReceiverWrapper metricReceiverWrapper) { + SECONDS_TO_WAIT_BEFORE_KILLING = config.secondsToWaitBeforeKillingContainer(); + + dockerClient = initDockerConnection(config, fallbackTo123OnErrors); + setMetrics(metricReceiverWrapper); } - @Override - public void start() { - if (started) return; - started = true; + @Inject + public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { + this( + config, + true, /* fallback to 1.23 on errors */ + metricReceiver); - if (config != null) { - if (dockerClient == null) { - dockerClient = initDockerConnection(); - } - if (inProduction) { - Duration minAgeToDelete = Duration.ofMinutes(config.imageGCMinTimeToLiveMinutes()); - dockerImageGC = Optional.of(new DockerImageGarbageCollector(minAgeToDelete)); - - try { - setupDockerNetworkIfNeeded(); - } catch (Exception e) { - throw new DockerException("Could not setup docker network", e); - } + if (!config.isRunningLocally()) { + Duration minAgeToDelete = Duration.ofMinutes(config.imageGCMinTimeToLiveMinutes()); + dockerImageGC = Optional.of(new DockerImageGarbageCollector(minAgeToDelete)); + + try { + setupDockerNetworkIfNeeded(); + } catch (Exception e) { + throw new DockerException("Could not setup docker network", e); } } } @@ -518,7 +489,7 @@ public class DockerImpl implements Docker { } } - private DockerClient initDockerConnection() { + private DockerClient initDockerConnection(final DockerConfig config, boolean fallbackTo123orErrors) { JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory() .withMaxPerRouteConnections(config.maxPerRouteConnections()) .withMaxTotalConnections(config.maxTotalConnections()) @@ -537,7 +508,7 @@ public class DockerImpl implements Docker { logger.info("Found version 1.24 or newer of remote API, using 1.23."); } } catch (Exception e) { - if (!fallbackTo123OnErrors) { + if (!fallbackTo123orErrors) { throw e; } logger.log(LogLevel.ERROR, "Failed when trying to figure out remote API version of docker, using 1.23", e); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java index 549af0d85cb..079d6876043 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java @@ -46,13 +46,11 @@ public class DockerTestUtils { public static DockerImpl getDocker() { if (docker == null) { - DockerImpl tmpDocker = new DockerImpl( + docker = new DockerImpl( dockerConfig, false, /* fallback to 1.23 on errors */ new MetricReceiverWrapper(MetricReceiver.nullImplementation)); - tmpDocker.start(); - createDockerTestNetworkIfNeeded(tmpDocker); - docker = tmpDocker; + createDockerTestNetworkIfNeeded(docker); } return docker; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java deleted file mode 100644 index 9caf1307aa4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.File; -import java.io.IOException; -import java.util.logging.Logger; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeAdminConfig { - private static final Logger logger = Logger.getLogger(NodeAdminConfig.class.getName()); - private static final ObjectMapper mapper = new ObjectMapper(); - - enum Mode { - tenant, - config_server_host - } - - @JsonProperty("mode") - public Mode mode = Mode.tenant; - - public static NodeAdminConfig fromFile(File file) { - if (!file.exists()) { - return new NodeAdminConfig(); - } - - try { - return mapper.readValue(file, NodeAdminConfig.class); - } catch (IOException e) { - throw new RuntimeException("Failed to read " + file + " as a " + - NodeAdminConfig.class.getName(), e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java deleted file mode 100644 index 02d63f5e5ed..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.concurrent.classlock.ClassLocking; -import com.yahoo.net.HostName; -import com.yahoo.system.ProcessExecuter; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; - -import java.io.File; -import java.time.Clock; -import java.time.Duration; -import java.util.function.Function; - -/** - * NodeAdminMain is the main component of the node admin JDisc application: - * - It will read config and check its environment to figure out its responsibilities - * - It will "start" (only) the necessary components. - * - Other components MUST NOT try to start (typically in constructor) since the features - * they provide is NOT WANTED and possibly destructive, and/or the environment may be - * incompatible. For instance, trying to contact the Docker daemon too early will - * be fatal: the node admin may not have installed and started the docker daemon. - */ -public class NodeAdminMain implements AutoCloseable { - private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); - private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); - - private final Docker docker; - private final MetricReceiverWrapper metricReceiver; - private final ClassLocking classLocking; - - private NodeAdminStateUpdater nodeAdminStateUpdater; - - public NodeAdminMain(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { - this.docker = docker; - this.metricReceiver = metricReceiver; - this.classLocking = classLocking; - start(); - } - - @Override - public void close() { - nodeAdminStateUpdater.stop(); - } - - public NodeAdminStateUpdater getNodeAdminStateUpdater() { - return nodeAdminStateUpdater; - } - - public void start() { - String staticConfigPath = Defaults.getDefaults().underVespaHome("conf/node-admin.json"); - NodeAdminConfig config = NodeAdminConfig.fromFile(new File(staticConfigPath)); - - switch (config.mode) { - case tenant: - setupTenantHostNodeAdmin(); - break; - case config_server_host: - setupConfigServerHostNodeAdmin(); - break; - default: - throw new IllegalStateException( - "Unknown bootstrap mode: " + config.mode.name()); - } - } - - private void setupTenantHostNodeAdmin() { - Clock clock = Clock.systemUTC(); - String dockerHostHostName = HostName.getLocalhost(); - ProcessExecuter processExecuter = new ProcessExecuter(); - Environment environment = new Environment(); - - ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - - docker.start(); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); - - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock); - AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); - - Function nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, - metricReceiver, clock); - - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, - dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); - - nodeAdminStateUpdater.start(); - } - - private void setupConfigServerHostNodeAdmin() { - // TODO: - // - install and start docker daemon - // - Read config that specifies which containers to start how - // - use thin static backends for node repo, orchestrator, and others - // - Start core node admin. - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java index ea9b38efa26..3777d7e20d1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java @@ -4,29 +4,78 @@ package com.yahoo.vespa.hosted.node.admin.provider; import com.google.inject.Inject; import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.net.HostName; + +import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminMain; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.time.Clock; +import java.time.Duration; +import java.util.function.Function; +/** + * Set up node admin for production. + * + * @author dybis + */ public class NodeAdminProvider implements Provider { - private final NodeAdminMain nodeAdminMain; + + // WARNING: reducing the node agent interval will increase the load on the config servers + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(60); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + + private final NodeAdminStateUpdater nodeAdminStateUpdater; @Inject - public NodeAdminProvider(Docker docker, - MetricReceiverWrapper metricReceiver, - ClassLocking classLocking) { - nodeAdminMain = new NodeAdminMain(docker, metricReceiver, classLocking); - nodeAdminMain.start(); + public NodeAdminProvider(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { + Clock clock = Clock.systemUTC(); + String dockerHostHostName = HostName.getLocalhost(); + ProcessExecuter processExecuter = new ProcessExecuter(); + Environment environment = new Environment(); + + ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); + DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); + + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock); + AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); + + Function nodeAgentFactory = + (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, + metricReceiver, clock); + + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, + dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); + + nodeAdminStateUpdater.start(); } @Override public NodeAdminStateUpdater get() { - return nodeAdminMain.getNodeAdminStateUpdater(); + return nodeAdminStateUpdater; } @Override public void deconstruct() { - nodeAdminMain.close(); + nodeAdminStateUpdater.stop(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index 7a5d713936d..a1cc1850d23 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -32,9 +32,6 @@ public class DockerMock implements Docker { this.callOrderVerifier = callOrderVerifier; } - @Override - public void start() { } - @Override public CreateContainerCommand createContainerCommand( DockerImage dockerImage, -- cgit v1.2.3