diff options
69 files changed, 1216 insertions, 329 deletions
diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index eea085bd103..7f0acd83b0d 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; -import com.yahoo.container.logging.AccessLog; import com.yahoo.log.LogLevel; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; import com.yahoo.vespa.clustercontroller.core.restapiv2.ClusterControllerStateRestAPI; @@ -13,19 +12,22 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.Executor; import java.util.logging.Logger; public class StateRestApiV2Handler extends JDiscHttpRequestHandler { private static final Logger log = Logger.getLogger(StateRestApiV2Handler.class.getName()); @Inject - public StateRestApiV2Handler(Executor executor, ClusterController cc, ClusterInfoConfig config, AccessLog accessLog) { - this(executor, new ClusterControllerStateRestAPI(cc, getClusterControllerSockets(config)), "/cluster/v2", accessLog); + public StateRestApiV2Handler(ClusterController cc, ClusterInfoConfig config, + JDiscHttpRequestHandler.Context ctx) + { + this(new ClusterControllerStateRestAPI(cc, getClusterControllerSockets(config)), "/cluster/v2", ctx); } - private StateRestApiV2Handler(Executor executor, ClusterControllerStateRestAPI restApi, String pathPrefix, AccessLog accessLog) { - super(new RestApiHandler(restApi).setDefaultPathPrefix(pathPrefix), executor, accessLog); + private StateRestApiV2Handler(ClusterControllerStateRestAPI restApi, String pathPrefix, + JDiscHttpRequestHandler.Context ctx) + { + super(new RestApiHandler(restApi).setDefaultPathPrefix(pathPrefix), ctx); } // This method is package-private instead of private to be accessible to unit-tests. diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java index ae7c32e0f95..6817a033675 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java @@ -2,22 +2,21 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; -import com.yahoo.container.logging.AccessLog; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; -import java.util.concurrent.Executor; - public class StatusHandler extends JDiscHttpRequestHandler { private final com.yahoo.vespa.clustercontroller.core.status.StatusHandler statusHandler; @Inject - public StatusHandler(ClusterController fc, Executor executor, AccessLog accessLog) { - this(new com.yahoo.vespa.clustercontroller.core.status.StatusHandler(fc), executor, accessLog); + public StatusHandler(ClusterController fc, JDiscHttpRequestHandler.Context ctx) { + this(new com.yahoo.vespa.clustercontroller.core.status.StatusHandler(fc), ctx); } - private StatusHandler(com.yahoo.vespa.clustercontroller.core.status.StatusHandler handler, Executor executor, AccessLog accessLog) { - super(handler, executor, accessLog); + private StatusHandler(com.yahoo.vespa.clustercontroller.core.status.StatusHandler handler, + JDiscHttpRequestHandler.Context ctx) + { + super(handler, ctx); this.statusHandler = handler; } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java index 524aba1398b..dbc7834a601 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2HandlerTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.yahoo.cloud.config.ClusterInfoConfig; -import com.yahoo.container.logging.AccessLog; import com.yahoo.vespa.clustercontroller.core.restapiv2.ClusterControllerStateRestAPI; import junit.framework.TestCase; @@ -15,13 +14,11 @@ import java.util.concurrent.TimeUnit; public class StateRestApiV2HandlerTest extends TestCase { public void testNoMatchingSockets() { - ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 100, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1000)); ClusterController controller = new ClusterController(); ClusterInfoConfig config = new ClusterInfoConfig( new ClusterInfoConfig.Builder().clusterId("cluster-id").nodeCount(1)); ClusterInfoConfig.Builder clusterConfig = new ClusterInfoConfig.Builder(); - new StateRestApiV2Handler(executor, controller, config, AccessLog.voidAccessLog()); - executor.shutdown(); + new StateRestApiV2Handler(controller, config, StateRestApiV2Handler.testOnlyContext()); } public void testMappingOfIndexToClusterControllers() { diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java index 66cb477e793..49e0a637368 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandlerTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; -import com.yahoo.container.logging.AccessLog; import junit.framework.TestCase; import java.util.concurrent.ArrayBlockingQueue; @@ -12,9 +11,7 @@ public class StatusHandlerTest extends TestCase { public void testSimple() { ClusterController controller = new ClusterController(); - ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 100, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1000)); - StatusHandler handler = new StatusHandler(controller, executor, AccessLog.voidAccessLog()); - executor.shutdown(); + StatusHandler handler = new StatusHandler(controller, StatusHandler.testOnlyContext()); } } diff --git a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java index f43f04d1aac..2029630a5de 100644 --- a/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java +++ b/clustercontroller-apputil/src/main/java/com/yahoo/vespa/clustercontroller/apputil/communication/http/JDiscHttpRequestHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.clustercontroller.apputil.communication.http; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.CompletionHandler; @@ -14,7 +13,6 @@ import org.apache.commons.io.IOUtils; import java.io.*; import java.time.Duration; -import java.util.concurrent.Executor; import java.util.logging.Logger; /** @@ -29,8 +27,8 @@ public class JDiscHttpRequestHandler extends LoggingRequestHandler { private static final Logger log = Logger.getLogger(JDiscHttpRequestHandler.class.getName()); private final HttpRequestHandler requestHandler; - public JDiscHttpRequestHandler(HttpRequestHandler handler, Executor executor, AccessLog accessLog) { - super(executor, accessLog); + public JDiscHttpRequestHandler(HttpRequestHandler handler, LoggingRequestHandler.Context parentCtx) { + super(parentCtx); this.requestHandler = handler; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 4f365ebbab3..933751fd9ad 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -50,6 +50,9 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { this.accessLog = other.accessLog; this.metric = other.metric; } + public Executor getExecutor() { return executor; } + public AccessLog getAccessLog() { return accessLog; } + public Metric getMetric() { return metric; } } public static Context testOnlyContext() { return new Context(new Executor() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java index 764ba9c2104..6f8ebc4c5db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java @@ -29,12 +29,16 @@ public class AthenzIdentityVerifier implements HostnameVerifier { public boolean verify(String hostname, SSLSession session) { try { X509Certificate cert = (X509Certificate) session.getPeerCertificates()[0]; - return allowedIdentities.contains(AthenzUtils.createAthenzIdentity(cert)); + return isTrusted(AthenzUtils.createAthenzIdentity(cert)); } catch (SSLPeerUnverifiedException e) { log.log(Level.WARNING, "Unverified client: " + hostname); return false; } } + public boolean isTrusted(AthenzIdentity identity) { + return allowedIdentities.contains(identity); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java index 8c8b5fdf22e..086696fc6a4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java @@ -67,12 +67,4 @@ public interface ConfigServerClient { */ EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) throws IOException; - /** - * The nodes allocated to the deployment - * - * @param deployment The application/zone pair - * @return The nodes for this deployment - * @throws IOException If unable to retrieve the node list - */ - NodeList getNodeList(DeploymentId deployment) throws IOException; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeList.java deleted file mode 100644 index 0169bfec5fe..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeList.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.configserver;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author smorgrav - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeList { - - @JsonProperty("nodes") - public List<Node> nodes; - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Node { - @JsonProperty("hostname") - public String hostname; - @JsonProperty("flavor") - public String flavor; - @JsonProperty("membership") - public Membership membership; - @JsonProperty("cost") - public int cost; - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Membership { - @JsonProperty("clustertype") - public String clusterType; - @JsonProperty("clusterid") - public String clusterId; - } - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobList.java new file mode 100644 index 00000000000..db5037af4d2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobList.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MaintenanceJobList { + @JsonProperty("jobs") + public List<MaintenanceJobName> jobs = new ArrayList<>(); + @JsonProperty("inactive") + public List<String> inactive = new ArrayList<>(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobName.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobName.java new file mode 100644 index 00000000000..ba54fed4cb1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/MaintenanceJobName.java @@ -0,0 +1,11 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MaintenanceJobName { + @JsonProperty("name") + public String name; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeEnvironment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeEnvironment.java new file mode 100644 index 00000000000..8510aacf3a8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeEnvironment.java @@ -0,0 +1,11 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +/** + * @author bjorncs + */ +public enum NodeEnvironment { + BARE_METAL, + VIRTUAL_MACHINE, + DOCKER_CONTAINER +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java new file mode 100644 index 00000000000..68696ac59df --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -0,0 +1,37 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Wire class for node-repository representation of the history of a node + * + * @author smorgrav + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class NodeHistory { + @JsonProperty("at") + public Long at; + @JsonProperty("agent") + public Agent agent; + @JsonProperty("event") + public String event; + + public Long getAt() { + return at; + } + + public Agent getAgent() { + return agent; + } + + public String getEvent() { + return event; + } + + public enum Agent { system, application, operator, NodeRetirer } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeList.java new file mode 100644 index 00000000000..d707cdc5995 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeList.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeList { + @JsonProperty + List<NodeRepositoryNode> nodes; + + public NodeList() { + } + + public NodeList(List<NodeRepositoryNode> nodes) { + this.nodes = nodes; + } + + public List<NodeRepositoryNode> nodes() { + return nodes; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeMembership.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeMembership.java new file mode 100644 index 00000000000..f567e27bdea --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeMembership.java @@ -0,0 +1,43 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeMembership { + + @JsonProperty + public String clustertype; + @JsonProperty + public String clusterid; + @JsonProperty + public String group; + @JsonProperty + public Integer index; + @JsonProperty + public Boolean retired; + + public String getClustertype() { + return clustertype; + } + + public String getClusterid() { + return clusterid; + } + + public String getGroup() { + return group; + } + + public Integer getIndex() { + return index; + } + + public Boolean getRetired() { + return retired; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeOwner.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeOwner.java new file mode 100644 index 00000000000..21c2c469077 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeOwner.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeOwner { + + @JsonProperty + public String tenant; + @JsonProperty + public String application; + @JsonProperty + public String instance; + + public NodeOwner() {} + + public String getTenant() { + return tenant; + } + + public String getApplication() { + return application; + } + + public String getInstance() { + return instance; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java new file mode 100644 index 00000000000..346c77bd316 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +import java.io.IOException; +import java.util.Collection; + +/** + * @author smorgrav + * @author bjorncs + */ +public interface NodeRepositoryClientInterface { + + enum WantTo { + Retire, + Deprovision + } + + void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) throws IOException; + + NodeRepositoryNode getNode(ZoneId zone, String hostname) throws IOException; + + void deleteNode(ZoneId zone, String hostname) throws IOException; + + NodeList listNodes(ZoneId zone, boolean recursive) throws IOException; + + NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) throws IOException; + + String resetFailureInformation(ZoneId zone, String nodename) throws IOException; + + String restart(ZoneId zone, String nodename) throws IOException; + + String reboot(ZoneId zone, String nodename) throws IOException; + + String cancelReboot(ZoneId zone, String nodename) throws IOException; + + String wantTo(ZoneId zone, String nodename, WantTo... actions) throws IOException; + + String cancelRestart(ZoneId zone, String nodename) throws IOException; + + String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) throws IOException; + + void setState(ZoneId zone, NodeState nodeState, String nodename) throws IOException; + + String enableMaintenanceJob(ZoneId zone, String jobName) throws IOException; + + String disableMaintenanceJob(ZoneId zone, String jobName) throws IOException; + + MaintenanceJobList listMaintenanceJobs(ZoneId zone) throws IOException; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java new file mode 100644 index 00000000000..7304daeee1d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -0,0 +1,386 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Set; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class NodeRepositoryNode { + + @JsonProperty("url") + private String url; + @JsonProperty("id") + private String id; + @JsonProperty("state") + private NodeState state; + @JsonProperty("hostname") + private String hostname; + @JsonProperty("ipAddresses") + private Set<String> ipAddresses; + @JsonProperty("additionalIpAddresses") + private Set<String> additionalIpAddresses; + @JsonProperty("openStackId") + private String openStackId; + @JsonProperty("flavor") + private String flavor; + @JsonProperty("canonicalFlavor") + private String canonicalFlavor; + @JsonProperty("membership") + private NodeMembership membership; + @JsonProperty("owner") + private NodeOwner owner; + @JsonProperty("restartGeneration") + private Integer restartGeneration; + @JsonProperty("rebootGeneration") + private Integer rebootGeneration; + @JsonProperty("currentRestartGeneration") + private Integer currentRestartGeneration; + @JsonProperty("currentRebootGeneration") + private Integer currentRebootGeneration; + @JsonProperty("vespaVersion") + private String vespaVersion; + @JsonProperty("wantedVespaVersion") + private String wantedVespaVersion; + @JsonProperty("failCount") + private Integer failCount; + @JsonProperty("hardwareFailure") + private Boolean hardwareFailure; + @JsonProperty("hardwareFailureDescription") + private String hardwareFailureDescription; + @JsonProperty("environment") + private NodeEnvironment environment; + @JsonProperty("type") + private NodeType type; + @JsonProperty("wantedDockerImage") + private String wantedDockerImage; + @JsonProperty("currentDockerImage") + private String currentDockerImage; + @JsonProperty("parentHostname") + private String parentHostname; + @JsonProperty("wantToRetire") + private Boolean wantToRetire; + @JsonProperty("wantToDeprovision") + private Boolean wantToDeprovision; + @JsonProperty("minDiskAvailableGb") + private Double minDiskAvailableGb; + @JsonProperty("minMainMemoryAvailableGb") + private Double minMainMemoryAvailableGb; + @JsonProperty("cost") + private Integer cost; + @JsonProperty("minCpuCores") + private Double minCpuCores; + @JsonProperty("description") + private String description; + @JsonProperty("history") + private NodeHistory[] history; + @JsonProperty("allowedToBeDown") + private Boolean allowedToBeDown; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public NodeState getState() { + return state; + } + + public void setState(NodeState state) { + this.state = state; + } + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } + + public Set<String> getIpAddresses() { + return ipAddresses; + } + + public Set<String> getAdditionalIpAddresses() { + return additionalIpAddresses; + } + + public void setIpAddresses(Set<String> ipAddresses) { + this.ipAddresses = ipAddresses; + } + + public void setAdditionalIpAddresses(Set<String> additionalIpAddresses) { + this.additionalIpAddresses = additionalIpAddresses; + } + + public String getOpenStackId() { + return openStackId; + } + + public void setOpenStackId(String openStackId) { + this.openStackId = openStackId; + } + + public String getFlavor() { + return flavor; + } + + public void setFlavor(String flavor) { + this.flavor = flavor; + } + + public String getCanonicalFlavor() { + return canonicalFlavor; + } + + public void setCanonicalFlavor(String canonicalFlavor) { + this.canonicalFlavor = canonicalFlavor; + } + + public NodeMembership getMembership() { + return membership; + } + + public void setMembership(NodeMembership membership) { + this.membership = membership; + } + + public NodeOwner getOwner() { + return owner; + } + + public void setOwner(NodeOwner owner) { + this.owner = owner; + } + + public Integer getRestartGeneration() { + return restartGeneration; + } + + public void setRestartGeneration(Integer restartGeneration) { + this.restartGeneration = restartGeneration; + } + + public Integer getRebootGeneration() { + return rebootGeneration; + } + + public void setRebootGeneration(Integer rebootGeneration) { + this.rebootGeneration = rebootGeneration; + } + + public Integer getCurrentRestartGeneration() { + return currentRestartGeneration; + } + + public void setCurrentRestartGeneration(Integer currentRestartGeneration) { + this.currentRestartGeneration = currentRestartGeneration; + } + + public Integer getCurrentRebootGeneration() { + return currentRebootGeneration; + } + + public void setCurrentRebootGeneration(Integer currentRebootGeneration) { + this.currentRebootGeneration = currentRebootGeneration; + } + + public String getVespaVersion() { + return vespaVersion; + } + + public void setVespaVersion(String vespaVersion) { + this.vespaVersion = vespaVersion; + } + + public String getWantedVespaVersion() { + return wantedVespaVersion; + } + + public void setWantedVespaVersion(String wantedVespaVersion) { + this.wantedVespaVersion = wantedVespaVersion; + } + + public Integer getFailCount() { + return failCount; + } + + public void setFailCount(Integer failCount) { + this.failCount = failCount; + } + + public Boolean getHardwareFailure() { + return hardwareFailure; + } + + public void setHardwareFailure(Boolean hardwareFailure) { + this.hardwareFailure = hardwareFailure; + } + + public String getHardwareFailureDescription() { + return hardwareFailureDescription; + } + + public void setHardwareFailureDescription(String hardwareFailureDescription) { + this.hardwareFailureDescription = hardwareFailureDescription; + } + + public NodeEnvironment getEnvironment() { + return environment; + } + + public void setEnvironment(NodeEnvironment environment) { + this.environment = environment; + } + + public NodeType getType() { + return type; + } + + public void setType(NodeType type) { + this.type = type; + } + + public String getWantedDockerImage() { + return wantedDockerImage; + } + + public void setWantedDockerImage(String wantedDockerImage) { + this.wantedDockerImage = wantedDockerImage; + } + + public String getCurrentDockerImage() { + return currentDockerImage; + } + + public void setCurrentDockerImage(String currentDockerImage) { + this.currentDockerImage = currentDockerImage; + } + + public String getParentHostname() { + return parentHostname; + } + + public void setParentHostname(String parentHostname) { + this.parentHostname = parentHostname; + } + + public Boolean getWantToRetire() { + return wantToRetire; + } + + public Boolean getWantToDeprovision() { return wantToDeprovision; } + + public void setWantToRetire(Boolean wantToRetire) { + this.wantToRetire = wantToRetire; + } + + public void setWantToDeprovision(Boolean wantToDeprovision) { + this.wantToDeprovision = wantToDeprovision; + } + + public Double getMinDiskAvailableGb() { + return minDiskAvailableGb; + } + + public void setMinDiskAvailableGb(Double minDiskAvailableGb) { + this.minDiskAvailableGb = minDiskAvailableGb; + } + + public Double getMinMainMemoryAvailableGb() { + return minMainMemoryAvailableGb; + } + + public void setMinMainMemoryAvailableGb(Double minMainMemoryAvailableGb) { + this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; + } + + public Integer getCost() { + return cost; + } + + public void setCost(Integer cost) { + this.cost = cost; + } + + public Double getMinCpuCores() { + return minCpuCores; + } + + public void setMinCpuCores(Double minCpuCores) { + this.minCpuCores = minCpuCores; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public NodeHistory[] getHistory() { + return history; + } + + public void setHistory(NodeHistory[] history) { + this.history = history; + } + + public Boolean getAllowedToBeDown() { + return allowedToBeDown; + } + + @Override + public String toString() { + return "NodeRepositoryNode{" + + "url='" + url + '\'' + + ", id='" + id + '\'' + + ", state=" + state + + ", hostname='" + hostname + '\'' + + ", ipAddresses='" + ipAddresses + '\'' + + ", additionalIpAddresses='" + additionalIpAddresses + '\'' + + ", openStackId='" + openStackId + '\'' + + ", flavor='" + flavor + '\'' + + ", canonicalFlavor='" + canonicalFlavor + '\'' + + ", membership=" + membership + + ", owner=" + owner + + ", restartGeneration=" + restartGeneration + + ", rebootGeneration=" + rebootGeneration + + ", currentRestartGeneration=" + currentRestartGeneration + + ", currentRebootGeneration=" + currentRebootGeneration + + ", vespaVersion='" + vespaVersion + '\'' + + ", wantedVespaVersion='" + wantedVespaVersion + '\'' + + ", failCount=" + failCount + + ", hardwareFailure=" + hardwareFailure + + ", hardwareFailureDescription='" + hardwareFailureDescription + '\'' + + ", environment=" + environment + + ", type=" + type + + ", wantedDockerImage='" + wantedDockerImage + '\'' + + ", currentDockerImage='" + currentDockerImage + '\'' + + ", wantToRetire='" + wantToRetire + '\'' + + ", wantToDeprovision='" + wantToDeprovision + '\'' + + ", minDiskAvailableGb='" + minDiskAvailableGb + '\'' + + ", minMainMemoryAvailableGb='" + minMainMemoryAvailableGb + '\'' + + ", cost='" + cost + '\'' + + ", minCpuCores='" + minCpuCores + '\'' + + ", description='" + description + '\'' + + ", allowedToBeDown='" + allowedToBeDown + '\'' + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java new file mode 100644 index 00000000000..6271671ca2d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java @@ -0,0 +1,15 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +public enum NodeState { + + provisioned, + ready, + reserved, + active, + inactive, + dirty, + failed, + parked; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeType.java new file mode 100644 index 00000000000..071cbe7100f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeType.java @@ -0,0 +1,22 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +/** + * The possible types of nodes in the node repository + * + * @author bjorncs + */ +public enum NodeType { + + /** A host of a set of (docker) tenant nodes */ + host, + + /** Nodes running the shared proxy layer */ + proxy, + + /** A node to be assigned to a tenant to run application workloads */ + tenant, + + /** A config server */ + config +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java new file mode 100644 index 00000000000..ee808399637 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java @@ -0,0 +1,80 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import java.util.Collection; + +/** + * @author mortent + * @author mpolden + * @author smorgrav + */ +@Consumes(MediaType.APPLICATION_JSON) +public interface ProvisionResource { + + @POST + @Path("/node") + String addNodes(Collection<NodeRepositoryNode> node); + + @DELETE + @Path("/node/{hostname}") + String deleteNode(@PathParam("hostname") String hostname); + + @GET + @Path("/node/{hostname}") + NodeRepositoryNode getNode(@PathParam("hostname") String hostname); + + @POST + @Path("/node/{hostname}") + String patchNode(@PathParam("hostname") String hostname, + NodeRepositoryNode patchValues, + @HeaderParam("X-HTTP-Method-Override") String patchOverride); + + @GET + @Path("/node/") + NodeList listNodes(@QueryParam("recursive") boolean recursive); + + @GET + @Path("/node/") + NodeList listNodes(@QueryParam("application") String applicationString, + @QueryParam("recursive") boolean recursive); + + @GET + @Path("/node/") + NodeList listNodesWithParent(@QueryParam("recursive") boolean recursive, + @QueryParam("parentHost") String parentHostname); + + @PUT + @Path("/state/{state}/{hostname}") + String setState(@PathParam("state") NodeState state, @PathParam("hostname") String hostname); + + @POST + @Path("/command/reboot") + String reboot(@QueryParam("hostname") String hostname); + + @POST + @Path("/command/restart") + String restart(@QueryParam("hostname") String hostname); + + @GET + @Path("/maintenance/") + MaintenanceJobList listMaintenanceJobs(); + + @POST + @Path("/maintenance/inactive/{jobname}") + String disableMaintenanceJob(@PathParam("jobname") String jobname); + + @DELETE + @Path("/maintenance/inactive/{jobname}") + String enableMaintenanceJob(@PathParam("jobname") String jobname); +} + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/package-info.java new file mode 100644 index 00000000000..08036880cd2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java index e68bf0ccc24..a7d51fa4d24 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java @@ -46,17 +46,17 @@ public class ZoneFilterMock implements ZoneList { } @Override - public ZoneList in(Environment environment) { - return filter(zoneId -> zoneId.environment() == environment); + public ZoneList in(Environment... environments) { + return filter(zoneId -> new HashSet<>(Arrays.asList(environments)).contains(zoneId.environment())); } @Override - public ZoneList in(RegionName region) { - return filter(zoneId -> zoneId.region().equals(region)); + public ZoneList in(RegionName... regions) { + return filter(zoneId -> new HashSet<>(Arrays.asList(regions)).contains(zoneId.region())); } @Override - public ZoneList zones(ZoneId... zones) { + public ZoneList among(ZoneId... zones) { return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java index cd263769864..408168e41da 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java @@ -19,14 +19,14 @@ public interface ZoneList extends ZoneFilter { @Override ZoneList not(); - /** Zones in the given environment. */ - ZoneList in(Environment environment); + /** Zones in one of the given environments. */ + ZoneList in(Environment... environments); - /** Zones in the given region. */ - ZoneList in(RegionName region); + /** Zones in one of the given regions. */ + ZoneList in(RegionName... regions); /** Only the given zones — combine with not() for best effect! */ - ZoneList zones(ZoneId... zones); + ZoneList among(ZoneId... zones); /** Returns the id of all zones in this list as — you guessed it — a list. */ List<ZoneId> ids(); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 989dda42641..b033286b82a 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -110,8 +110,7 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.5.2</version> + <artifactId>httpcore</artifactId> </dependency> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 44e4cf0740f..1f386b662aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -66,6 +67,7 @@ public class Controller extends AbstractComponent { private final GlobalRoutingService globalRoutingService; private final ZoneRegistry zoneRegistry; private final ConfigServerClient configServerClient; + private final NodeRepositoryClientInterface nodeRepositoryClient; private final MetricsService metricsService; private final Chef chefClient; private final Organization organization; @@ -81,19 +83,19 @@ public class Controller extends AbstractComponent { public Controller(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, EntityService entityService, Organization organization, GlobalRoutingService globalRoutingService, - ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, + ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, NodeRepositoryClientInterface nodeRepositoryClient, MetricsService metricsService, NameService nameService, RoutingGenerator routingGenerator, Chef chefClient, AthenzClientFactory athenzClientFactory) { this(db, curator, rotationsConfig, gitHub, entityService, organization, globalRoutingService, zoneRegistry, - configServerClient, metricsService, nameService, routingGenerator, chefClient, + configServerClient, nodeRepositoryClient, metricsService, nameService, routingGenerator, chefClient, Clock.systemUTC(), athenzClientFactory); } public Controller(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, EntityService entityService, Organization organization, GlobalRoutingService globalRoutingService, - ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, + ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, NodeRepositoryClientInterface nodeRepositoryClient, MetricsService metricsService, NameService nameService, RoutingGenerator routingGenerator, Chef chefClient, Clock clock, AthenzClientFactory athenzClientFactory) { @@ -106,6 +108,7 @@ public class Controller extends AbstractComponent { Objects.requireNonNull(globalRoutingService, "GlobalRoutingService cannot be null"); Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); Objects.requireNonNull(configServerClient, "ConfigServerClient cannot be null"); + Objects.requireNonNull(nodeRepositoryClient, "NodeRepositoryClientInterface cannot be null"); Objects.requireNonNull(metricsService, "MetricsService cannot be null"); Objects.requireNonNull(nameService, "NameService cannot be null"); Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"); @@ -120,6 +123,7 @@ public class Controller extends AbstractComponent { this.globalRoutingService = globalRoutingService; this.zoneRegistry = zoneRegistry; this.configServerClient = configServerClient; + this.nodeRepositoryClient = nodeRepositoryClient; this.metricsService = metricsService; this.chefClient = chefClient; this.clock = clock; @@ -239,6 +243,10 @@ public class Controller extends AbstractComponent { return curator; } + public NodeRepositoryClientInterface nodeRepositoryClient() { + return nodeRepositoryClient; + } + private String printableVersion(Optional<VespaVersion> vespaVersion) { return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("Unknown"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index cf0600f87bd..100fb11d68c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -38,24 +39,24 @@ public class ClusterInfoMaintainer extends Maintainer { this.controller = controller; } - private static String clusterid(NodeList.Node node) { - return node.membership.clusterId; + private static String clusterid(NodeRepositoryNode node) { + return node.getMembership().clusterid; } private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes, ZoneId zone) { Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); // Group nodes by clusterid - Map<String, List<NodeList.Node>> clusters = nodes.nodes.stream() - .filter(node -> node.membership != null) + Map<String, List<NodeRepositoryNode>> clusters = nodes.nodes().stream() + .filter(node -> node.getMembership() != null) .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid)); // For each cluster - get info for (String id : clusters.keySet()) { - List<NodeList.Node> clusterNodes = clusters.get(id); + List<NodeRepositoryNode> clusterNodes = clusters.get(id); // Assume they are all equal and use first node as a representative for the cluster - NodeList.Node node = clusterNodes.get(0); + NodeRepositoryNode node = clusterNodes.get(0); // Extract flavor info double cpu = 0; @@ -73,9 +74,9 @@ public class ClusterInfoMaintainer extends Maintainer { }*/ // Add to map - List<String> hostnames = clusterNodes.stream().map(node1 -> node1.hostname).collect(Collectors.toList()); - ClusterInfo inf = new ClusterInfo(node.flavor, node.cost, cpu, mem, disk, - ClusterSpec.Type.from(node.membership.clusterType), hostnames); + List<String> hostnames = clusterNodes.stream().map(NodeRepositoryNode::getHostname).collect(Collectors.toList()); + ClusterInfo inf = new ClusterInfo(node.getFlavor(), node.getCost(), cpu, mem, disk, + ClusterSpec.Type.from(node.getMembership().clustertype), hostnames); infoMap.put(new ClusterSpec.Id(id), inf); } @@ -88,7 +89,11 @@ public class ClusterInfoMaintainer extends Maintainer { for (Deployment deployment : application.deployments().values()) { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { - NodeList nodes = controller().applications().configserverClient().getNodeList(deploymentId); + NodeList nodes = controller.nodeRepositoryClient() + .listNodes(deploymentId.zoneId(), + deploymentId.applicationId().tenant().value(), + deploymentId.applicationId().application().value(), + deploymentId.applicationId().instance().value()); Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); controller().applications().lockIfPresent(application.id(), lockedApplication -> controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 3f8e177ac8a..379e5c10847 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -7,8 +7,10 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Environment; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzSslContextProvider; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -21,14 +23,19 @@ import org.apache.http.client.methods.HttpPatch; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.conn.ssl.X509HostnameVerifier; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; @@ -253,9 +260,43 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-client") - .setSSLContext(sslContextProvider.get()) - .setSSLHostnameVerifier(hostnameVerifier) + .setSslcontext(sslContextProvider.get()) + .setHostnameVerifier(new AthenzIdentityVerifierAdapter(hostnameVerifier)) .setDefaultRequestConfig(config) .build(); } + + private static class AthenzIdentityVerifierAdapter implements X509HostnameVerifier { + + private final AthenzIdentityVerifier verifier; + + AthenzIdentityVerifierAdapter(AthenzIdentityVerifier verifier) { + this.verifier = verifier; + } + + @Override + public boolean verify(String hostname, SSLSession sslSession) { + return verifier.verify(hostname, sslSession); + } + + @Override + public void verify(String host, SSLSocket ssl) { /* All sockets accepted */} + + @Override + public void verify(String hostname, X509Certificate certificate) throws SSLException { + AthenzIdentity identity = AthenzUtils.createAthenzIdentity(certificate); + if (!verifier.isTrusted(identity)) { + throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); + } + } + + @Override + public void verify(String hostname, String[] cns, String[] subjectAlts) throws SSLException { + AthenzIdentity identity = AthenzUtils.createAthenzIdentity(cns[0]); + if (!verifier.isTrusted(identity)) { + throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); + } + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java index 50c4efc2d27..89baa1cfa39 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java @@ -9,12 +9,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import java.io.IOException; import java.io.OutputStream; import java.net.URI; -import java.util.concurrent.Executor; /** * Responds to requests for the root path of the controller by listing the available web service API's. @@ -28,8 +26,8 @@ import java.util.concurrent.Executor; */ public class RootHandler extends LoggingRequestHandler { - public RootHandler(Executor executor, AccessLog accessLog) { - super(executor, accessLog); + public RootHandler(LoggingRequestHandler.Context parentCtx) { + super(parentCtx); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a7d072d1dae..21230cb8e0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; @@ -94,7 +93,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Scanner; -import java.util.concurrent.Executor; import java.util.logging.Level; /** @@ -112,9 +110,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private final AthenzClientFactory athenzClientFactory; @Inject - public ApplicationApiHandler(Executor executor, AccessLog accessLog, Controller controller, Authorizer authorizer, + public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx, + Controller controller, Authorizer authorizer, AthenzClientFactory athenzClientFactory) { - super(executor, accessLog); + super(parentCtx); this.controller = controller; this.authorizer = authorizer; this.athenzClientFactory = athenzClientFactory; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 162827cdb99..a9eaaf4048c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.text.Utf8; @@ -20,7 +19,6 @@ import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.util.concurrent.Executor; import java.util.logging.Level; /** @@ -34,8 +32,8 @@ public class ControllerApiHandler extends LoggingRequestHandler { private final ControllerMaintenance maintenance; - public ControllerApiHandler(Executor executor, AccessLog accessLog, ControllerMaintenance maintenance) { - super(executor, accessLog); + public ControllerApiHandler(LoggingRequestHandler.Context parentCtx, ControllerMaintenance maintenance) { + super(parentCtx); this.maintenance = maintenance; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 27b219cd892..8338d341a2b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; @@ -23,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.restapi.Path; import com.yahoo.yolean.Exceptions; import java.util.Optional; -import java.util.concurrent.Executor; import java.util.logging.Level; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; @@ -40,8 +38,8 @@ public class DeploymentApiHandler extends LoggingRequestHandler { private final Controller controller; - public DeploymentApiHandler(Executor executor, AccessLog accessLog, Controller controller) { - super(executor, accessLog); + public DeploymentApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller) { + super(parentCtx); this.controller = controller; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 4d4f01bc1a6..17181950a29 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.slime.Cursor; @@ -29,7 +28,6 @@ import java.io.InputStream; import java.util.List; import java.util.Optional; import java.util.Scanner; -import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.logging.Logger; @@ -47,8 +45,8 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { private final Controller controller; - public ScrewdriverApiHandler(Executor executor, AccessLog accessLog, Controller controller) { - super(executor, accessLog); + public ScrewdriverApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller) { + super(parentCtx); this.controller = controller; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 282dd79b317..f38ea14bbd8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -18,7 +17,6 @@ import com.yahoo.yolean.Exceptions; import java.util.Comparator; import java.util.List; -import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.stream.Collectors; @@ -32,8 +30,8 @@ public class ZoneApiHandler extends LoggingRequestHandler { private final ZoneRegistry zoneRegistry; - public ZoneApiHandler(Executor executor, AccessLog accessLog, ZoneRegistry zoneRegistry) { - super(executor, accessLog); + public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ZoneRegistry zoneRegistry) { + super(parentCtx); this.zoneRegistry = zoneRegistry; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 68dc2325687..21c7000ef6a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -18,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.yolean.Exceptions; import java.io.IOException; -import java.util.concurrent.Executor; import java.util.logging.Level; /** @@ -34,9 +32,9 @@ public class ZoneApiHandler extends LoggingRequestHandler { private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; - public ZoneApiHandler(Executor executor, AccessLog accessLog, ZoneRegistry zoneRegistry, + public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ZoneRegistry zoneRegistry, ConfigServerRestExecutor proxy) { - super(executor, accessLog); + super(parentCtx); this.zoneRegistry = zoneRegistry; this.proxy = proxy; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index bf7f19a996c..e966cff652b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -201,28 +200,4 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS return endpoints.getOrDefault(endpoint, result); } - @Override - public NodeList getNodeList(DeploymentId deployment) { - NodeList list = new NodeList(); - list.nodes = new ArrayList<>(); - NodeList.Node hostA = new NodeList.Node(); - hostA.hostname = "hostA"; - hostA.cost = 10; - hostA.flavor = "C-2B/24/500"; - hostA.membership = new NodeList.Node.Membership(); - hostA.membership.clusterId = "clusterA"; - hostA.membership.clusterType = "container"; - list.nodes.add(hostA); - - NodeList.Node hostB = new NodeList.Node(); - hostB.hostname = "hostB"; - hostB.cost = 20; - hostB.flavor = "C-2C/24/500"; - hostB.membership = new NodeList.Node.Membership(); - hostB.membership.clusterId = "clusterB"; - hostB.membership.clusterType = "content"; - list.nodes.add(hostB); - - return list; - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index b3ca5491e91..06bde36afc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -241,6 +241,7 @@ public final class ControllerTester { new MemoryGlobalRoutingService(), zoneRegistryMock, configServerClientMock, + new NodeRepositoryClientMock(), new MockMetricsService(), nameService, new MockRoutingGenerator(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryClientMock.java new file mode 100644 index 00000000000..f73f2e992d9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryClientMock.java @@ -0,0 +1,125 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.MaintenanceJobList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +/** + * @author bjorncs + */ +public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { + @Override + public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public NodeRepositoryNode getNode(ZoneId zone, String hostname) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteNode(ZoneId zone, String hostname) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public NodeList listNodes(ZoneId zone, boolean recursive) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) throws IOException { + NodeRepositoryNode nodeA = createNodeA(); + NodeRepositoryNode nodeB = createNodeB(); + return new NodeList(Arrays.asList(nodeA, nodeB)); + } + + private static NodeRepositoryNode createNodeA() { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname("hostA"); + node.setCost(10); + node.setFlavor("C-2B/24/500"); + NodeMembership membership = new NodeMembership(); + membership.clusterid = "clusterA"; + membership.clustertype = "container"; + node.setMembership(membership); + return node; + } + + private static NodeRepositoryNode createNodeB() { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname("hostB"); + node.setCost(20); + node.setFlavor("C-2C/24/500"); + NodeMembership membership = new NodeMembership(); + membership.clusterid = "clusterB"; + membership.clustertype = "content"; + node.setMembership(membership); + return node; + } + + @Override + public String resetFailureInformation(ZoneId zone, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String restart(ZoneId zone, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String reboot(ZoneId zone, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String cancelReboot(ZoneId zone, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String wantTo(ZoneId zone, String nodename, WantTo... actions) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String cancelRestart(ZoneId zone, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void setState(ZoneId zone, NodeState nodeState, String nodename) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String enableMaintenanceJob(ZoneId zone, String jobName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String disableMaintenanceJob(ZoneId zone, String jobName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public MaintenanceJobList listMaintenanceJobs(ZoneId zone) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 631ceab98a5..92753ff3205 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -57,6 +57,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerClientMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.NodeRepositoryClientMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.ZoneRegistryMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerProxyMock'/>\n" + diff --git a/dist/.tito/tito.props b/dist/.tito/tito.props index db2c33744dc..80642a6b87d 100644 --- a/dist/.tito/tito.props +++ b/dist/.tito/tito.props @@ -1,6 +1,6 @@ [buildconfig] builder = tito.builder.Builder tagger = tito.tagger.VersionTagger -tag_format = {component}-{version} +tag_format = {component}-{version}-{release} changelog_do_not_remove_cherrypick = 0 -changelog_format = %s (%ae)
\ No newline at end of file +changelog_format = %s (%ae) diff --git a/dist/release-vespa-rpm.sh b/dist/release-vespa-rpm.sh index 0317e3aebc0..c21550215f0 100755 --- a/dist/release-vespa-rpm.sh +++ b/dist/release-vespa-rpm.sh @@ -41,7 +41,7 @@ git push -u origin --follow-tags $RPM_BRANCH curl -X POST \ -H "Content-type: application/json" \ -H "X-GitHub-Event: create" \ - -d '{ "ref_type": "tag", "repository": { "clone_url": "https://github.com/vespa-engine/vespa.git" } }' \ + -d '{ "ref": "rpmbuild", "ref_type": "branch", "repository": { "clone_url": "https://github.com/vespa-engine/vespa.git" } }' \ https://copr.fedorainfracloud.org/webhooks/github/8037/d1dd5867-b493-4647-a888-0c887e6087b3/ git reset --hard HEAD diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp Binary files differindex d4c7c5fbbe5..c0b2b3a165a 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__cpp +++ b/document/src/test/resources/tensor/multi_cell_tensor__cpp diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5de006cd17c..eb69a1492bf 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -107,7 +107,7 @@ public class FileDownloader { } else if (!file.canRead()) { throw new RuntimeException("File with reference '" + fileReference.value() + "'exists, but unable to read it"); } else { - fileReferenceDownloader.setDownloadStatus(fileReference, 100.0); + fileReferenceDownloader.setDownloadStatus(fileReference, 1.0); return Optional.of(file); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index d3715a1ff89..6fac2becf1b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -39,7 +39,7 @@ public class FileReferenceDownloader { Executors.newFixedThreadPool(10, new DaemonThreadFactory("filereference downloader")); private final ConnectionPool connectionPool; private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>(); - private final Map<FileReference, Double> downloadStatus = new HashMap<>(); + private final Map<FileReference, Double> downloadStatus = new HashMap<>(); // between 0 and 1 private final Duration downloadTimeout; private final FileReceiver fileReceiver; diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index d2da020539a..dc19c7521a9 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -72,7 +72,7 @@ public class FileDownloaderTest { assertEquals("content", IOUtils.readFile(pathToFile.get())); // Verify download status when downloaded - assertDownloadStatus(fileDownloader, fileReference, 100.0); + assertDownloadStatus(fileDownloader, fileReference, 1.0); } { @@ -109,7 +109,7 @@ public class FileDownloaderTest { assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); // Verify download status when downloaded - assertDownloadStatus(fileDownloader, fileReference, 100.0); + assertDownloadStatus(fileDownloader, fileReference, 1.0); } { @@ -143,7 +143,7 @@ public class FileDownloaderTest { assertEquals("bar", IOUtils.readFile(downloadedBar)); // Verify download status when downloaded - assertDownloadStatus(fileDownloader, fileReference, 100.0); + assertDownloadStatus(fileDownloader, fileReference, 1.0); } } @@ -174,7 +174,7 @@ public class FileDownloaderTest { assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); // Verify download status when downloaded - assertDownloadStatus(fileDownloader, fileReference, 100.0); + assertDownloadStatus(fileDownloader, fileReference, 1.0); assertEquals(timesToFail, responseHandler.failedTimes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommand.java new file mode 100644 index 00000000000..87bb5fddf23 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommand.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables; + +import java.net.Inet6Address; +import java.net.InetAddress; + +/** + * Creates two commands that: + * + * 1. replaces an external/public destination ip to an internal/private ip before routing it (pre-routing) + * 2. replaces an internal/private source ip to an external/public ip before writing it on the wire (post-routing) + * + * @author smorgrav + */ +public class NATCommand implements Command { + + private final String snatCommand; + private final String dnatCommand; + + NATCommand(InetAddress externalIp, InetAddress internalIp, String iface) { + String command = externalIp instanceof Inet6Address ? "ip6tables" : "iptables"; + this.snatCommand = String.format("%s -t nat -A POSTROUTING -o %s -s %s -j SNAT --to %s", + command, + iface, + internalIp.getHostAddress(), + externalIp.getHostAddress()); + + this.dnatCommand = String.format("%s -t nat -A PREROUTING -i %s -d %s -j DNAT --to-destination %s", + command, + iface, + externalIp.getHostAddress(), + internalIp.getHostAddress()); + } + + @Override + public String asString() { + return snatCommand + "; " + dnatCommand; + } + + @Override + public String asString(String commandName) { return asString(); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java index 42b36d95374..03217c85329 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -37,10 +37,10 @@ public class RestApiHandler extends LoggingRequestHandler{ private final MetricReceiverWrapper metricReceiverWrapper; @Inject - public RestApiHandler(Executor executor, AccessLog accessLog, + public RestApiHandler(LoggingRequestHandler.Context parentCtx, NodeAdminStateUpdater nodeAdminStateUpdater, MetricReceiverWrapper metricReceiverWrapper) { - super(executor, accessLog); + super(parentCtx); this.refresher = nodeAdminStateUpdater; this.metricReceiverWrapper = metricReceiverWrapper; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommandTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommandTest.java new file mode 100644 index 00000000000..c2a2575f6b1 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/iptables/NATCommandTest.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Test DNAT and SNAT Commands + * + * @author smorgrav + */ +public class NATCommandTest { + + @Test + public void sampleNATCommandIPv6() throws UnknownHostException{ + InetAddress externalIP = Inet6Address.getByName("2001:db8::1"); + InetAddress internalIP = Inet6Address.getByName("2001:db8::2"); + String iface = "eth0"; + + NATCommand command = new NATCommand(externalIP, internalIP, iface); + Assert.assertEquals("ip6tables -t nat -A POSTROUTING -o eth0 -s 2001:db8:0:0:0:0:0:2 -j SNAT --to 2001:db8:0:0:0:0:0:1; ip6tables -t nat -A PREROUTING -i eth0 -d 2001:db8:0:0:0:0:0:1 -j DNAT --to-destination 2001:db8:0:0:0:0:0:2", command.asString()); + } + + @Test + public void sampleNATCommandIPv4() throws UnknownHostException{ + InetAddress externalIP = Inet4Address.getByName("192.168.0.1"); + InetAddress internalIP = Inet4Address.getByName("192.168.0.2"); + String iface = "eth0"; + + NATCommand command = new NATCommand(externalIP, internalIP, iface); + Assert.assertEquals("iptables -t nat -A POSTROUTING -o eth0 -s 192.168.0.2 -j SNAT --to 192.168.0.1; iptables -t nat -A PREROUTING -i eth0 -d 192.168.0.1 -j DNAT --to-destination 192.168.0.2", command.asString()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index b47b3544d17..b188ae7fb20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -38,6 +38,7 @@ import java.util.Set; import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.stream.Collectors; +import javax.inject.Inject; import static com.yahoo.vespa.config.SlimeUtils.optionalString; @@ -55,11 +56,11 @@ public class NodesApiHandler extends LoggingRequestHandler { private final NodeFlavors nodeFlavors; private static final String nodeTypeKey = "type"; - - public NodesApiHandler(Executor executor, AccessLog accessLog, Orchestrator orchestrator, + @Inject + public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, NodeRepositoryMaintenance maintenance, NodeFlavors flavors) { - super(executor, accessLog); + super(parentCtx); this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; this.maintenance = maintenance; diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 40f9ed9b749..4b126a6d7c3 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -224,15 +224,21 @@ struct Fixture { oldAttrMgr.addReferenceAttribute("parent2_ref"); oldAttrMgr.addReferenceAttribute("parent3_ref"); } - ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay) { - DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter); + ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay, bool useReferences) { + DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter, useReferences); return resolver.resolve(attrMgr, oldAttrMgr, std::shared_ptr<search::IDocumentMetaStoreContext>(), visibilityDelay); } + ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay) { + return resolve(visibilityDelay, true); + } + ImportedAttributesRepo::UP resolveReplay() { + return resolve(fastos::TimeStamp(0), false); + } ImportedAttributesRepo::UP resolve() { return resolve(fastos::TimeStamp(0)); } void teardown() { - DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter); + DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter, false); resolver.teardown(attrMgr); } const IGidToLidMapperFactory *getMapperFactoryPtr(const vespalib::string &attrName) { @@ -277,6 +283,13 @@ TEST_F("require that reference attributes are connected to gid mapper", Fixture) EXPECT_EQUAL(f.factory.get(), f.getMapperFactoryPtr("other_ref")); } +TEST_F("require that reference attributes are not connected to gid mapper during replay", Fixture) +{ + f.resolveReplay(); + EXPECT_EQUAL(static_cast<IGidToLidMapperFactory *>(nullptr), f.getMapperFactoryPtr("ref")); + EXPECT_EQUAL(static_cast<IGidToLidMapperFactory *>(nullptr), f.getMapperFactoryPtr("other_ref")); +} + TEST_F("require that imported attributes are instantiated without search cache as default", Fixture) { auto repo = f.resolve(); diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp index 5932742baf9..9abe8181282 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp @@ -140,12 +140,14 @@ DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManage bool useSearchCache) { auto result = std::make_unique<ImportedAttributesRepo>(); - for (const auto &attr : _importedFieldsCfg.attribute) { - ReferenceAttribute::SP refAttr = getReferenceAttribute(attr.referencefield, attrMgr); - AttributeVector::SP targetAttr = getTargetDocumentDB(refAttr->getName())->getAttribute(attr.targetfield); - ImportedAttributeVector::SP importedAttr = + if (_useReferences) { + for (const auto &attr : _importedFieldsCfg.attribute) { + ReferenceAttribute::SP refAttr = getReferenceAttribute(attr.referencefield, attrMgr); + AttributeVector::SP targetAttr = getTargetDocumentDB(refAttr->getName())->getAttribute(attr.targetfield); + ImportedAttributeVector::SP importedAttr = std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr, documentMetaStore, useSearchCache); - result->add(importedAttr->getName(), importedAttr); + result->add(importedAttr->getName(), importedAttr); + } } return result; } @@ -156,13 +158,15 @@ DocumentDBReferenceResolver::DocumentDBReferenceResolver(const IDocumentDBRefere const document::DocumentType &prevThisDocType, MonitoredRefCount &refCount, - ISequencedTaskExecutor &attributeFieldWriter) + ISequencedTaskExecutor &attributeFieldWriter, + bool useReferences) : _registry(registry), _thisDocType(thisDocType), _importedFieldsCfg(importedFieldsCfg), _prevThisDocType(prevThisDocType), _refCount(refCount), _attributeFieldWriter(attributeFieldWriter), + _useReferences(useReferences), _registrators() { } @@ -177,9 +181,11 @@ DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr, const std::shared_ptr<search::IDocumentMetaStoreContext> &documentMetaStore, fastos::TimeStamp visibilityDelay) { - connectReferenceAttributesToGidMapper(newAttrMgr); detectOldListeners(oldAttrMgr); - listenToGidToLidChanges(newAttrMgr); + if (_useReferences) { + connectReferenceAttributesToGidMapper(newAttrMgr); + listenToGidToLidChanges(newAttrMgr); + } return createImportedAttributesRepo(newAttrMgr, documentMetaStore, (visibilityDelay > 0)); } diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h index fc8b3751ea2..1a2c28580cb 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h @@ -32,6 +32,7 @@ private: const document::DocumentType &_prevThisDocType; MonitoredRefCount &_refCount; search::ISequencedTaskExecutor &_attributeFieldWriter; + bool _useReferences; std::map<vespalib::string, std::unique_ptr<GidToLidChangeRegistrator>> _registrators; GidToLidChangeRegistrator &getRegistrator(const vespalib::string &docTypeName); @@ -49,7 +50,8 @@ public: const ImportedFieldsConfig &importedFieldsCfg, const document::DocumentType &prevThisDocType, MonitoredRefCount &refCount, - search::ISequencedTaskExecutor &attributeFieldWriter); + search::ISequencedTaskExecutor &attributeFieldWriter, + bool useReferences); ~DocumentDBReferenceResolver(); virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h index 1744d99ed00..30415048b06 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h @@ -58,7 +58,6 @@ public: */ bool enterLoadState(); bool enterReplayTransactionLogState(); - bool enterReplaySpoolerState(); bool enterRedoReprocessState(); bool enterApplyLiveConfigState(); bool enterReprocessState(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 4dbff7c8e58..2cc2517fe2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -353,7 +353,7 @@ DocumentDB::applySubDBConfig(const DocumentDBConfig &newConfigSnapshot, auto newDocType = newRepo->getDocumentType(_docTypeName.getName()); assert(newDocType != nullptr); DocumentDBReferenceResolver resolver(*registry, *newDocType, newConfigSnapshot.getImportedFieldsConfig(), - *oldDocType, _refCount, _writeService.attributeFieldWriter()); + *oldDocType, _refCount, _writeService.attributeFieldWriter(), _state.getAllowReconfig()); _subDBs.applyConfig(newConfigSnapshot, *_activeConfigSnapshot, serialNum, params, resolver); } @@ -385,6 +385,9 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum } cmpres = _activeConfigSnapshot->compare(*configSnapshot); } + if (_state.getState() == DDBState::State::APPLY_LIVE_CONFIG) { + cmpres.importedFieldsChanged = true; + } const ReconfigParams params(cmpres); // Save config via config manager if replay is done. bool equalReplayConfig = @@ -496,7 +499,8 @@ DocumentDB::tearDownReferences() activeConfig->getImportedFieldsConfig(), *docType, _refCount, - _writeService.attributeFieldWriter()); + _writeService.attributeFieldWriter(), + false); _subDBs.tearDownReferences(resolver); registry->remove(_docTypeName.getName()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp index c2e393cd99b..4bece3e6860 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp @@ -71,7 +71,7 @@ void getStatus(TransLogClient & client, size_t & count) { TransLogClient::Session::UP session = client.open(domainName); - if (session.get() == NULL) { + if ( ! session) { throw IllegalStateException( make_string( "Could not open session with domain '%s' on TLS '%s'", @@ -123,9 +123,9 @@ TransactionLogManager::startReplay(SerialNum first, SerialNum syncToken, TransLogClient::Session::Callback &callback) { - assert(_visitor.get() == NULL); + assert( !_visitor); _visitor = createTlcVisitor(callback); - if (_visitor.get() == NULL) { + if (!_visitor) { throw IllegalStateException( make_string( "Could not create visitor for " @@ -153,12 +153,10 @@ TransactionLogManager::startReplay(SerialNum first, void TransactionLogManager::replayDone() { - assert(_visitor.get() != NULL); - LOG(debug, - "Transaction log replayed for domain '%s'", getDomainName().c_str()); + assert(_visitor); + LOG(debug, "Transaction log replayed for domain '%s'", getDomainName().c_str()); changeReplayDone(); - LOG(debug, - "Broadcasted replay done for domain '%s'", getDomainName().c_str()); + LOG(debug, "Broadcasted replay done for domain '%s'", getDomainName().c_str()); if (LOG_WOULD_LOG(event)) { logReplayComplete(); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 03785258f28..88c2dd9ecc3 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/closuretask.h> #include <vespa/fastos/file.h> #include <algorithm> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.domain"); @@ -18,6 +19,7 @@ using vespalib::Monitor; using vespalib::MonitorGuard; using search::common::FileHeaderContext; using std::runtime_error; +using namespace std::chrono_literals; namespace search::transactionlog { @@ -195,7 +197,7 @@ Domain::triggerSyncNow() if (!_pendingSync) { _pendingSync = true; DomainPart::SP dp(_parts.rbegin()->second); - _sessionExecutor.execute(Sync::UP(new Sync(_syncMonitor, dp, _pendingSync))); + _commitExecutor.execute(Sync::UP(new Sync(_syncMonitor, dp, _pendingSync))); } } @@ -354,16 +356,17 @@ int Domain::closeSession(int sessionId) if (found != _sessions.end()) { sessionRunTime = (std::chrono::steady_clock::now() - found->second->getStartTime()); retval = 1; - _sessionExecutor.sync(); } } - if (retval == 1) { - FastOS_Thread::Sleep(10); + while (retval == 1) { + std::this_thread::sleep_for(10ms); LockGuard guard(_sessionLock); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { - _sessions.erase(sessionId); - retval = 0; + if ( ! found->second->isVisitRunning()) { + _sessions.erase(sessionId); + retval = 0; + } } else { retval = 0; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index 6f0ab998e26..cbcbc68fdff 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -3,7 +3,6 @@ #include "domain.h" #include <vespa/fnet/frt/supervisor.h> #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/vespalib/util/closuretask.h> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.session"); @@ -22,22 +21,17 @@ Session::createTask(const Session::SP & session) return Task::UP(new VisitTask(session)); } -void -Session::VisitTask::run() +Session::VisitTask::VisitTask(const Session::SP & session) + : _session(session) { - _session->visitOnly(); + _session->startVisit(); } +Session::VisitTask::~VisitTask() = default; void -Session::SendTask::run() -{ - _session->sendPending(); -} - -bool -Session::inSync() const +Session::VisitTask::run() { - return _inSync; + _session->visitOnly(); } bool @@ -49,7 +43,7 @@ Session::visit(FastOS_FileInterface & file, DomainPart & dp) { } else { more = dp.visit(_range, packet); } - if (packet.getHandle().size() > 0) { + if ( ! packet.getHandle().empty()) { send(packet); } return more; @@ -78,11 +72,17 @@ Session::visit() } void +Session::startVisit() { + assert(!_visitRunning); + _visitRunning = true; +} +void Session::visitOnly() { visit(); sendDone(); finalize(); + _visitRunning = false; } bool Session::finished() const { @@ -90,42 +90,6 @@ bool Session::finished() const { } void -Session::enQ(const SP & session, SerialNum serial, const Packet & packet) -{ - LockGuard guard(session->_lock); - session->_packetQ.push_back(QPacket(serial,packet)); - if (session->_inSync) { - session->_domain->execute(Task::UP(new SendTask(session))); - } -} - -void -Session::sendPending() -{ - for (;;) { - QPacket packet; - { - LockGuard guard(_lock); - if (_packetQ.empty() || !ok()) - break; - packet = std::move(_packetQ.front()); - _packetQ.pop_front(); - } - sendPacket(packet._serial, *packet._packet); - } -} - -void -Session::sendPacket(SerialNum serial, const Packet & packet) -{ - if (_range.from() < serial) { - send(packet); - } else { - LOG(debug, "[%d] : Skipping %" PRIu64 ". Last sent is %" PRIu64, _id, serial, _range.from()); - } -} - -void Session::finalize() { if (!ok()) { @@ -172,21 +136,6 @@ Session::RequestDone(FRT_RPCRequest * req) req->SubRef(); } -int32_t -Session::rpcAsync(FRT_RPCRequest * req) -{ - int32_t retval(-7); - LOG(debug, "rpcAsync %s starting.", req->GetMethodName()); - FRT_Supervisor::InvokeAsync(_supervisor.GetTransport(), _connection, req, NEVER, this); - if (ok()) { - LOG(debug, "rpcAsync %s OK", req->GetMethodName()); - retval = 0; - } else { - LOG(warning, "rpcAsync %s FAILED", req->GetMethodName()); - } - return retval; -} - Session::Session(int sId, const SerialNumRange & r, const Domain::SP & d, FRT_Supervisor & supervisor, FNET_Connection *conn) : _supervisor(supervisor), @@ -194,12 +143,11 @@ Session::Session(int sId, const SerialNumRange & r, const Domain::SP & d, _domain(d), _range(r), _id(sId), - _inSync(false), _ok(true), + _visitRunning(false), + _inSync(false), _finished(false), - _packetQ(), - _startTime(), - _lock() + _startTime() { _connection->AddRef(); } @@ -217,22 +165,18 @@ Session::send(const Packet & packet) req->GetParams()->AddString(_domain->name().c_str()); req->GetParams()->AddInt32(id()); req->GetParams()->AddData(packet.getHandle().c_str(), packet.getHandle().size()); - return send(req, true); + return send(req); } bool -Session::send(FRT_RPCRequest * req, bool wait) +Session::send(FRT_RPCRequest * req) { - int32_t retval(-1); - if (wait) { - retval = rpc(req); - if ( ! ((retval == RPC::OK) || (retval == FRTE_RPC_CONNECTION)) ) { - LOG(error, "Return value != OK(%d) in send for method 'visitCallback'.", retval); - } - req->SubRef(); - } else { - retval = rpcAsync(req); + int32_t retval = rpc(req); + if ( ! ((retval == RPC::OK) || (retval == FRTE_RPC_CONNECTION)) ) { + LOG(error, "Return value != OK(%d) in send for method 'visitCallback'.", retval); } + req->SubRef(); + return (retval == RPC::OK); } @@ -243,8 +187,7 @@ Session::sendDone() req->SetMethodName("eofCallback"); req->GetParams()->AddString(_domain->name().c_str()); req->GetParams()->AddInt32(id()); - bool retval(send(req, true)); - LockGuard guard(_lock); + bool retval(send(req)); _inSync = true; return retval; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.h b/searchlib/src/vespa/searchlib/transactionlog/session.h index c42d6839dfa..29038ec5290 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.h +++ b/searchlib/src/vespa/searchlib/transactionlog/session.h @@ -30,61 +30,43 @@ public: ~Session(); const SerialNumRange & range() const { return _range; } int id() const { return _id; } - bool inSync() const; + bool inSync() const { return _inSync; } bool ok() const { return _ok; } bool finished() const; - static void enQ(const SP & session, SerialNum serial, const Packet & packet); static Task::UP createTask(const Session::SP & session); void setStartTime(time_point startTime) { _startTime = startTime; } time_point getStartTime() const { return _startTime; } + bool isVisitRunning() const { return _visitRunning; } private: - struct QPacket { - QPacket() : _serial(0), _packet() {} - QPacket(SerialNum s, const Packet & p) - : _serial(s), - _packet(new Packet(p)) - { } - SerialNum _serial; - std::unique_ptr<Packet> _packet; - }; class VisitTask : public Task { public: - VisitTask(const Session::SP & session) : _session(session) { } + VisitTask(const Session::SP & session); + ~VisitTask(); private: void run() override; Session::SP _session; }; - class SendTask : public Task { - public: - SendTask(const Session::SP & session) : _session(session) { } - void run() override; - private: - Session::SP _session; - }; - bool send(FRT_RPCRequest * req, bool wait); + bool send(FRT_RPCRequest * req); void RequestDone(FRT_RPCRequest *req) override; bool send(const Packet & packet); - void sendPacket(SerialNum serial, const Packet & packet); bool sendDone(); - void sendPending(); void visit(); void visitOnly(); + void startVisit(); void finalize(); bool visit(FastOS_FileInterface & file, DomainPart & dp) __attribute__((noinline)); int32_t rpc(FRT_RPCRequest * req); - int32_t rpcAsync(FRT_RPCRequest * req); FRT_Supervisor & _supervisor; FNET_Connection * _connection; DomainSP _domain; SerialNumRange _range; int _id; - bool _inSync; bool _ok; - bool _finished; - std::deque<QPacket> _packetQ; + std::atomic<bool> _visitRunning; + std::atomic<bool> _inSync; + std::atomic<bool> _finished; time_point _startTime; - vespalib::Lock _lock; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index e793aafd38f..4c3c5609a93 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -147,6 +147,10 @@ TransLogServer::~TransLogServer() { stop(); join(); + _commitExecutor.shutdown(); + _commitExecutor.sync(); + _sessionExecutor.shutdown(); + _sessionExecutor.sync(); _supervisor->ShutDown(true); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index c12e37dd1c8..d78d3d39887 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -28,12 +28,11 @@ public: const common::FileHeaderContext &fileHeaderContext, uint64_t domainPartSize); TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext); - virtual ~TransLogServer(); + ~TransLogServer() override; DomainStats getDomainStats() const; void commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) override; - class Session { bool _down; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 5e0fea8ab7d..d5d6a0dc40b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.jdisc.Metric; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -60,9 +61,11 @@ public class RestApi extends LoggingRequestHandler { private final AtomicInteger threadsAvailableForApi; @Inject - public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig, - LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, MetricReceiver metricReceiver) { - super(executor, accessLog); + public RestApi(LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, + LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, + MetricReceiver metricReceiver) + { + super(parentCtx); MessageBusParams params = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); params.setDocumentmanagerConfig(documentManagerConfig); this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(params), metricReceiver); @@ -82,7 +85,7 @@ public class RestApi extends LoggingRequestHandler { AccessLog accessLog, OperationHandler operationHandler, int threadsAvailable) { - super(executor, accessLog); + super(executor, accessLog, null); this.operationHandler = operationHandler; this.threadsAvailableForApi = new AtomicInteger(threadsAvailable); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index ddc652dedbb..61ff84cfdaa 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -61,24 +61,22 @@ public class FeedHandler extends LoggingRequestHandler { @Inject public FeedHandler( - Executor executor, + LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, SessionCache sessionCache, - Metric metric, - AccessLog accessLog, ThreadpoolConfig threadpoolConfig, MetricReceiver metricReceiver) throws Exception { - super(executor, accessLog, metric); + super(parentCtx); DocumentApiMetrics metricsHelper = new DocumentApiMetrics(metricReceiver, "vespa.http.server"); - feedHandlerV3 = new FeedHandlerV3(executor, documentManagerConfig, sessionCache, metric, accessLog, threadpoolConfig, metricsHelper); + feedHandlerV3 = new FeedHandlerV3(parentCtx, documentManagerConfig, sessionCache, threadpoolConfig, metricsHelper); docTypeManager = createDocumentManager(documentManagerConfig); clients = new HashMap<>(); this.sessionCache = sessionCache; sessionId = new AtomicLong(new Random(System.currentTimeMillis()).nextLong()); - feedReplyHandler = new FeedReplyReader(metric, metricsHelper); + feedReplyHandler = new FeedReplyReader(parentCtx.getMetric(), metricsHelper); cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron")); cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES); - this.metric = metric; + this.metric = parentCtx.getMetric(); this.localHostname = resolveLocalHostname(); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 28bcb50a144..4cb82e74db6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -49,20 +49,18 @@ public class FeedHandlerV3 extends LoggingRequestHandler { private static final Logger log = Logger.getLogger(FeedHandlerV3.class.getName()); public FeedHandlerV3( - Executor executor, + LoggingRequestHandler.Context parentCtx, DocumentmanagerConfig documentManagerConfig, SessionCache sessionCache, - Metric metric, - AccessLog accessLog, ThreadpoolConfig threadpoolConfig, DocumentApiMetrics metricsHelper) throws Exception { - super(executor, accessLog, metric); + super(parentCtx); docTypeManager = new DocumentTypeManager(documentManagerConfig); this.sessionCache = sessionCache; - feedReplyHandler = new FeedReplyReader(metric, metricsHelper); + feedReplyHandler = new FeedReplyReader(parentCtx.getMetric(), metricsHelper); cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron")); cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES); - this.metric = metric; + this.metric = parentCtx.getMetric(); // 40% of the threads can be blocking on feeding before we deny requests. if (threadpoolConfig != null) { threadsAvailableForFeeding = new AtomicInteger(Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1)); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java index a3d4da1c60c..c0cc907c671 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java @@ -39,7 +39,10 @@ public class FeedHandlerTest { private final CountDownLatch countDownLatch = new CountDownLatch(1); public TestFeedHandler() throws Exception { - super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null, MetricReceiver.nullImplementation); + super(new FeedHandler.Context(Executors.newCachedThreadPool(), + mock(AccessLog.class), + mock(Metric.class)), + null, null, null, MetricReceiver.nullImplementation); } /** diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java index e237978878e..b5c3998432c 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -101,7 +101,11 @@ public class FeedTesterV3 { Executor threadPool = Executors.newCachedThreadPool(); DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( - threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null, new DocumentApiMetrics(MetricReceiver.nullImplementation, "test")) { + new FeedHandlerV3.Context(threadPool, AccessLog.voidAccessLog(), new NullFeedMetric()), + docMan, + null /* session cache */, + null /* thread pool config */, + new DocumentApiMetrics(MetricReceiver.nullImplementation, "test")) { @Override protected ReferencedResource<SharedSourceSession> retainSource( SessionCache sessionCache, SourceSessionParams sessionParams) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java index f1cf9b2cfa1..47745b29032 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java @@ -60,7 +60,10 @@ public class V2ErrorsInResultTestCase { private static class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); + super(new FeedHandler.Context(executor, + AccessLog.voidAccessLog(), + new DummyMetric()), + null, null, null, MetricReceiver.nullImplementation); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java index a0eb1323c53..9960d98f7f1 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java @@ -115,7 +115,10 @@ public class V2ExternalFeedTestCase { volatile DataFormat lastFormatSeen; public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); + super(new FeedHandler.Context(executor, + AccessLog.voidAccessLog(), + new DummyMetric()), + null, null, null, MetricReceiver.nullImplementation); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java index 8e8da1bed99..6290c22f694 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java @@ -61,7 +61,8 @@ public class V2FailingMessagebusTestCase { private class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); + super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), + null, null, null, MetricReceiver.nullImplementation); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java index c044a5963cb..633477dcc79 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java @@ -58,7 +58,8 @@ public class V2NoXmlReaderTestCase { private static class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); + super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), + null, null, null, MetricReceiver.nullImplementation); } |